From 9175278147b9aaaa9dc09f73e34e20f07a791c33 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 16 Jan 2015 16:57:49 -0500 Subject: [PATCH] 4904: Moved to services/, finished first draft documentation. --- apps/arv-web/Dockerfile | 27 ------ apps/arv-web/README | 28 ------ doc/_config.yml | 2 + doc/user/topics/arv-web.html.textile.liquid | 90 +++++++++++++++++++ docker/arv-web/Dockerfile | 15 ++++ .../arv-web/apache2_foreground.sh | 0 {apps => docker}/arv-web/apache2_vhost | 0 sdk/python/arvados/events.py | 1 + services/arv-web/README | 6 ++ {apps => services}/arv-web/arv-web.py | 15 +++- .../arv-web/sample-cgi-app/public/.htaccess | 0 .../arv-web/sample-cgi-app/public/index.cgi | 0 .../arv-web/sample-cgi-app/tmp/.keepkeep | 0 .../arv-web/sample-rack-app/config.ru | 0 .../arv-web/sample-rack-app/public/.keepkeep | 0 .../arv-web/sample-rack-app/tmp/.keepkeep | 0 .../sample-static-page/public/index.html | 0 .../arv-web/sample-static-page/tmp/.keepkeep | 0 .../arv-web/sample-wsgi-app/passenger_wsgi.py | 0 .../arv-web/sample-wsgi-app/public/.keepkeep | 0 .../arv-web/sample-wsgi-app/tmp/.keepkeep | 0 21 files changed, 128 insertions(+), 56 deletions(-) delete mode 100644 apps/arv-web/Dockerfile delete mode 100644 apps/arv-web/README create mode 100644 doc/user/topics/arv-web.html.textile.liquid create mode 100644 docker/arv-web/Dockerfile rename {apps => docker}/arv-web/apache2_foreground.sh (100%) rename {apps => docker}/arv-web/apache2_vhost (100%) create mode 100644 services/arv-web/README rename {apps => services}/arv-web/arv-web.py (89%) mode change 100644 => 100755 rename {apps => services}/arv-web/sample-cgi-app/public/.htaccess (100%) rename {apps => services}/arv-web/sample-cgi-app/public/index.cgi (100%) rename {apps => services}/arv-web/sample-cgi-app/tmp/.keepkeep (100%) rename {apps => services}/arv-web/sample-rack-app/config.ru (100%) rename {apps => services}/arv-web/sample-rack-app/public/.keepkeep (100%) rename {apps => services}/arv-web/sample-rack-app/tmp/.keepkeep (100%) rename {apps => services}/arv-web/sample-static-page/public/index.html (100%) rename {apps => services}/arv-web/sample-static-page/tmp/.keepkeep (100%) rename {apps => services}/arv-web/sample-wsgi-app/passenger_wsgi.py (100%) rename {apps => services}/arv-web/sample-wsgi-app/public/.keepkeep (100%) rename {apps => services}/arv-web/sample-wsgi-app/tmp/.keepkeep (100%) diff --git a/apps/arv-web/Dockerfile b/apps/arv-web/Dockerfile deleted file mode 100644 index d0a6abe829..0000000000 --- a/apps/arv-web/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM arvados/base -MAINTAINER Peter Amstutz - -RUN apt-get update -qq -RUN apt-get install -qqy \ - apt-utils git curl procps apache2-mpm-worker \ - libcurl4-openssl-dev apache2-threaded-dev \ - libapr1-dev libaprutil1-dev - -RUN cd /usr/src/arvados/services/api && \ - /usr/local/rvm/bin/rvm-exec default bundle exec passenger-install-apache2-module --auto --languages ruby,python - -RUN cd /usr/src/arvados/services/api && \ - /usr/local/rvm/bin/rvm-exec default bundle exec passenger-install-apache2-module --snippet > /etc/apache2/conf.d/passenger - -ADD apache2_foreground.sh /etc/apache2/foreground.sh - -ADD apache2_vhost /etc/apache2/sites-available/arv-web -RUN \ - mkdir /var/run/apache2 && \ - a2dissite default && \ - a2ensite arv-web && \ - a2enmod rewrite - -EXPOSE 80 - -CMD ["/etc/apache2/foreground.sh"] \ No newline at end of file diff --git a/apps/arv-web/README b/apps/arv-web/README deleted file mode 100644 index 826092d447..0000000000 --- a/apps/arv-web/README +++ /dev/null @@ -1,28 +0,0 @@ -Run a web service from Arvados. - -usage: arv-web.py [-h] --project PROJECT [--port PORT] --image IMAGE - -optional arguments: - -h, --help show this help message and exit - --project PROJECT Project to watch - --port PORT Local bind port - --image IMAGE Docker image to run - - -This queries an Arvados project and FUSE mounts the most recently modified -collection into a temporary directory. It then runs the supplied Docker image -with the collection bind mounted to /mnt inside the container. - -When a new collection is added to the project, or an existing project is -updated, it will detect the change, it will stop the running Docker container, -unmount the old collection, mount the new most recently modified collection, -and restart the Docker container with the new mount. - -The supplied Dockerfile builds a Docker image that runs Apache with /mnt as the -DocumentRoot. It is configured to run web applications based on Python WSGI, -Ruby Rack, CGI, to serve static HTML, or simply browse the contents of the -/public subdirectory of the collection using Apache's default index pages. - -To build the Docker image: - -$ docker build -t arvados/arv-web . diff --git a/doc/_config.yml b/doc/_config.yml index af5160f526..4b33036004 100644 --- a/doc/_config.yml +++ b/doc/_config.yml @@ -40,6 +40,8 @@ navbar: - user/tutorials/tutorial-submit-job.html.textile.liquid - user/topics/tutorial-parallel.html.textile.liquid - user/topics/arv-docker.html.textile.liquid + - Develop a web service: + - user/topics/arv-web.html.textile.liquid - Reference: - user/topics/run-command.html.textile.liquid - user/reference/job-pipeline-ref.html.textile.liquid diff --git a/doc/user/topics/arv-web.html.textile.liquid b/doc/user/topics/arv-web.html.textile.liquid new file mode 100644 index 0000000000..75c7ff229a --- /dev/null +++ b/doc/user/topics/arv-web.html.textile.liquid @@ -0,0 +1,90 @@ +--- +layout: default +navsection: userguide +title: "Using arv-web" +... + +@arv-web@ enables you to run a custom web service using the contents of an Arvados collection. + +h2. Using arv-web + +@arv-web@ is a long running service for managing web services in response to Arvados events. To start, @arv-web@ queries an Arvados project and mounts the most recently modified collection into a temporary directory. It then runs a Docker image with the collection bound to /mnt inside the container. When a new collection is added to the project, or an existing project is updated, it will stop the running Docker container, unmount the old collection, mount the new most recently modified collection, and restart the Docker container with the new mount. + +
+usage: arv-web.py [-h] --project PROJECT [--port PORT] [--image IMAGE]
+
+optional arguments:
+  -h, --help         show this help message and exit
+  --project PROJECT  Project to watch
+  --port PORT        Local bind port
+  --image IMAGE      Docker image to run
+
+ +h2. Docker container + +The @Dockerfile@ in @arvados/docker/arv-web@ builds a Docker image that runs Apache with /mnt as the DocumentRoot. It is configured to run web applications which use Python WSGI, Ruby Rack, or CGI; to serve static HTML; or browse the contents of the @public@ subdirectory of the collection using default Apache index pages. + +To build the Docker image: + + +
~$ cd arvados/docker
+~/arvados/docker$ docker build -t arvados/arv-web arv-web
+
+ + +h2. Running sample applications + +First, in Arvados Workbench, create a new project. Copy the project uuid from the URL bar (this is part section of the URL after @projects/...@) + +Next, to run a Python WSGI web app: + + +
~$ cd arvados/services/arv-web
+~/arvados/services/arv-web$ arv-put --project [zzzzz-j7d0g-yourprojectuuid] --name sample-wsgi-app sample-wsgi-app
+0M / 0M 100.0%
+Collection saved as 'sample-wsgi-app'
+zzzzz-4zz18-ebohzfbzh82qmqy
+~/arvados/services/arv-web$ ./arv-web.py --project [zzzzz-j7d0g-yourprojectuuid] --image arvados/arv-web --port 8888
+INFO:root:Mounting zzzzz-4zz18-ebohzfbzh82qmqy
+INFO:root:Starting docker container
+INFO:root:Container id is e79e70558d585a3e038e4bfbc97e5c511f21b6101443b29a8017bdf3d84689a3
+INFO:root:Waiting for events
+
+ + +The sample application will be available at @http://hostcomputer:8888@. + +If you upload a new collection to the same project, arv-web will restart the web service and serve the new collection. For example, + + +
~$ cd arvados/services/arv-web
+~/arvados/services/arv-web$ arv-put --project [zzzzz-j7d0g-yourprojectuuid] --name sample-rack-app sample-rack-app
+0M / 0M 100.0%
+Collection saved as 'sample-rack-app'
+zzzzz-4zz18-dhhm0ay8k8cqkvg
+
+ + +@arv-web@ will automatically notice the change and restart the service: + +
+INFO:root:Waiting for events
+INFO:root:create 4n8aq-4zz18-dhhm0ay8k8cqkvg
+INFO:root:restarting web service
+INFO:root:Stopping docker container
+e79e70558d585a3e038e4bfbc97e5c511f21b6101443b29a8017bdf3d84689a3
+INFO:root:Unmounting
+INFO:root:Mounting zzzzz-4zz18-dhhm0ay8k8cqkvg
+INFO:root:Starting docker container
+INFO:root:Container id is ec4f5824c0215b94af80b2058b1c3d6adbd0f76f01dfc0cafdb7b8fad2a62677
+INFO:root:Waiting for events
+
+ +h2. Writing your own applications + +Python and Ruby applications are served using Phusion Passenger and Apache @mod_passenger@. See https://www.phusionpassenger.com/documentation/Users%20guide%20Apache.html for details, and look at the sample apps @arvados/services/arv-web/sample-wsgi-app@ and @arvados/services/arv-web/sample-rack-app@. + +You can serve CGI applications using standard Apache CGI support. See +https://httpd.apache.org/docs/current/howto/cgi.html for details, and look at the sample app @arvados/services/arv-web/sample-cgi-app@ + +You can also just serve static content. Look at @arvados/services/arv-web/sample-static-page@ for an example. diff --git a/docker/arv-web/Dockerfile b/docker/arv-web/Dockerfile new file mode 100644 index 0000000000..11a9c17087 --- /dev/null +++ b/docker/arv-web/Dockerfile @@ -0,0 +1,15 @@ +FROM arvados/passenger +MAINTAINER Peter Amstutz + +ADD apache2_foreground.sh /etc/apache2/foreground.sh + +ADD apache2_vhost /etc/apache2/sites-available/arv-web +RUN \ + mkdir /var/run/apache2 && \ + a2dissite default && \ + a2ensite arv-web && \ + a2enmod rewrite + +EXPOSE 80 + +CMD ["/etc/apache2/foreground.sh"] \ No newline at end of file diff --git a/apps/arv-web/apache2_foreground.sh b/docker/arv-web/apache2_foreground.sh similarity index 100% rename from apps/arv-web/apache2_foreground.sh rename to docker/arv-web/apache2_foreground.sh diff --git a/apps/arv-web/apache2_vhost b/docker/arv-web/apache2_vhost similarity index 100% rename from apps/arv-web/apache2_vhost rename to docker/arv-web/apache2_vhost diff --git a/sdk/python/arvados/events.py b/sdk/python/arvados/events.py index d1abc0f7de..f0dd5db393 100644 --- a/sdk/python/arvados/events.py +++ b/sdk/python/arvados/events.py @@ -121,6 +121,7 @@ def subscribe(api, filters, on_event, poll_fallback=15): if poll_fallback: _logger.warn("Websockets not available, falling back to log table polling") p = PollClient(api, filters, on_event, poll_fallback) + p.daemon = True p.start() return p else: diff --git a/services/arv-web/README b/services/arv-web/README new file mode 100644 index 0000000000..eaf7624dc4 --- /dev/null +++ b/services/arv-web/README @@ -0,0 +1,6 @@ +arv-web enables you to run a custom web service using the contents of an +Arvados collection. + +See "Using arv-web" in the Arvados user guide: + +http://doc.arvados.org/user/topics/arv-web.html diff --git a/apps/arv-web/arv-web.py b/services/arv-web/arv-web.py old mode 100644 new mode 100755 similarity index 89% rename from apps/arv-web/arv-web.py rename to services/arv-web/arv-web.py index 63d760d2f1..9b4ccdf911 --- a/apps/arv-web/arv-web.py +++ b/services/arv-web/arv-web.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import arvados import subprocess from arvados_fuse import Operations, SafeApi, CollectionDirectory @@ -16,7 +18,7 @@ logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser() parser.add_argument('--project', type=str, required=True, help="Project to watch") parser.add_argument('--port', type=int, default=8080, help="Local bind port") -parser.add_argument('--image', type=str, required=True, help="Docker image to run") +parser.add_argument('--image', type=str, help="Docker image to run") args = parser.parse_args() @@ -77,9 +79,19 @@ signal.signal(signal.SIGTERM, lambda signal, frame: sys.exit(0)) loop = True cid = None while loop: + loop = False logging.info("Mounting %s" % collection) mountdir = run_fuse_mount(collection) + try: + if not args.image: + if os.path.exists(os.path.join(mountdir, "docker_image")): + with open(os.path.join(mountdir, "docker_image")) as di: + docker_image = di.read().strip() + else: + logging.error("Collection must contain a file 'docker_image' or must specify --image on the command line.") + sys.exit(1) + logging.info("Starting docker container") cid = subprocess.check_output(["docker", "run", "--detach=true", @@ -91,6 +103,7 @@ while loop: logging.info("Waiting for events") running = True + loop = True while running: try: eq = evqueue.get(True, 1) diff --git a/apps/arv-web/sample-cgi-app/public/.htaccess b/services/arv-web/sample-cgi-app/public/.htaccess similarity index 100% rename from apps/arv-web/sample-cgi-app/public/.htaccess rename to services/arv-web/sample-cgi-app/public/.htaccess diff --git a/apps/arv-web/sample-cgi-app/public/index.cgi b/services/arv-web/sample-cgi-app/public/index.cgi similarity index 100% rename from apps/arv-web/sample-cgi-app/public/index.cgi rename to services/arv-web/sample-cgi-app/public/index.cgi diff --git a/apps/arv-web/sample-cgi-app/tmp/.keepkeep b/services/arv-web/sample-cgi-app/tmp/.keepkeep similarity index 100% rename from apps/arv-web/sample-cgi-app/tmp/.keepkeep rename to services/arv-web/sample-cgi-app/tmp/.keepkeep diff --git a/apps/arv-web/sample-rack-app/config.ru b/services/arv-web/sample-rack-app/config.ru similarity index 100% rename from apps/arv-web/sample-rack-app/config.ru rename to services/arv-web/sample-rack-app/config.ru diff --git a/apps/arv-web/sample-rack-app/public/.keepkeep b/services/arv-web/sample-rack-app/public/.keepkeep similarity index 100% rename from apps/arv-web/sample-rack-app/public/.keepkeep rename to services/arv-web/sample-rack-app/public/.keepkeep diff --git a/apps/arv-web/sample-rack-app/tmp/.keepkeep b/services/arv-web/sample-rack-app/tmp/.keepkeep similarity index 100% rename from apps/arv-web/sample-rack-app/tmp/.keepkeep rename to services/arv-web/sample-rack-app/tmp/.keepkeep diff --git a/apps/arv-web/sample-static-page/public/index.html b/services/arv-web/sample-static-page/public/index.html similarity index 100% rename from apps/arv-web/sample-static-page/public/index.html rename to services/arv-web/sample-static-page/public/index.html diff --git a/apps/arv-web/sample-static-page/tmp/.keepkeep b/services/arv-web/sample-static-page/tmp/.keepkeep similarity index 100% rename from apps/arv-web/sample-static-page/tmp/.keepkeep rename to services/arv-web/sample-static-page/tmp/.keepkeep diff --git a/apps/arv-web/sample-wsgi-app/passenger_wsgi.py b/services/arv-web/sample-wsgi-app/passenger_wsgi.py similarity index 100% rename from apps/arv-web/sample-wsgi-app/passenger_wsgi.py rename to services/arv-web/sample-wsgi-app/passenger_wsgi.py diff --git a/apps/arv-web/sample-wsgi-app/public/.keepkeep b/services/arv-web/sample-wsgi-app/public/.keepkeep similarity index 100% rename from apps/arv-web/sample-wsgi-app/public/.keepkeep rename to services/arv-web/sample-wsgi-app/public/.keepkeep diff --git a/apps/arv-web/sample-wsgi-app/tmp/.keepkeep b/services/arv-web/sample-wsgi-app/tmp/.keepkeep similarity index 100% rename from apps/arv-web/sample-wsgi-app/tmp/.keepkeep rename to services/arv-web/sample-wsgi-app/tmp/.keepkeep -- 2.30.2