4904: Moved to services/, finished first draft documentation.

diff --git a/apps/arv-web/Dockerfile b/apps/arv-web/Dockerfile
deleted file mode 100644 (file)
index d0a6abe..0000000
--- a/apps/arv-web/Dockerfile
+++ /dev/null
@@ -1,27 +0,0 @@
-FROM arvados/base
-MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
-
-RUN apt-get update -qq
-RUN apt-get install -qqy \
-        apt-utils git curl procps apache2-mpm-worker \
-        libcurl4-openssl-dev apache2-threaded-dev \
-        libapr1-dev libaprutil1-dev
-
-RUN cd /usr/src/arvados/services/api && \
-    /usr/local/rvm/bin/rvm-exec default bundle exec passenger-install-apache2-module --auto --languages ruby,python
-
-RUN cd /usr/src/arvados/services/api && \
-    /usr/local/rvm/bin/rvm-exec default bundle exec passenger-install-apache2-module --snippet > /etc/apache2/conf.d/passenger
-
-ADD apache2_foreground.sh /etc/apache2/foreground.sh
-
-ADD apache2_vhost /etc/apache2/sites-available/arv-web
-RUN \
-  mkdir /var/run/apache2 && \
-  a2dissite default && \
-  a2ensite arv-web && \
-  a2enmod rewrite
-
-EXPOSE 80
-
-CMD ["/etc/apache2/foreground.sh"]
\ No newline at end of file

diff --git a/apps/arv-web/README b/apps/arv-web/README
deleted file mode 100644 (file)
index 826092d..0000000
--- a/apps/arv-web/README
+++ /dev/null
@@ -1,28 +0,0 @@
-Run a web service from Arvados.
-
-usage: arv-web.py [-h] --project PROJECT [--port PORT] --image IMAGE
-
-optional arguments:
-  -h, --help         show this help message and exit
-  --project PROJECT  Project to watch
-  --port PORT        Local bind port
-  --image IMAGE      Docker image to run
-
-
-This queries an Arvados project and FUSE mounts the most recently modified
-collection into a temporary directory.  It then runs the supplied Docker image
-with the collection bind mounted to /mnt inside the container.
-
-When a new collection is added to the project, or an existing project is
-updated, it will detect the change, it will stop the running Docker container,
-unmount the old collection, mount the new most recently modified collection,
-and restart the Docker container with the new mount.
-
-The supplied Dockerfile builds a Docker image that runs Apache with /mnt as the
-DocumentRoot.  It is configured to run web applications based on Python WSGI,
-Ruby Rack, CGI, to serve static HTML, or simply browse the contents of the
-/public subdirectory of the collection using Apache's default index pages.
-
-To build the Docker image:
-
-$ docker build -t arvados/arv-web .

diff --git a/doc/_config.yml b/doc/_config.yml
index af5160f52602af292cf6470987200510a91dc03b..4b33036004ea27346232da174f6403d133580508 100644 (file)
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -40,6 +40,8 @@ navbar:
       - user/tutorials/tutorial-submit-job.html.textile.liquid
       - user/topics/tutorial-parallel.html.textile.liquid
       - user/topics/arv-docker.html.textile.liquid
       - user/tutorials/tutorial-submit-job.html.textile.liquid
       - user/topics/tutorial-parallel.html.textile.liquid
       - user/topics/arv-docker.html.textile.liquid

+    - Develop a web service:
+      - user/topics/arv-web.html.textile.liquid

     - Reference:
       - user/topics/run-command.html.textile.liquid
       - user/reference/job-pipeline-ref.html.textile.liquid
     - Reference:
       - user/topics/run-command.html.textile.liquid
       - user/reference/job-pipeline-ref.html.textile.liquid

diff --git a/doc/user/topics/arv-web.html.textile.liquid b/doc/user/topics/arv-web.html.textile.liquid
new file mode 100644 (file)
index 0000000..75c7ff2
--- /dev/null
+++ b/doc/user/topics/arv-web.html.textile.liquid
@@ -0,0 +1,90 @@
+---
+layout: default
+navsection: userguide
+title: "Using arv-web"
+...
+
+@arv-web@ enables you to run a custom web service using the contents of an Arvados collection.
+
+h2. Using arv-web
+
+@arv-web@ is a long running service for managing web services in response to Arvados events.  To start, @arv-web@ queries an Arvados project and mounts the most recently modified collection into a temporary directory.  It then runs a Docker image with the collection bound to /mnt inside the container.  When a new collection is added to the project, or an existing project is updated, it will stop the running Docker container, unmount the old collection, mount the new most recently modified collection, and restart the Docker container with the new mount.
+
+<pre>
+usage: arv-web.py [-h] --project PROJECT [--port PORT] [--image IMAGE]
+
+optional arguments:
+  -h, --help         show this help message and exit
+  --project PROJECT  Project to watch
+  --port PORT        Local bind port
+  --image IMAGE      Docker image to run
+</pre>
+
+h2. Docker container
+
+The @Dockerfile@ in @arvados/docker/arv-web@ builds a Docker image that runs Apache with /mnt as the DocumentRoot.  It is configured to run web applications which use Python WSGI, Ruby Rack, or CGI; to serve static HTML; or browse the contents of the @public@ subdirectory of the collection using default Apache index pages.
+
+To build the Docker image:
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd arvados/docker</span>
+~/arvados/docker$ <span class="userinput">docker build -t arvados/arv-web arv-web</span>
+</code></pre>
+</notextile>
+
+h2. Running sample applications
+
+First, in Arvados Workbench, create a new project.  Copy the project uuid from the URL bar (this is part section of the URL after @projects/...@)
+
+Next, to run a Python WSGI web app:
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd arvados/services/arv-web</span>
+~/arvados/services/arv-web$ <span class="userinput">arv-put --project [zzzzz-j7d0g-yourprojectuuid] --name sample-wsgi-app sample-wsgi-app</span>
+0M / 0M 100.0%
+Collection saved as 'sample-wsgi-app'
+zzzzz-4zz18-ebohzfbzh82qmqy
+~/arvados/services/arv-web$ <span class="userinput">./arv-web.py --project [zzzzz-j7d0g-yourprojectuuid] --image arvados/arv-web --port 8888</span>
+INFO:root:Mounting zzzzz-4zz18-ebohzfbzh82qmqy
+INFO:root:Starting docker container
+INFO:root:Container id is e79e70558d585a3e038e4bfbc97e5c511f21b6101443b29a8017bdf3d84689a3
+INFO:root:Waiting for events
+</code></pre>
+</notextile>
+
+The sample application will be available at @http://hostcomputer:8888@.
+
+If you upload a new collection to the same project, arv-web will restart the web service and serve the new collection.  For example,
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd arvados/services/arv-web</span>
+~/arvados/services/arv-web$ <span class="userinput">arv-put --project [zzzzz-j7d0g-yourprojectuuid] --name sample-rack-app sample-rack-app</span>
+0M / 0M 100.0%
+Collection saved as 'sample-rack-app'
+zzzzz-4zz18-dhhm0ay8k8cqkvg
+</code></pre>
+</notextile>
+
+@arv-web@ will automatically notice the change and restart the service:
+
+<pre>
+INFO:root:Waiting for events
+INFO:root:create 4n8aq-4zz18-dhhm0ay8k8cqkvg
+INFO:root:restarting web service
+INFO:root:Stopping docker container
+e79e70558d585a3e038e4bfbc97e5c511f21b6101443b29a8017bdf3d84689a3
+INFO:root:Unmounting
+INFO:root:Mounting zzzzz-4zz18-dhhm0ay8k8cqkvg
+INFO:root:Starting docker container
+INFO:root:Container id is ec4f5824c0215b94af80b2058b1c3d6adbd0f76f01dfc0cafdb7b8fad2a62677
+INFO:root:Waiting for events
+</pre>
+
+h2. Writing your own applications
+
+Python and Ruby applications are served using Phusion Passenger and Apache @mod_passenger@.  See https://www.phusionpassenger.com/documentation/Users%20guide%20Apache.html for details, and look at the sample apps @arvados/services/arv-web/sample-wsgi-app@ and @arvados/services/arv-web/sample-rack-app@.
+
+You can serve CGI applications using standard Apache CGI support.  See
+https://httpd.apache.org/docs/current/howto/cgi.html for details, and look at the sample app @arvados/services/arv-web/sample-cgi-app@
+
+You can also just serve static content.  Look at @arvados/services/arv-web/sample-static-page@ for an example.

diff --git a/docker/arv-web/Dockerfile b/docker/arv-web/Dockerfile
new file mode 100644 (file)
index 0000000..11a9c17
--- /dev/null
+++ b/docker/arv-web/Dockerfile
@@ -0,0 +1,15 @@
+FROM arvados/passenger
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+ADD apache2_foreground.sh /etc/apache2/foreground.sh
+
+ADD apache2_vhost /etc/apache2/sites-available/arv-web
+RUN \
+  mkdir /var/run/apache2 && \
+  a2dissite default && \
+  a2ensite arv-web && \
+  a2enmod rewrite
+
+EXPOSE 80
+
+CMD ["/etc/apache2/foreground.sh"]
\ No newline at end of file

diff --git a/apps/arv-web/apache2_foreground.sh b/docker/arv-web/apache2_foreground.sh
similarity index 100%
rename from apps/arv-web/apache2_foreground.sh
rename to docker/arv-web/apache2_foreground.sh

diff --git a/apps/arv-web/apache2_vhost b/docker/arv-web/apache2_vhost
similarity index 100%
rename from apps/arv-web/apache2_vhost
rename to docker/arv-web/apache2_vhost

diff --git a/sdk/python/arvados/events.py b/sdk/python/arvados/events.py
index d1abc0f7de0cd1a7c35cffb254605be302e85fd1..f0dd5db393751299d2f9bae76fa174325e1b917d 100644 (file)
--- a/sdk/python/arvados/events.py
+++ b/sdk/python/arvados/events.py
@@ -121,6 +121,7 @@ def subscribe(api, filters, on_event, poll_fallback=15):
     if poll_fallback:
         _logger.warn("Websockets not available, falling back to log table polling")
         p = PollClient(api, filters, on_event, poll_fallback)
     if poll_fallback:
         _logger.warn("Websockets not available, falling back to log table polling")
         p = PollClient(api, filters, on_event, poll_fallback)

+        p.daemon = True

         p.start()
         return p
     else:
         p.start()
         return p
     else:

diff --git a/services/arv-web/README b/services/arv-web/README
new file mode 100644 (file)
index 0000000..eaf7624
--- /dev/null
+++ b/services/arv-web/README
@@ -0,0 +1,6 @@
+arv-web enables you to run a custom web service using the contents of an
+Arvados collection.
+
+See "Using arv-web" in the Arvados user guide:
+
+http://doc.arvados.org/user/topics/arv-web.html

diff --git a/apps/arv-web/arv-web.py b/services/arv-web/arv-web.py
old mode 100644 (file)
new mode 100755 (executable)
similarity index 89%
rename from apps/arv-web/arv-web.py
rename to services/arv-web/arv-web.py
index 63d760d..9b4ccdf
--- a/apps/arv-web/arv-web.py
+++ b/services/arv-web/arv-web.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+

 import arvados
 import subprocess
 from arvados_fuse import Operations, SafeApi, CollectionDirectory
 import arvados
 import subprocess
 from arvados_fuse import Operations, SafeApi, CollectionDirectory


@@ -16,7 +18,7 @@ logging.basicConfig(level=logging.INFO)
 parser = argparse.ArgumentParser()
 parser.add_argument('--project', type=str, required=True, help="Project to watch")
 parser.add_argument('--port', type=int, default=8080, help="Local bind port")
 parser = argparse.ArgumentParser()
 parser.add_argument('--project', type=str, required=True, help="Project to watch")
 parser.add_argument('--port', type=int, default=8080, help="Local bind port")

-parser.add_argument('--image', type=str, required=True, help="Docker image to run")
+parser.add_argument('--image', type=str, help="Docker image to run")

 
 args = parser.parse_args()
 
 
 args = parser.parse_args()
 


@@ -77,9 +79,19 @@ signal.signal(signal.SIGTERM, lambda signal, frame: sys.exit(0))
 loop = True
 cid = None
 while loop:
 loop = True
 cid = None
 while loop:

+    loop = False

     logging.info("Mounting %s" % collection)
     mountdir = run_fuse_mount(collection)
     logging.info("Mounting %s" % collection)
     mountdir = run_fuse_mount(collection)

+

     try:
     try:

+        if not args.image:
+            if os.path.exists(os.path.join(mountdir, "docker_image")):
+                with open(os.path.join(mountdir, "docker_image")) as di:
+                    docker_image = di.read().strip()
+            else:
+                logging.error("Collection must contain a file 'docker_image' or must specify --image on the command line.")
+                sys.exit(1)
+

         logging.info("Starting docker container")
         cid = subprocess.check_output(["docker", "run",
                                        "--detach=true",
         logging.info("Starting docker container")
         cid = subprocess.check_output(["docker", "run",
                                        "--detach=true",


@@ -91,6 +103,7 @@ while loop:
 
         logging.info("Waiting for events")
         running = True
 
         logging.info("Waiting for events")
         running = True

+        loop = True

         while running:
             try:
                 eq = evqueue.get(True, 1)
         while running:
             try:
                 eq = evqueue.get(True, 1)

diff --git a/apps/arv-web/sample-cgi-app/public/.htaccess b/services/arv-web/sample-cgi-app/public/.htaccess
similarity index 100%
rename from apps/arv-web/sample-cgi-app/public/.htaccess
rename to services/arv-web/sample-cgi-app/public/.htaccess

diff --git a/apps/arv-web/sample-cgi-app/public/index.cgi b/services/arv-web/sample-cgi-app/public/index.cgi
similarity index 100%
rename from apps/arv-web/sample-cgi-app/public/index.cgi
rename to services/arv-web/sample-cgi-app/public/index.cgi

diff --git a/apps/arv-web/sample-cgi-app/tmp/.keepkeep b/services/arv-web/sample-cgi-app/tmp/.keepkeep
similarity index 100%
rename from apps/arv-web/sample-cgi-app/tmp/.keepkeep
rename to services/arv-web/sample-cgi-app/tmp/.keepkeep

diff --git a/apps/arv-web/sample-rack-app/config.ru b/services/arv-web/sample-rack-app/config.ru
similarity index 100%
rename from apps/arv-web/sample-rack-app/config.ru
rename to services/arv-web/sample-rack-app/config.ru

diff --git a/apps/arv-web/sample-rack-app/public/.keepkeep b/services/arv-web/sample-rack-app/public/.keepkeep
similarity index 100%
rename from apps/arv-web/sample-rack-app/public/.keepkeep
rename to services/arv-web/sample-rack-app/public/.keepkeep

diff --git a/apps/arv-web/sample-rack-app/tmp/.keepkeep b/services/arv-web/sample-rack-app/tmp/.keepkeep
similarity index 100%
rename from apps/arv-web/sample-rack-app/tmp/.keepkeep
rename to services/arv-web/sample-rack-app/tmp/.keepkeep

diff --git a/apps/arv-web/sample-static-page/public/index.html b/services/arv-web/sample-static-page/public/index.html
similarity index 100%
rename from apps/arv-web/sample-static-page/public/index.html
rename to services/arv-web/sample-static-page/public/index.html

diff --git a/apps/arv-web/sample-static-page/tmp/.keepkeep b/services/arv-web/sample-static-page/tmp/.keepkeep
similarity index 100%
rename from apps/arv-web/sample-static-page/tmp/.keepkeep
rename to services/arv-web/sample-static-page/tmp/.keepkeep

diff --git a/apps/arv-web/sample-wsgi-app/passenger_wsgi.py b/services/arv-web/sample-wsgi-app/passenger_wsgi.py
similarity index 100%
rename from apps/arv-web/sample-wsgi-app/passenger_wsgi.py
rename to services/arv-web/sample-wsgi-app/passenger_wsgi.py

diff --git a/apps/arv-web/sample-wsgi-app/public/.keepkeep b/services/arv-web/sample-wsgi-app/public/.keepkeep
similarity index 100%
rename from apps/arv-web/sample-wsgi-app/public/.keepkeep
rename to services/arv-web/sample-wsgi-app/public/.keepkeep

diff --git a/apps/arv-web/sample-wsgi-app/tmp/.keepkeep b/services/arv-web/sample-wsgi-app/tmp/.keepkeep
similarity index 100%
rename from apps/arv-web/sample-wsgi-app/tmp/.keepkeep
rename to services/arv-web/sample-wsgi-app/tmp/.keepkeep