crunch-job: Support runtime Docker image.

This adds support for the allow_other mount option in arv-mount to
make the Keep mount available to the container.

diff --git a/doc/api/schema/Job.html.textile.liquid b/doc/api/schema/Job.html.textile.liquid
index 82f9242db362e5a7408dc4c132370f7f04094b8b..097d0df898bea9d79acd666478d94e59c376de02 100644 (file)
--- a/doc/api/schema/Job.html.textile.liquid
+++ b/doc/api/schema/Job.html.textile.liquid
@@ -50,6 +50,7 @@ h3. Runtime constraints
 
 table(table table-bordered table-condensed).
 |_. Key|_. Type|_. Description|_. Implemented|
+|docker_image|string|The name of a Docker image that this Job needs to run.  If specified, Crunch will create a Docker container from this image, and run the Job's script inside that.  The Keep mount and work directories will be available as volumes inside this container.  You may specify the image in any format that Docker accepts, such as "arvados/jobs" or a hash identifier.  If you specify a name, Crunch will try to install the latest version using @docker.io pull@.|✓|
 |min_nodes|integer||✓|
 |max_nodes|integer|||
 |max_tasks_per_node|integer|Maximum simultaneous tasks on a single node|✓|

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 48a6c9dea7f7f5be8fa20367e46e29de969f5b62..515aaa668dfaeb65c0b2be44340804541d56ffb2 100755 (executable)
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -498,7 +498,9 @@ if (!$have_slurm)
   must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here.");
 }
 
-
+# If this job requires a Docker image, install that.
+my $docker_bin = "/usr/bin/docker.io";
+my $docker_image = $Job->{runtime_constraints}->{docker_image} || "";
 
 foreach (qw (script script_version script_parameters runtime_constraints))
 {
@@ -603,7 +605,6 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
       qw(-n1 -c1 -N1 -D), $ENV{'TMPDIR'},
       "--job-name=$job_id.$id.$$",
        );
-    my @execargs = qw(sh);
     my $build_script_to_send = "";
     my $command =
        "if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; "
@@ -615,8 +616,27 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
       $command .=
          "&& perl -";
     }
-    $command .=
-        "&& exec arv-mount $ENV{TASK_KEEPMOUNT} --exec $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
+    $command .= "&& exec arv-mount --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
+    if ($docker_image)
+    {
+      $command .= "$docker_bin run -i -a stdin -a stdout -a stderr ";
+      # Dynamically configure the container to use the host system as its
+      # DNS server.  Get the host's global addresses from the ip command,
+      # and turn them into docker --dns options using gawk.
+      $command .=
+          q{$(ip -o address show scope global |
+              gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
+      foreach my $env_key (qw(CRUNCH_SRC CRUNCH_TMP TASK_KEEPMOUNT))
+      {
+        $command .= "-v \Q$ENV{$env_key}:$ENV{$env_key}:rw\E ";
+      }
+      while (my ($env_key, $env_val) = each %ENV)
+      {
+        $command .= "-e \Q$env_key=$env_val\E ";
+      }
+      $command .= "$docker_image ";
+    }
+    $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
     my @execargs = ('bash', '-c', $command);
     srun (\@srunargs, \@execargs, undef, $build_script_to_send);
     exit (111);

diff --git a/sdk/python/bin/arv-mount b/sdk/python/bin/arv-mount
index 5e773dfbc6c0185f32b02b76ab3255692a8b4cbe..cc7e28889104e2b0ab2db6480a724c1b343a7f64 100755 (executable)
--- a/sdk/python/bin/arv-mount
+++ b/sdk/python/bin/arv-mount
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-from arvados.fuse import * 
+from arvados.fuse import *
 import arvados
 import subprocess
 import argparse
@@ -15,6 +15,8 @@ mountpoint before --exec, or mark the end of your --exec arguments
 with "--".
 """)
     parser.add_argument('mountpoint', type=str, help="""Mount point.""")
+    parser.add_argument('--allow-other', action='store_true',
+                        help="""Let other users read the mount""")
     parser.add_argument('--collection', type=str, help="""Collection locator""")
     parser.add_argument('--debug', action='store_true', help="""Debug mode""")
     parser.add_argument('--exec', type=str, nargs=argparse.REMAINDER,
@@ -35,12 +37,9 @@ with "--".
         operations.inodes.add_entry(MagicDirectory(llfuse.ROOT_INODE, operations.inodes))
 
     # FUSE options, see mount.fuse(8)
-    opts = []
+    opts = [optname for optname in ['allow_other', 'debug']
+            if getattr(args, optname)]
 
-    # Enable FUSE debugging (logs each FUSE request)
-    if args.debug:
-        opts += ['debug']    
-    
     # Initialize the fuse connection
     llfuse.init(operations, args.mountpoint, opts)