8815: Crunch-job bind mounts crunchrunner binary and certificates from host.

[arvados.git] / sdk / cli / bin / crunch-job

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 3425519ba6f7cb4bbe56bc8373c7bd7b54d25120..88f9d561b46c41b4f31ea2efc1cfc225869e453b 100755 (executable)
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -109,6 +109,9 @@ unless (defined $ENV{"CRUNCH_TMP"}) {
   }
 }
 
+$ENV{"HOST_CRUNCHRUNNER_BIN"} ||= `which crunchrunner`;
+$ENV{"HOST_CERTS"} ||= "/etc/ssl/certs/ca-certificates.crt";
+
 # Create the tmp directory if it does not exist
 if ( ! -d $ENV{"CRUNCH_TMP"} ) {
   make_path $ENV{"CRUNCH_TMP"} or die "Failed to create temporary working directory: " . $ENV{"CRUNCH_TMP"};
@@ -432,7 +435,7 @@ fi
 
   # Determine whether this version of Docker supports memory+swap limits.
   ($exited, $stdout, $stderr) = srun_sync(
-    ["srun", "--nodelist=" . $node[0]],
+    ["srun", "--nodes=1"],
     [$docker_bin, 'run', '--help'],
     {label => "check --memory-swap feature"});
   $docker_limitmem = ($stdout =~ /--memory-swap/);
@@ -455,7 +458,7 @@ fi
       $try_user_arg = "--user=$try_user";
     }
     my ($exited, $stdout, $stderr) = srun_sync(
-      ["srun", "--nodelist=" . $node[0]],
+      ["srun", "--nodes=1"],
       ["/bin/sh", "-ec",
        "$docker_bin run $docker_run_args $try_user_arg $docker_hash id --user"],
       {label => $label});
@@ -917,6 +920,11 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
       # For now, use the same approach as TASK_WORK above.
       $ENV{"JOB_WORK"} = "/tmp/crunch-job-work";
 
+      # Bind mount the crunchrunner binary and host TLS certificates file into
+      # the container.
+      $command .= "--volume=\Q$ENV{HOST_CRUNCHRUNNER_BIN}:/usr/lib/crunchrunner/crunchrunner\E ";
+      $command .= "--volume=\Q$ENV{HOST_CERTS}:/usr/lib/crunchrunner/ca-certificates.crt\E ";
+
       while (my ($env_key, $env_val) = each %ENV)
       {
         if ($env_key =~ /^(ARVADOS|CRUNCH|JOB|TASK)_/) {
@@ -1237,16 +1245,16 @@ sub reapchildren
     $progress_is_dirty = 1;
   }
 
-  if ($children_reaped > 0 && (scalar(@successful_task_uuids) == $children_reaped))
+  if (scalar(@successful_task_uuids) > 0)
   {
-    Log (undef, sprintf("Reaped %d children and all were successful, checking for new tasks from API server.", $children_reaped));
+    Log (undef, sprintf("%d tasks exited (%d succeeded), checking for new tasks from API server.", $children_reaped, scalar(@successful_task_uuids)));
     # Load new tasks
     my $newtask_list = [];
     my $newtask_results;
     do {
       $newtask_results = api_call(
         "job_tasks/list",
-        'filters' => '[["created_by_job_task_uuid","in",['.join(',',map {"\"$_\""} @successful_task_uuids).']]]',
+        'filters' => [["created_by_job_task_uuid","in",\@successful_task_uuids]],
         'order' => 'qsequence',
         'offset' => scalar(@$newtask_list),
           );