Merge branch '3570-crunch-race' closes #3570

[arvados.git] / sdk / cli / bin / crunch-job

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 1654646e1b27771f7e4c7b31a37e3ca9597fa36e..e7cac1816f890e87d3b2767d0930dd57f3d5562b 100755 (executable)
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -84,6 +84,8 @@ use File::Temp;
 use Fcntl ':flock';
 use File::Path qw( make_path );
 
+use constant EX_TEMPFAIL => 75;
+
 $ENV{"TMPDIR"} ||= "/tmp";
 unless (defined $ENV{"CRUNCH_TMP"}) {
   $ENV{"CRUNCH_TMP"} = $ENV{"TMPDIR"} . "/crunch-job";
@@ -150,17 +152,25 @@ if ($job_has_uuid)
 {
   $Job = $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
   if (!$force_unlock) {
+    # If some other crunch-job process has grabbed this job (or we see
+    # other evidence that the job is already underway) we exit
+    # EX_TEMPFAIL so crunch-dispatch (our parent process) doesn't
+    # mark the job as failed.
     if ($Job->{'is_locked_by_uuid'}) {
-      croak("Job is locked: " . $Job->{'is_locked_by_uuid'});
+      Log(undef, "Job is locked by " . $Job->{'is_locked_by_uuid'});
+      exit EX_TEMPFAIL;
     }
     if ($Job->{'success'} ne undef) {
-      croak("Job 'success' flag (" . $Job->{'success'} . ") is not null");
+      Log(undef, "Job 'success' flag (" . $Job->{'success'} . ") is not null");
+      exit EX_TEMPFAIL;
     }
     if ($Job->{'running'}) {
-      croak("Job 'running' flag is already set");
+      Log(undef, "Job 'running' flag is already set");
+      exit EX_TEMPFAIL;
     }
     if ($Job->{'started_at'}) {
-      croak("Job 'started_at' time is already set (" . $Job->{'started_at'} . ")");
+      Log(undef, "Job 'started_at' time is already set (" . $Job->{'started_at'} . ")");
+      exit EX_TEMPFAIL;
     }
   }
 }
@@ -273,7 +283,8 @@ if ($job_has_uuid)
   # Claim this job, and make sure nobody else does
   unless ($Job->update_attributes('is_locked_by_uuid' => $User->{'uuid'}) &&
           $Job->{'is_locked_by_uuid'} == $User->{'uuid'}) {
-    croak("Error while updating / locking job");
+    Log(undef, "Error while updating / locking job, exiting ".EX_TEMPFAIL);
+    exit EX_TEMPFAIL;
   }
   $Job->update_attributes('started_at' => scalar gmtime,
                           'running' => 1,
@@ -499,16 +510,17 @@ if (!$have_slurm)
 
 # If this job requires a Docker image, install that.
 my $docker_bin = "/usr/bin/docker.io";
-my ($docker_locator, $docker_hash);
+my ($docker_locator, $docker_stream, $docker_hash);
 if ($docker_locator = $Job->{docker_image_locator}) {
-  $docker_hash = find_docker_hash($docker_locator);
+  ($docker_stream, $docker_hash) = find_docker_image($docker_locator);
   if (!$docker_hash)
   {
     croak("No Docker image hash found from locator $docker_locator");
   }
+  $docker_stream =~ s/^\.//;
   my $docker_install_script = qq{
 if ! $docker_bin images -q --no-trunc | grep -qxF \Q$docker_hash\E; then
-    arv-get \Q$docker_locator/$docker_hash.tar\E | $docker_bin load
+    arv-get \Q$docker_locator$docker_stream/$docker_hash.tar\E | $docker_bin load
 fi
 };
   my $docker_pid = fork();
@@ -687,7 +699,9 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
 
     my @execargs = ('bash', '-c', $command);
     srun (\@srunargs, \@execargs, undef, $build_script_to_send);
-    exit (111);
+    # exec() failed, we assume nothing happened.
+    Log(undef, "srun() failed on build script");
+    die;
   }
   close("writer");
   if (!defined $childpid)
@@ -881,7 +895,7 @@ else {
 Log (undef, "finish");
 
 save_meta();
-exit 0;
+exit ($Job->{'success'} ? 1 : 0);
 
 
 
@@ -1466,19 +1480,19 @@ sub must_lock_now
   }
 }
 
-sub find_docker_hash {
-  # Given a Keep locator, search for a matching link to find the Docker hash
-  # of the stored image.
+sub find_docker_image {
+  # Given a Keep locator, check to see if it contains a Docker image.
+  # If so, return its stream name and Docker hash.
+  # If not, return undef for both values.
   my $locator = shift;
-  my $links_result = $arv->{links}->{list}->execute(
-    filters => [["head_uuid", "=", $locator],
-                ["link_class", "=", "docker_image_hash"]],
-    limit => 1);
-  my $docker_hash;
-  foreach my $link (@{$links_result->{items}}) {
-    $docker_hash = lc($link->{name});
+  if (my $image = $arv->{collections}->{get}->execute(uuid => $locator)) {
+    my @file_list = @{$image->{files}};
+    if ((scalar(@file_list) == 1) &&
+        ($file_list[0][1] =~ /^([0-9A-Fa-f]{64})\.tar$/)) {
+      return ($file_list[0][0], $1);
+    }
   }
-  return $docker_hash;
+  return (undef, undef);
 }
 
 __DATA__