X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/9479599527ffdf5e4de14e8ccc79eeb3c6776526..c99ba4cdc8f6ff0be1847ffe4ec6f791f6b83396:/sdk/cli/bin/crunch-job

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 94865a11cf..83931f2c65 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -74,6 +74,7 @@ behavior (e.g., cancel job if cancelled_at becomes non-nil).
 
 use strict;
 use POSIX ':sys_wait_h';
+use POSIX qw(strftime);
 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
 use Arvados;
 use Digest::MD5 qw(md5_hex);
@@ -447,9 +448,10 @@ else
   if ($treeish =~ m{^\d{1,4}$}) {
     my $gitlog = `git --git-dir=\Q$repo\E log --pretty="format:%H" --grep="git-svn-id:.*\@"\Q$treeish\E" " master`;
     chomp $gitlog;
-    if ($gitlog =~ /^[a-f0-9]{40}$/) {
+    Log(undef, "git Subversion search exited $?");
+    if (($? == 0) && ($gitlog =~ /^[a-f0-9]{40}$/)) {
       $commit = $gitlog;
-      Log (undef, "Using commit $commit for script_version $treeish");
+      Log(undef, "Using commit $commit for Subversion revision $treeish");
     }
   }
 
@@ -458,21 +460,19 @@ else
   if (!defined $commit) {
     my $found = `git --git-dir=\Q$repo\E rev-list -1 ''\Q$treeish\E`;
     chomp $found;
-    if ($found =~ /^[0-9a-f]{40}$/s) {
+    Log(undef, "git rev-list exited $? with result '$found'");
+    if (($? == 0) && ($found =~ /^[0-9a-f]{40}$/s)) {
       $commit = $found;
+      Log(undef, "Using commit $commit for tree-ish $treeish");
       if ($commit ne $treeish) {
 	# Make sure we record the real commit id in the database,
 	# frozentokey, logs, etc. -- instead of an abbreviation or a
 	# branch name which can become ambiguous or point to a
 	# different commit in the future.
-	$ENV{"CRUNCH_SRC_COMMIT"} = $commit;
-	Log (undef, "Using commit $commit for tree-ish $treeish");
-        if ($commit ne $treeish) {
-          $Job->{'script_version'} = $commit;
-          !$job_has_uuid or
-              $Job->update_attributes('script_version' => $commit) or
-              croak("Error while updating job");
-        }
+        $Job->{'script_version'} = $commit;
+        !$job_has_uuid or
+            $Job->update_attributes('script_version' => $commit) or
+            croak("Error while updating job");
       }
     }
   }
@@ -482,6 +482,7 @@ else
     @execargs = ("sh", "-c",
 		 "mkdir -p $ENV{CRUNCH_INSTALL} && cd $ENV{CRUNCH_TMP} && perl -");
     $git_archive = `git --git-dir=\Q$repo\E archive ''\Q$commit\E`;
+    croak("git archive failed: exit " . ($? >> 8)) if ($? != 0);
   }
   else {
     croak ("could not figure out commit id for $treeish");
@@ -656,6 +657,8 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
     if ($build_script)
     {
       $build_script_to_send = $build_script;
+      $command .=
+	  "&& perl -";
     }
     $command .= "&& exec arv-mount --by-id --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
     if ($docker_hash)
@@ -668,12 +671,16 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
       $command .=
           q{$(ip -o address show scope global |
               gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
+      $command .= "--volume=\Q$ENV{CRUNCH_SRC}:/tmp/crunch-src:ro\E ";
       $command .= "--volume=\Q$ENV{TASK_KEEPMOUNT}:/keep:ro\E ";
       $command .= "--env=\QHOME=/home/crunch\E ";
       while (my ($env_key, $env_val) = each %ENV)
       {
         if ($env_key =~ /^(ARVADOS|JOB|TASK)_/) {
-          if ($env_key eq "TASK_KEEPMOUNT") {
+          if ($env_key eq "TASK_WORK") {
+            $command .= "--env=\QTASK_WORK=/tmp/crunch-job\E ";
+          }
+          elsif ($env_key eq "TASK_KEEPMOUNT") {
             $command .= "--env=\QTASK_KEEPMOUNT=/keep\E ";
           }
           else {
@@ -685,13 +692,11 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
       $command .= "--env=\QCRUNCH_SRC=/tmp/crunch-src\E ";
       $command .= "\Q$docker_hash\E ";
       $command .= "stdbuf --output=0 --error=0 ";
-      $command .= "perl - ";
       $command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
     } else {
       # Non-docker run
       $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 ";
       $command .= "stdbuf --output=0 --error=0 ";
-      $command .= "perl - ";
       $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
     }
 
@@ -725,6 +730,9 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
   delete $Jobstep->{stderr};
   delete $Jobstep->{finishtime};
 
+  $Jobstep->{'arvados_task'}->{started_at} = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime($Jobstep->{starttime});
+  $Jobstep->{'arvados_task'}->save;
+
   splice @jobstep_todo, $todo_ptr, 1;
   --$todo_ptr;
 
@@ -860,7 +868,8 @@ else {
     my $output = $arv->{'collections'}->{'create'}->execute('collection' => {
       'manifest_text' => $orig_manifest_text,
     });
-    Log(undef, "output " . $output->{portable_data_hash});
+    Log(undef, "output uuid " . $output->{uuid});
+    Log(undef, "output hash " . $output->{portable_data_hash});
     $Job->update_attributes('output' => $output->{portable_data_hash}) if $job_has_uuid;
   };
   if ($@) {
@@ -974,6 +983,8 @@ sub reapchildren
   }
   $Jobstep->{exitcode} = $childstatus;
   $Jobstep->{finishtime} = time;
+  $Jobstep->{'arvados_task'}->{finished_at} = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime($Jobstep->{finishtime});
+  $Jobstep->{'arvados_task'}->save;
   process_stderr ($jobstepid, $task_success);
   Log ($jobstepid, "output " . $Jobstep->{'arvados_task'}->{output});
 
@@ -1335,7 +1346,7 @@ sub save_meta
   return if $justcheckpoint;  # checkpointing is not relevant post-Warehouse.pm
 
   $local_logfile->flush;
-  my $cmd = "arv-put --filename ''\Q$keep_logfile\E "
+  my $cmd = "arv-put --portable-data-hash --filename ''\Q$keep_logfile\E "
       . quotemeta($local_logfile->filename);
   my $loglocator = `$cmd`;
   die "system $cmd failed: $?" if $?;
@@ -1460,14 +1471,26 @@ sub find_docker_image {
   # If so, return its stream name and Docker hash.
   # If not, return undef for both values.
   my $locator = shift;
+  my ($streamname, $filename);
   if (my $image = $arv->{collections}->{get}->execute(uuid => $locator)) {
-    my @file_list = @{$image->{files}};
-    if ((scalar(@file_list) == 1) &&
-        ($file_list[0][1] =~ /^([0-9A-Fa-f]{64})\.tar$/)) {
-      return ($file_list[0][0], $1);
+    foreach my $line (split(/\n/, $image->{manifest_text})) {
+      my @tokens = split(/\s+/, $line);
+      next if (!@tokens);
+      $streamname = shift(@tokens);
+      foreach my $filedata (grep(/^\d+:\d+:/, @tokens)) {
+        if (defined($filename)) {
+          return (undef, undef);  # More than one file in the Collection.
+        } else {
+          $filename = (split(/:/, $filedata, 3))[2];
+        }
+      }
     }
   }
-  return (undef, undef);
+  if (defined($filename) and ($filename =~ /^([0-9A-Fa-f]{64})\.tar$/)) {
+    return ($streamname, $1);
+  } else {
+    return (undef, undef);
+  }
 }
 
 __DATA__
@@ -1493,7 +1516,12 @@ for my $dir ($destdir, $task_work) {
 open L, ">", "$destdir.lock" or die "$destdir.lock: $!";
 flock L, LOCK_EX;
 if (readlink ("$destdir.commit") eq $commit && -d $destdir) {
-    exit 0;
+    if (@ARGV) {
+        exec(@ARGV);
+        die "Cannot exec `@ARGV`: $!";
+    } else {
+        exit 0;
+    }
 }
 
 unlink "$destdir.commit";
@@ -1541,6 +1569,7 @@ close L;
 
 if (@ARGV) {
     exec(@ARGV);
+    die "Cannot exec `@ARGV`: $!";
 } else {
     exit 0;
 }