X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d13ad5bbb7f5e74cb43d0570d9d1a8159237f5ff..7178ac911e7b160c1348da404b5aa07b9829d3c8:/sdk/cli/bin/crunch-job

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 0ba32b0588..70f379e53f 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -74,6 +74,7 @@ behavior (e.g., cancel job if cancelled_at becomes non-nil).
 
 use strict;
 use POSIX ':sys_wait_h';
+use POSIX qw(strftime);
 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
 use Arvados;
 use Digest::MD5 qw(md5_hex);
@@ -447,9 +448,10 @@ else
   if ($treeish =~ m{^\d{1,4}$}) {
     my $gitlog = `git --git-dir=\Q$repo\E log --pretty="format:%H" --grep="git-svn-id:.*\@"\Q$treeish\E" " master`;
     chomp $gitlog;
-    if ($gitlog =~ /^[a-f0-9]{40}$/) {
+    Log(undef, "git Subversion search exited $?");
+    if (($? == 0) && ($gitlog =~ /^[a-f0-9]{40}$/)) {
       $commit = $gitlog;
-      Log (undef, "Using commit $commit for script_version $treeish");
+      Log(undef, "Using commit $commit for Subversion revision $treeish");
     }
   }
 
@@ -458,21 +460,19 @@ else
   if (!defined $commit) {
     my $found = `git --git-dir=\Q$repo\E rev-list -1 ''\Q$treeish\E`;
     chomp $found;
-    if ($found =~ /^[0-9a-f]{40}$/s) {
+    Log(undef, "git rev-list exited $? with result '$found'");
+    if (($? == 0) && ($found =~ /^[0-9a-f]{40}$/s)) {
       $commit = $found;
+      Log(undef, "Using commit $commit for tree-ish $treeish");
       if ($commit ne $treeish) {
 	# Make sure we record the real commit id in the database,
 	# frozentokey, logs, etc. -- instead of an abbreviation or a
 	# branch name which can become ambiguous or point to a
 	# different commit in the future.
-	$ENV{"CRUNCH_SRC_COMMIT"} = $commit;
-	Log (undef, "Using commit $commit for tree-ish $treeish");
-        if ($commit ne $treeish) {
-          $Job->{'script_version'} = $commit;
-          !$job_has_uuid or
-              $Job->update_attributes('script_version' => $commit) or
-              croak("Error while updating job");
-        }
+        $Job->{'script_version'} = $commit;
+        !$job_has_uuid or
+            $Job->update_attributes('script_version' => $commit) or
+            croak("Error while updating job");
       }
     }
   }
@@ -482,6 +482,7 @@ else
     @execargs = ("sh", "-c",
 		 "mkdir -p $ENV{CRUNCH_INSTALL} && cd $ENV{CRUNCH_TMP} && perl -");
     $git_archive = `git --git-dir=\Q$repo\E archive ''\Q$commit\E`;
+    croak("git archive failed: exit " . ($? >> 8)) if ($? != 0);
   }
   else {
     croak ("could not figure out commit id for $treeish");
@@ -659,7 +660,7 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
       $command .=
 	  "&& perl -";
     }
-    $command .= "&& exec arv-mount --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
+    $command .= "&& exec arv-mount --by-id --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
     if ($docker_hash)
     {
       $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -cgroup-parent=docker -cgroup-cid=$ENV{TASK_WORK}/docker.cid -poll=10000 ";
@@ -729,6 +730,9 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
   delete $Jobstep->{stderr};
   delete $Jobstep->{finishtime};
 
+  $Jobstep->{'arvados_task'}->{started_at} = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime($Jobstep->{starttime});
+  $Jobstep->{'arvados_task'}->save;
+
   splice @jobstep_todo, $todo_ptr, 1;
   --$todo_ptr;
 
@@ -844,12 +848,6 @@ release_allocation();
 freeze();
 my $collated_output = &collate_output();
 
-if ($job_has_uuid) {
-  $Job->update_attributes('running' => 0,
-                          'success' => $collated_output && $main::success,
-                          'finished_at' => scalar gmtime)
-}
-
 if (!$collated_output) {
   Log(undef, "output undef");
 }
@@ -876,6 +874,13 @@ else {
 Log (undef, "finish");
 
 save_meta();
+
+if ($job_has_uuid) {
+  $Job->update_attributes('running' => 0,
+                          'success' => $collated_output && $main::success,
+                          'finished_at' => scalar gmtime)
+}
+
 exit ($Job->{'success'} ? 1 : 0);
 
 
@@ -979,6 +984,8 @@ sub reapchildren
   }
   $Jobstep->{exitcode} = $childstatus;
   $Jobstep->{finishtime} = time;
+  $Jobstep->{'arvados_task'}->{finished_at} = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime($Jobstep->{finishtime});
+  $Jobstep->{'arvados_task'}->save;
   process_stderr ($jobstepid, $task_success);
   Log ($jobstepid, "output " . $Jobstep->{'arvados_task'}->{output});
 
@@ -1200,7 +1207,8 @@ sub collate_output
   Log (undef, "collate");
 
   my ($child_out, $child_in);
-  my $pid = open2($child_out, $child_in, 'arv-put', '--raw');
+  my $pid = open2($child_out, $child_in, 'arv-put', '--raw',
+                  '--retries', put_retry_count());
   my $joboutput;
   for (@jobstep)
   {
@@ -1340,8 +1348,9 @@ sub save_meta
   return if $justcheckpoint;  # checkpointing is not relevant post-Warehouse.pm
 
   $local_logfile->flush;
-  my $cmd = "arv-put --portable-data-hash --filename ''\Q$keep_logfile\E "
-      . quotemeta($local_logfile->filename);
+  my $retry_count = put_retry_count();
+  my $cmd = "arv-put --portable-data-hash --retries $retry_count " .
+      "--filename ''\Q$keep_logfile\E " . quotemeta($local_logfile->filename);
   my $loglocator = `$cmd`;
   die "system $cmd failed: $?" if $?;
   chomp($loglocator);
@@ -1487,6 +1496,20 @@ sub find_docker_image {
   }
 }
 
+sub put_retry_count {
+  # Calculate a --retries argument for arv-put that will have it try
+  # approximately as long as this Job has been running.
+  my $stoptime = shift || time;
+  my $starttime = $jobstep[0]->{starttime};
+  my $timediff = defined($starttime) ? ($stoptime - $starttime) : 1;
+  my $retries = 0;
+  while ($timediff >= 2) {
+    $retries++;
+    $timediff /= 2;
+  }
+  return ($retries > 3) ? $retries : 3;
+}
+
 __DATA__
 #!/usr/bin/perl