Merge branch '3663-collection-reader-performance'

[arvados.git] / sdk / cli / bin / crunch-job

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index b52d8c6cff08e58c6e873b9e13211d40ddbaadc7..0ba32b05883ad8a565e4b694407049c39fa40ae7 100755 (executable)
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -487,6 +487,8 @@ else
     croak ("could not figure out commit id for $treeish");
   }
 
+  # Note: this section is almost certainly unnecessary if we're
+  # running tasks in docker containers.
   my $installpid = fork();
   if ($installpid == 0)
   {
@@ -674,7 +676,10 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
       while (my ($env_key, $env_val) = each %ENV)
       {
         if ($env_key =~ /^(ARVADOS|JOB|TASK)_/) {
-          if ($env_key eq "TASK_KEEPMOUNT") {
+          if ($env_key eq "TASK_WORK") {
+            $command .= "--env=\QTASK_WORK=/tmp/crunch-job\E ";
+          }
+          elsif ($env_key eq "TASK_KEEPMOUNT") {
             $command .= "--env=\QTASK_KEEPMOUNT=/keep\E ";
           }
           else {
@@ -859,7 +864,8 @@ else {
     my $output = $arv->{'collections'}->{'create'}->execute('collection' => {
       'manifest_text' => $orig_manifest_text,
     });
-    Log(undef, "output " . $output->{portable_data_hash});
+    Log(undef, "output uuid " . $output->{uuid});
+    Log(undef, "output hash " . $output->{portable_data_hash});
     $Job->update_attributes('output' => $output->{portable_data_hash}) if $job_has_uuid;
   };
   if ($@) {
@@ -1334,7 +1340,7 @@ sub save_meta
   return if $justcheckpoint;  # checkpointing is not relevant post-Warehouse.pm
 
   $local_logfile->flush;
-  my $cmd = "arv-put --filename ''\Q$keep_logfile\E "
+  my $cmd = "arv-put --portable-data-hash --filename ''\Q$keep_logfile\E "
       . quotemeta($local_logfile->filename);
   my $loglocator = `$cmd`;
   die "system $cmd failed: $?" if $?;
@@ -1459,14 +1465,26 @@ sub find_docker_image {
   # If so, return its stream name and Docker hash.
   # If not, return undef for both values.
   my $locator = shift;
+  my ($streamname, $filename);
   if (my $image = $arv->{collections}->{get}->execute(uuid => $locator)) {
-    my @file_list = @{$image->{files}};
-    if ((scalar(@file_list) == 1) &&
-        ($file_list[0][1] =~ /^([0-9A-Fa-f]{64})\.tar$/)) {
-      return ($file_list[0][0], $1);
+    foreach my $line (split(/\n/, $image->{manifest_text})) {
+      my @tokens = split(/\s+/, $line);
+      next if (!@tokens);
+      $streamname = shift(@tokens);
+      foreach my $filedata (grep(/^\d+:\d+:/, @tokens)) {
+        if (defined($filename)) {
+          return (undef, undef);  # More than one file in the Collection.
+        } else {
+          $filename = (split(/:/, $filedata, 3))[2];
+        }
+      }
     }
   }
-  return (undef, undef);
+  if (defined($filename) and ($filename =~ /^([0-9A-Fa-f]{64})\.tar$/)) {
+    return ($streamname, $1);
+  } else {
+    return (undef, undef);
+  }
 }
 
 __DATA__
@@ -1482,15 +1500,22 @@ my $commit = $ENV{"CRUNCH_SRC_COMMIT"};
 my $repo = $ENV{"CRUNCH_SRC_URL"};
 my $task_work = $ENV{"TASK_WORK"};
 
-if ($task_work) {
-    make_path $task_work;
-    -e $task_work or die "Failed to create temporary working directory ($task_work): $!";
+for my $dir ($destdir, $task_work) {
+    if ($dir) {
+        make_path $dir;
+        -e $dir or die "Failed to create temporary directory ($dir): $!";
+    }
 }
 
 open L, ">", "$destdir.lock" or die "$destdir.lock: $!";
 flock L, LOCK_EX;
 if (readlink ("$destdir.commit") eq $commit && -d $destdir) {
-    exit 0;
+    if (@ARGV) {
+        exec(@ARGV);
+        die "Cannot exec `@ARGV`: $!";
+    } else {
+        exit 0;
+    }
 }
 
 unlink "$destdir.commit";
@@ -1536,7 +1561,12 @@ if ($commit) {
 
 close L;
 
-exit 0;
+if (@ARGV) {
+    exec(@ARGV);
+    die "Cannot exec `@ARGV`: $!";
+} else {
+    exit 0;
+}
 
 sub shell_or_die
 {