Merge branch 'master' into 3586-job-priority

[arvados.git] / sdk / cli / bin / crunch-job

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 47f9894a415557b559a2f7170428e0132b30e7cf..0d31afcb874a5c02b965827aa88636f7b29463ea 100755 (executable)
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -394,7 +394,7 @@ else
     if ($cleanpid == 0)
     {
       srun (["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
-           ['bash', '-c', 'if mount | grep -q $JOB_WORK/; then sudo /bin/umount $JOB_WORK/* 2>/dev/null; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*']);
+           ['bash', '-c', 'if mount | grep -q $JOB_WORK/; then for i in $JOB_WORK/*keep; do /bin/fusermount -z -u $i; done; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*']);
       exit (1);
     }
     while (1)
@@ -487,6 +487,8 @@ else
     croak ("could not figure out commit id for $treeish");
   }
 
+  # Note: this section is almost certainly unnecessary if we're
+  # running tasks in docker containers.
   my $installpid = fork();
   if ($installpid == 0)
   {
@@ -649,13 +651,11 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
     my $build_script_to_send = "";
     my $command =
        "if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; "
-        ."mkdir -p $ENV{JOB_WORK} $ENV{CRUNCH_TMP} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT}"
+        ."mkdir -p $ENV{CRUNCH_TMP} $ENV{JOB_WORK} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT} "
        ."&& cd $ENV{CRUNCH_TMP} ";
     if ($build_script)
     {
       $build_script_to_send = $build_script;
-      $command .=
-         "&& perl -";
     }
     $command .= "&& exec arv-mount --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
     if ($docker_hash)
@@ -668,16 +668,12 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
       $command .=
           q{$(ip -o address show scope global |
               gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
-      $command .= "--volume=\Q$ENV{CRUNCH_SRC}:/tmp/crunch-src:ro\E ";
       $command .= "--volume=\Q$ENV{TASK_KEEPMOUNT}:/keep:ro\E ";
       $command .= "--env=\QHOME=/home/crunch\E ";
       while (my ($env_key, $env_val) = each %ENV)
       {
         if ($env_key =~ /^(ARVADOS|JOB|TASK)_/) {
-          if ($env_key eq "TASK_WORK") {
-            $command .= "--env=\QTASK_WORK=/tmp/crunch-job\E ";
-          }
-          elsif ($env_key eq "TASK_KEEPMOUNT") {
+          if ($env_key eq "TASK_KEEPMOUNT") {
             $command .= "--env=\QTASK_KEEPMOUNT=/keep\E ";
           }
           else {
@@ -689,11 +685,13 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
       $command .= "--env=\QCRUNCH_SRC=/tmp/crunch-src\E ";
       $command .= "\Q$docker_hash\E ";
       $command .= "stdbuf --output=0 --error=0 ";
+      $command .= "perl - " if ($build_script);
       $command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
     } else {
       # Non-docker run
       $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 ";
       $command .= "stdbuf --output=0 --error=0 ";
+      $command .= "perl - " if ($build_script);
       $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
     }
 
@@ -855,37 +853,16 @@ else {
   eval {
     open(my $orig_manifest, '-|', 'arv-get', $collated_output)
         or die "failed to get collated manifest: $!";
-    # Read the original manifest, and strip permission hints from it,
-    # so we can put the result in a Collection.
-    my @stripped_manifest_lines = ();
     my $orig_manifest_text = '';
     while (my $manifest_line = <$orig_manifest>) {
       $orig_manifest_text .= $manifest_line;
-      my @words = split(/ /, $manifest_line, -1);
-      foreach my $ii (0..$#words) {
-        if ($words[$ii] =~ /^[0-9a-f]{32}\+/) {
-          $words[$ii] =~ s/\+A[0-9a-f]{40}@[0-9a-f]{8}\b//;
-        }
-      }
-      push(@stripped_manifest_lines, join(" ", @words));
     }
-    my $stripped_manifest_text = join("", @stripped_manifest_lines);
     my $output = $arv->{'collections'}->{'create'}->execute('collection' => {
-      'uuid' => md5_hex($stripped_manifest_text),
       'manifest_text' => $orig_manifest_text,
     });
-    Log(undef, "output " . $output->{uuid});
-    $Job->update_attributes('output' => $output->{uuid}) if $job_has_uuid;
-    if ($Job->{'output_is_persistent'}) {
-      $arv->{'links'}->{'create'}->execute('link' => {
-        'tail_kind' => 'arvados#user',
-        'tail_uuid' => $User->{'uuid'},
-        'head_kind' => 'arvados#collection',
-        'head_uuid' => $Job->{'output'},
-        'link_class' => 'resources',
-        'name' => 'wants',
-      });
-    }
+    Log(undef, "output uuid " . $output->{uuid});
+    Log(undef, "output hash " . $output->{portable_data_hash});
+    $Job->update_attributes('output' => $output->{portable_data_hash}) if $job_has_uuid;
   };
   if ($@) {
     Log (undef, "Failed to register output manifest: $@");
@@ -1359,7 +1336,7 @@ sub save_meta
   return if $justcheckpoint;  # checkpointing is not relevant post-Warehouse.pm
 
   $local_logfile->flush;
-  my $cmd = "arv-put --filename ''\Q$keep_logfile\E "
+  my $cmd = "arv-put --portable-data-hash --filename ''\Q$keep_logfile\E "
       . quotemeta($local_logfile->filename);
   my $loglocator = `$cmd`;
   die "system $cmd failed: $?" if $?;
@@ -1500,15 +1477,29 @@ __DATA__
 # checkout-and-build
 
 use Fcntl ':flock';
+use File::Path qw( make_path );
 
 my $destdir = $ENV{"CRUNCH_SRC"};
 my $commit = $ENV{"CRUNCH_SRC_COMMIT"};
 my $repo = $ENV{"CRUNCH_SRC_URL"};
+my $task_work = $ENV{"TASK_WORK"};
+
+for my $dir ($destdir, $task_work) {
+    if ($dir) {
+        make_path $dir;
+        -e $dir or die "Failed to create temporary directory ($dir): $!";
+    }
+}
 
 open L, ">", "$destdir.lock" or die "$destdir.lock: $!";
 flock L, LOCK_EX;
 if (readlink ("$destdir.commit") eq $commit && -d $destdir) {
-    exit 0;
+    if (@ARGV) {
+        exec(@ARGV);
+        die "Cannot exec `@ARGV`: $!";
+    } else {
+        exit 0;
+    }
 }
 
 unlink "$destdir.commit";
@@ -1554,7 +1545,12 @@ if ($commit) {
 
 close L;
 
-exit 0;
+if (@ARGV) {
+    exec(@ARGV);
+    die "Cannot exec `@ARGV`: $!";
+} else {
+    exit 0;
+}
 
 sub shell_or_die
 {