From: Peter Amstutz <peter.amstutz@curoverse.com>
Date: Wed, 27 Aug 2014 21:04:24 +0000 (-0400)
Subject: Merge branch '3342-pipeline-output-current-project' closes #3342
X-Git-Tag: 1.1.0~2257
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/0042e4b42d9f4d3900aefc68617cb28c5a61a522?ds=inline;hp=-c

Merge branch '3342-pipeline-output-current-project' closes #3342
---

0042e4b42d9f4d3900aefc68617cb28c5a61a522
diff --combined sdk/cli/bin/crunch-job
index da168dfcf3,b52d8c6cff..9e7454f47a
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@@ -487,8 -487,6 +487,8 @@@ els
      croak ("could not figure out commit id for $treeish");
    }
  
 +  # Note: this section is almost certainly unnecessary if we're
 +  # running tasks in docker containers.
    my $installpid = fork();
    if ($installpid == 0)
    {
@@@ -656,6 -654,8 +656,6 @@@ for (my $todo_ptr = 0; $todo_ptr <= $#j
      if ($build_script)
      {
        $build_script_to_send = $build_script;
 -      $command .=
 -	  "&& perl -";
      }
      $command .= "&& exec arv-mount --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
      if ($docker_hash)
@@@ -668,6 -668,7 +668,6 @@@
        $command .=
            q{$(ip -o address show scope global |
                gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
 -      $command .= "--volume=\Q$ENV{CRUNCH_SRC}:/tmp/crunch-src:ro\E ";
        $command .= "--volume=\Q$ENV{TASK_KEEPMOUNT}:/keep:ro\E ";
        $command .= "--env=\QHOME=/home/crunch\E ";
        while (my ($env_key, $env_val) = each %ENV)
@@@ -685,13 -686,11 +685,13 @@@
        $command .= "--env=\QCRUNCH_SRC=/tmp/crunch-src\E ";
        $command .= "\Q$docker_hash\E ";
        $command .= "stdbuf --output=0 --error=0 ";
 +      $command .= "perl - ";
        $command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
      } else {
        # Non-docker run
        $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 ";
        $command .= "stdbuf --output=0 --error=0 ";
 +      $command .= "perl - ";
        $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
      }
  
@@@ -853,37 -852,15 +853,15 @@@ else 
    eval {
      open(my $orig_manifest, '-|', 'arv-get', $collated_output)
          or die "failed to get collated manifest: $!";
-     # Read the original manifest, and strip permission hints from it,
-     # so we can put the result in a Collection.
-     my @stripped_manifest_lines = ();
      my $orig_manifest_text = '';
      while (my $manifest_line = <$orig_manifest>) {
        $orig_manifest_text .= $manifest_line;
-       my @words = split(/ /, $manifest_line, -1);
-       foreach my $ii (0..$#words) {
-         if ($words[$ii] =~ /^[0-9a-f]{32}\+/) {
-           $words[$ii] =~ s/\+A[0-9a-f]{40}@[0-9a-f]{8}\b//;
-         }
-       }
-       push(@stripped_manifest_lines, join(" ", @words));
      }
-     my $stripped_manifest_text = join("", @stripped_manifest_lines);
      my $output = $arv->{'collections'}->{'create'}->execute('collection' => {
-       'uuid' => md5_hex($stripped_manifest_text),
        'manifest_text' => $orig_manifest_text,
      });
-     Log(undef, "output " . $output->{uuid});
-     $Job->update_attributes('output' => $output->{uuid}) if $job_has_uuid;
-     if ($Job->{'output_is_persistent'}) {
-       $arv->{'links'}->{'create'}->execute('link' => {
-         'tail_kind' => 'arvados#user',
-         'tail_uuid' => $User->{'uuid'},
-         'head_kind' => 'arvados#collection',
-         'head_uuid' => $Job->{'output'},
-         'link_class' => 'resources',
-         'name' => 'wants',
-       });
-     }
+     Log(undef, "output " . $output->{portable_data_hash});
+     $Job->update_attributes('output' => $output->{portable_data_hash}) if $job_has_uuid;
    };
    if ($@) {
      Log (undef, "Failed to register output manifest: $@");
@@@ -1505,11 -1482,9 +1483,11 @@@ my $commit = $ENV{"CRUNCH_SRC_COMMIT"}
  my $repo = $ENV{"CRUNCH_SRC_URL"};
  my $task_work = $ENV{"TASK_WORK"};
  
 -if ($task_work) {
 -    make_path $task_work;
 -    -e $task_work or die "Failed to create temporary working directory ($task_work): $!";
 +for my $dir ($destdir, $task_work) {
 +    if ($dir) {
 +        make_path $dir;
 +        -e $dir or die "Failed to create temporary directory ($dir): $!";
 +    }
  }
  
  open L, ">", "$destdir.lock" or die "$destdir.lock: $!";
@@@ -1561,11 -1536,7 +1539,11 @@@ if ($commit) 
  
  close L;
  
 -exit 0;
 +if (@ARGV) {
 +    exec(@ARGV);
 +} else {
 +    exit 0;
 +}
  
  sub shell_or_die
  {