Merge branch 'master' into 2882-job-process-stats refs #2882

diff --combined sdk/cli/bin/crunch-job
index 5fa5f2bbbafc27a4dc07392da9cf488c778c46a3,0befdd5dbf9e6c8af770c187824ad90978a05c71..167d3ddbe9074e3edba2716d6247f03bd642aba7
--- 1/sdk/cli/bin/crunch-job
--- 2/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@@ -76,6 -76,7 +76,7 @@@ use strict
  use POSIX ':sys_wait_h';
  use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  use Arvados;
+ use Digest::MD5 qw(md5_hex);
  use Getopt::Long;
  use IPC::Open2;
  use IO::Select;
@@@ -640,8 -641,7 +641,8 @@@ for (my $todo_ptr = 0; $todo_ptr <= $#j
      $command .= "&& exec arv-mount --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
      if ($docker_image)
      {
 -      $command .= "$docker_bin run -i -a stdin -a stdout -a stderr ";
 +      $command .= "crunchstat -cgroup-parent=/sys/fs/cgroup/lxc -cgroup-cid=$ENV{TASK_WORK}/docker.cid -poll=1000 ";
 +      $command .= "$docker_bin run -i -a stdin -a stdout -a stderr -cidfile=$ENV{TASK_WORK}/docker.cid ";
        # Dynamically configure the container to use the host system as its
        # DNS server.  Get the host's global addresses from the ip command,
        # and turn them into docker --dns options using gawk.
@@@ -654,13 -654,9 +655,13 @@@
        }
        while (my ($env_key, $env_val) = each %ENV)
        {
 -        $command .= "-e \Q$env_key=$env_val\E ";
 +        if ($env_key =~ /^(JOB|TASK)_/) {
 +          $command .= "-e \Q$env_key=$env_val\E ";
 +        }
        }
        $command .= "\Q$docker_image\E ";
 +    } else {
 +      $command .= "crunchstat -cgroup-path=/sys/fs/cgroup "
      }
      $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
      my @execargs = ('bash', '-c', $command);
@@@ -804,21 -800,37 +805,37 @@@ goto ONELEVEL if !defined $main::succes
  
  release_allocation();
  freeze();
+ my $collated_output = &collate_output();
+ 
  if ($job_has_uuid) {
-   $Job->update_attributes('output' => &collate_output(),
-                           'running' => 0,
-                           'success' => $Job->{'output'} && $main::success,
+   $Job->update_attributes('running' => 0,
+                           'success' => $collated_output && $main::success,
                            'finished_at' => scalar gmtime)
  }
  
- if ($Job->{'output'})
+ if ($collated_output)
  {
    eval {
-     my $manifest_text = `arv keep get ''\Q$Job->{'output'}\E`;
-     $arv->{'collections'}->{'create'}->execute('collection' => {
-       'uuid' => $Job->{'output'},
+     open(my $orig_manifest, '-|', 'arv', 'keep', 'get', $collated_output)
+         or die "failed to get collated manifest: $!";
+     # Read the original manifest, and strip permission hints from it,
+     # so we can put the result in a Collection.
+     my @manifest_lines = ();
+     while (my $manifest_line = <$orig_manifest>) {
+       my @words = split(/ /, $manifest_line, -1);
+       foreach my $ii (0..$#words) {
+         if ($words[$ii] =~ /^[0-9a-f]{32}\+/) {
+           $words[$ii] =~ s/\+A[0-9a-f]{40}@[0-9a-f]{8}\b//;
+         }
+       }
+       push(@manifest_lines, join(" ", @words));
+     }
+     my $manifest_text = join("", @manifest_lines);
+     my $output = $arv->{'collections'}->{'create'}->execute('collection' => {
+       'uuid' => md5_hex($manifest_text),
        'manifest_text' => $manifest_text,
      });
+     $Job->update_attributes('output' => $output->{uuid});
      if ($Job->{'output_is_persistent'}) {
        $arv->{'links'}->{'create'}->execute('link' => {
          'tail_kind' => 'arvados#user',