From: Tom Clegg <tclegg@veritasgenetics.com>
Date: Thu, 1 Nov 2018 17:56:13 +0000 (-0400)
Subject: 14406: Pass output through collectionFS if it has remote block refs.
X-Git-Tag: 1.3.0~54^2~6
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/8181eb2b50cb2462e0fd52d67203350ed802537b

14406: Pass output through collectionFS if it has remote block refs.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>
---

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 27136b4522..66534b4309 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -1293,6 +1293,17 @@ func (runner *ContainerRunner) CaptureOutput() error {
 	if err != nil {
 		return err
 	}
+	if n := len(regexp.MustCompile(` [0-9a-f]+\+\S*\+R`).FindAllStringIndex(txt, -1)); n > 0 {
+		runner.CrunchLog.Printf("Copying %d data blocks from remote input collections...", n)
+		fs, err := (&arvados.Collection{ManifestText: txt}).FileSystem(runner.client, runner.Kc)
+		if err != nil {
+			return err
+		}
+		txt, err = fs.MarshalManifest(".")
+		if err != nil {
+			return err
+		}
+	}
 	var resp arvados.Collection
 	err = runner.ArvClient.Create("collections", arvadosclient.Dict{
 		"ensure_unique_name": true,