From 1903e0e26b3677d9686e1d19cea897690945e3ed Mon Sep 17 00:00:00 2001
From: Peter Amstutz <peter.amstutz@curoverse.com>
Date: Mon, 15 May 2017 14:06:33 -0400
Subject: [PATCH] 11693: Dereference symlinks to input files and add
 incorporate them into output collection manifest.

---
 services/crunch-run/crunchrun.go | 73 +++++++++++++++++++++++++++-----
 1 file changed, 63 insertions(+), 10 deletions(-)

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 2cd3417a58..a7616192b8 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -769,14 +769,14 @@ func (runner *ContainerRunner) getStdoutFile(mntPath string) (*os.File, error) {
 			if err != nil {
 				return nil, fmt.Errorf("While Stat on temp dir: %v", err)
 			}
-			stdoutPath := path.Join(runner.HostOutputDir, subdirs)
+			stdoutPath := filepath.Join(runner.HostOutputDir, subdirs)
 			err = os.MkdirAll(stdoutPath, st.Mode()|os.ModeSetgid|0777)
 			if err != nil {
 				return nil, fmt.Errorf("While MkdirAll %q: %v", stdoutPath, err)
 			}
 		}
 	}
-	stdoutFile, err := os.Create(path.Join(runner.HostOutputDir, stdoutPath))
+	stdoutFile, err := os.Create(filepath.Join(runner.HostOutputDir, stdoutPath))
 	if err != nil {
 		return nil, fmt.Errorf("While creating file %q: %v", stdoutPath, err)
 	}
@@ -919,14 +919,74 @@ func (runner *ContainerRunner) CaptureOutput() error {
 		return fmt.Errorf("While checking host output path: %v", err)
 	}
 
+	// Pre-populate output from the configured mount points
+	var binds []string
+	for bind, _ := range runner.Container.Mounts {
+		binds = append(binds, bind)
+	}
+	sort.Strings(binds)
+
 	var manifestText string
 
 	collectionMetafile := fmt.Sprintf("%s/.arvados#collection", runner.HostOutputDir)
 	_, err = os.Stat(collectionMetafile)
 	if err != nil {
 		// Regular directory
+
+		// Find symlinks to arv-mounted files & dirs.
+		err = filepath.Walk(runner.HostOutputDir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.Mode()&os.ModeSymlink == 0 {
+				return nil
+			}
+			// read link to get container internal path
+			var tgt string
+			tgt, err = os.Readlink(path)
+			if err != nil {
+				return err
+			}
+			if !strings.HasPrefix(tgt, "/") {
+				// Link is relative, don't handle it
+				return nil
+			}
+			// go through mounts and reverse map to collection reference
+			for _, bind := range binds {
+				if tgt == bind || strings.HasPrefix(tgt, bind+"/") {
+					mnt := runner.Container.Mounts[bind]
+
+					// get path relative to bind
+					sourceSuffix := tgt[len(bind):]
+					// get path relative to output dir
+					bindSuffix := path[len(runner.HostOutputDir):]
+
+					// Copy mount and adjust the path to add path relative to the bind
+					adjustedMount := mnt
+					adjustedMount.Path = filepath.Join(adjustedMount.Path, sourceSuffix)
+
+					// get manifest text
+					var m string
+					m, err = runner.getCollectionManifestForPath(adjustedMount, bindSuffix)
+					if err != nil {
+						return err
+					}
+					manifestText = manifestText + m
+					// delete symlink so WriteTree won't try to to dereference it.
+					os.Remove(path)
+					return nil
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("While checking output symlinks: %v", err)
+		}
+
 		cw := CollectionWriter{0, runner.Kc, nil, nil, sync.Mutex{}}
-		manifestText, err = cw.WriteTree(runner.HostOutputDir, runner.CrunchLog.Logger)
+		var m string
+		m, err = cw.WriteTree(runner.HostOutputDir, runner.CrunchLog.Logger)
+		manifestText = manifestText + m
 		if err != nil {
 			return fmt.Errorf("While uploading output files: %v", err)
 		}
@@ -946,13 +1006,6 @@ func (runner *ContainerRunner) CaptureOutput() error {
 		manifestText = rec.ManifestText
 	}
 
-	// Pre-populate output from the configured mount points
-	var binds []string
-	for bind, _ := range runner.Container.Mounts {
-		binds = append(binds, bind)
-	}
-	sort.Strings(binds)
-
 	for _, bind := range binds {
 		mnt := runner.Container.Mounts[bind]
 
-- 
2.30.2