13100: Handle output symlinks to writable collections.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/services/crunch-run/copier.go b/services/crunch-run/copier.go
index 3ba6eed3f73dae7e44715634cf97a6de904e6d80..4c45f6acb9bed4162dc9eaad28f7f6af82715b81 100644 (file)
--- a/services/crunch-run/copier.go
+++ b/services/crunch-run/copier.go
@@ -54,6 +54,7 @@ type copier struct {
        keepClient    IKeepClient
        hostOutputDir string
        ctrOutputDir  string
+       binds         []string
        mounts        map[string]arvados.Mount
        secretMounts  map[string]arvados.Mount
        logger        printfer
@@ -157,8 +158,12 @@ func (cp *copier) walkMount(dest, src string, maxSymlinks int, walkMountsBelow b
                        return err
                }
                cp.manifest += mft.Extract(srcRelPath, dest).Text
-       case srcRoot == cp.ctrOutputDir:
-               f, err := os.Open(filepath.Join(cp.hostOutputDir, ".arvados#collection"))
+       default:
+               hostRoot, err := cp.hostRoot(srcRoot)
+               if err != nil {
+                       return err
+               }
+               f, err := os.Open(filepath.Join(hostRoot, ".arvados#collection"))
                if err != nil {
                        return err
                }
@@ -170,8 +175,6 @@ func (cp *copier) walkMount(dest, src string, maxSymlinks int, walkMountsBelow b
                }
                mft := manifest.Manifest{Text: coll.ManifestText}
                cp.manifest += mft.Extract(srcRelPath, dest).Text
-       default:
-               return fmt.Errorf("cannot output %q as %q: writable collection mounted at %q", src, dest, srcRoot)
        }
        if walkMountsBelow {
                return cp.walkMountsBelow(dest, src)
@@ -316,6 +319,21 @@ func (cp *copier) walkHostFS(dest, src string, maxSymlinks int, includeMounts bo
        return fmt.Errorf("Unsupported file type (mode %o) in output dir: %q", fi.Mode(), src)
 }
 
+// Return the host path that was mounted at the given path in the
+// container.
+func (cp *copier) hostRoot(ctrRoot string) (string, error) {
+       if ctrRoot == cp.ctrOutputDir {
+               return cp.hostOutputDir, nil
+       }
+       for _, bind := range cp.binds {
+               tokens := strings.Split(bind, ":")
+               if len(tokens) >= 2 && tokens[1] == ctrRoot {
+                       return tokens[0], nil
+               }
+       }
+       return "", fmt.Errorf("not bind-mounted: %q", ctrRoot)
+}
+
 func (cp *copier) copyRegularFiles(m arvados.Mount) bool {
        return m.Kind == "text" || m.Kind == "json" || (m.Kind == "collection" && m.Writable)
 }

diff --git a/services/crunch-run/copier_test.go b/services/crunch-run/copier_test.go
index 41c2acdf90b758b391f6b7aaf9e3954f19b581d5..d678cf6eedca529325e56a509b24544abbfe442b 100644 (file)
--- a/services/crunch-run/copier_test.go
+++ b/services/crunch-run/copier_test.go
@@ -108,16 +108,35 @@ func (s *copierSuite) TestSecretInOutputDir(c *check.C) {
 }
 
 func (s *copierSuite) TestSymlinkToMountedCollection(c *check.C) {
+       // simulate mounted read-only collection
        s.cp.mounts["/mnt"] = arvados.Mount{
                Kind:             "collection",
                PortableDataHash: arvadostest.FooPdh,
        }
+
+       // simulate mounted writable collection
+       bindtmp, err := ioutil.TempDir("", "crunch-run.test.")
+       c.Assert(err, check.IsNil)
+       defer os.RemoveAll(bindtmp)
+       f, err := os.OpenFile(bindtmp+"/.arvados#collection", os.O_CREATE|os.O_WRONLY, 0644)
+       c.Assert(err, check.IsNil)
+       _, err = io.WriteString(f, `{"manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"}`)
+       c.Assert(err, check.IsNil)
+       c.Assert(f.Close(), check.IsNil)
+       s.cp.mounts["/mnt-w"] = arvados.Mount{
+               Kind:             "collection",
+               PortableDataHash: arvadostest.FooPdh,
+               Writable:         true,
+       }
+       s.cp.binds = append(s.cp.binds, bindtmp+":/mnt-w")
+
        c.Assert(os.Symlink("../../mnt", s.cp.hostOutputDir+"/l_dir"), check.IsNil)
        c.Assert(os.Symlink("/mnt/foo", s.cp.hostOutputDir+"/l_file"), check.IsNil)
+       c.Assert(os.Symlink("/mnt-w/bar", s.cp.hostOutputDir+"/l_file_w"), check.IsNil)
 
-       err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
+       err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
        c.Check(err, check.IsNil)
-       c.Check(s.cp.manifest, check.Matches, `(?ms)\./l_dir acbd\S+ 0:3:foo\n\. acbd\S+ 0:3:l_file\n`)
+       c.Check(s.cp.manifest, check.Matches, `(?ms)\./l_dir acbd\S+ 0:3:foo\n\. acbd\S+ 0:3:l_file\n\. 37b5\S+ 0:3:l_file_w\n`)
 }
 
 func (s *copierSuite) TestSymlink(c *check.C) {

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index aea5917be4c0c0d2b118d17584284bf6231ab2ef..2f9ccf52460a667215cdfb9156b7df56605712a5 100644 (file)
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -1126,6 +1126,7 @@ func (runner *ContainerRunner) CaptureOutput() error {
                keepClient:    runner.Kc,
                hostOutputDir: runner.HostOutputDir,
                ctrOutputDir:  runner.Container.OutputPath,
+               binds:         runner.Binds,
                mounts:        runner.Container.Mounts,
                secretMounts:  runner.SecretMounts,
                logger:        runner.CrunchLog,