12183: Support uploading symlinks to directories.

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 61179975b0e2b5c9fe47125c8d566ba15d6ae290..c50f799e8c43ccadf112ab24f980f46ffbe207a9 100644 (file)
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -926,12 +926,12 @@ func (runner *ContainerRunner) WaitFinish() (err error) {
 // they must remain within the output directory.
 //
 // Assumes initial value of "path" is absolute, and located within runner.HostOutputDir.
-func (runner *ContainerRunner) EvalSymlinks(path string, binds []string, symlinksToRemove *[]string) (manifestText string, err error) {
+func (runner *ContainerRunner) EvalSymlinks(path string, binds []string) (manifestText string, symlinksToRemove []string, err error) {
        var links []string
 
        defer func() {
                if err != nil {
-                       *symlinksToRemove = append(*symlinksToRemove, links...)
+                       symlinksToRemove = append(symlinksToRemove, links...)
                }
        }()
 
@@ -989,7 +989,7 @@ func (runner *ContainerRunner) EvalSymlinks(path string, binds []string, symlink
                                                return
                                        }
                                        manifestText = manifestText + m
-                                       *symlinksToRemove = append(*symlinksToRemove, l)
+                                       symlinksToRemove = append(symlinksToRemove, l)
                                }
                                return
                        }
@@ -1064,18 +1064,19 @@ func (runner *ContainerRunner) CaptureOutput() error {
 
                var symlinksToRemove []string
                var m string
+               var srm []string
                // Find symlinks to arv-mounted files & dirs.
                err = filepath.Walk(runner.HostOutputDir, func(path string, info os.FileInfo, err error) error {
                        if err != nil {
                                return err
                        }
-                       m, err = runner.EvalSymlinks(path, binds, &symlinksToRemove)
+                       m, srm, err = runner.EvalSymlinks(path, binds)
+                       symlinksToRemove = append(symlinksToRemove, srm...)
                        if err == nil {
                                manifestText = manifestText + m
                        }
                        return err
                })
-               runner.CrunchLog.Printf("sm %q", symlinksToRemove)
                for _, l := range symlinksToRemove {
                        os.Remove(l)
                }

diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index dbf9cc7b989521ac380fa313cb869e187fd7a7dc..9081783bdd59fd1bcb71722306742f8f9705e9ec 100644 (file)
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -1543,10 +1543,18 @@ func (s *TestSuite) TestOutputSymlinkToOutput(c *C) {
                rf, _ := os.Create(t.realTemp + "/2/realfile")
                rf.Write([]byte("foo"))
                rf.Close()
+
+               os.Mkdir(t.realTemp+"/2/realdir", 0700)
+               rf, _ = os.Create(t.realTemp + "/2/realdir/subfile")
+               rf.Write([]byte("bar"))
+               rf.Close()
+
                os.Symlink("/tmp/realfile", t.realTemp+"/2/file1")
                os.Symlink("realfile", t.realTemp+"/2/file2")
                os.Symlink("/tmp/file1", t.realTemp+"/2/file3")
                os.Symlink("file2", t.realTemp+"/2/file4")
+               os.Symlink("realdir", t.realTemp+"/2/dir1")
+               os.Symlink("/tmp/realdir", t.realTemp+"/2/dir2")
                t.logWriter.Close()
        })
 
@@ -1557,7 +1565,12 @@ func (s *TestSuite) TestOutputSymlinkToOutput(c *C) {
                        collection := v["collection"].(arvadosclient.Dict)
                        if strings.Index(collection["name"].(string), "output") == 0 {
                                manifest := collection["manifest_text"].(string)
-                               c.Check(manifest, Equals, ". 7a2c86e102dcc231bd232aad99686dfa+15 0:3:file1 3:3:file2 6:3:file3 9:3:file4 12:3:realfile\n")
+                               c.Check(manifest, Equals,
+                                       `. 7a2c86e102dcc231bd232aad99686dfa+15 0:3:file1 3:3:file2 6:3:file3 9:3:file4 12:3:realfile
+./dir1 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+./dir2 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+./realdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+`)
                        }
                }
        }

diff --git a/services/crunch-run/upload.go b/services/crunch-run/upload.go
index bb2776a4266342568a3eb05c755ed64d700659ec..faf20b4f8b4568076b17151655927885a6d038de 100644 (file)
--- a/services/crunch-run/upload.go
+++ b/services/crunch-run/upload.go
@@ -19,6 +19,7 @@ import (
        "errors"
        "fmt"
        "io"
+       "io/ioutil"
        "log"
        "os"
        "path/filepath"
@@ -282,6 +283,20 @@ func (m *WalkUpload) WalkFunc(path string, info os.FileInfo, err error) error {
                if err != nil {
                        return fmt.Errorf("stat symlink %q target %q: %s", path, targetPath, err)
                }
+               if targetInfo.Mode()&os.ModeDir != 0 {
+                       // Symlinks to directories don't get walked, so do it
+                       // here.  We've previously checked that they stay in
+                       // the output directory and don't result in an endless
+                       // loop.
+                       var rd []os.FileInfo
+                       rd, err = ioutil.ReadDir(path)
+                       if err != nil {
+                               return err
+                       }
+                       for _, ent := range rd {
+                               err = filepath.Walk(filepath.Join(path, ent.Name()), m.WalkFunc)
+                       }
+               }
        }
 
        if targetInfo.Mode()&os.ModeType != 0 {