X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b6d7efab2c4bffa3fabd55b166e44cca8ac1391f..3c69a3d346adac763b6426538676810921d25646:/lib/crunchrun/copier_test.go diff --git a/lib/crunchrun/copier_test.go b/lib/crunchrun/copier_test.go index 777b715d76..30e13f6503 100644 --- a/lib/crunchrun/copier_test.go +++ b/lib/crunchrun/copier_test.go @@ -5,30 +5,32 @@ package crunchrun import ( + "bytes" "io" + "io/fs" "io/ioutil" "os" + "sort" + "syscall" "git.arvados.org/arvados.git/sdk/go/arvados" - "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" + "github.com/sirupsen/logrus" check "gopkg.in/check.v1" ) var _ = check.Suite(&copierSuite{}) type copierSuite struct { - cp copier + cp copier + log bytes.Buffer } func (s *copierSuite) SetUpTest(c *check.C) { - tmpdir, err := ioutil.TempDir("", "crunch-run.test.") - c.Assert(err, check.IsNil) - api, err := arvadosclient.MakeArvadosClient() - c.Assert(err, check.IsNil) + tmpdir := c.MkDir() + s.log = bytes.Buffer{} s.cp = copier{ client: arvados.NewClientFromEnv(), - arvClient: api, hostOutputDir: tmpdir, ctrOutputDir: "/ctr/outdir", mounts: map[string]arvados.Mount{ @@ -37,13 +39,10 @@ func (s *copierSuite) SetUpTest(c *check.C) { secretMounts: map[string]arvados.Mount{ "/secret_text": {Kind: "text", Content: "xyzzy"}, }, + logger: &logrus.Logger{Out: &s.log, Formatter: &logrus.TextFormatter{}, Level: logrus.InfoLevel}, } } -func (s *copierSuite) TearDownTest(c *check.C) { - os.RemoveAll(s.cp.hostOutputDir) -} - func (s *copierSuite) TestEmptyOutput(c *check.C) { err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true) c.Check(err, check.IsNil) @@ -59,6 +58,8 @@ func (s *copierSuite) TestRegularFilesAndDirs(c *check.C) { _, err = io.WriteString(f, "foo") c.Assert(err, check.IsNil) c.Assert(f.Close(), check.IsNil) + err = syscall.Mkfifo(s.cp.hostOutputDir+"/dir1/fifo", 0644) + c.Assert(err, check.IsNil) err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true) c.Check(err, check.IsNil) @@ -67,6 +68,7 @@ func (s *copierSuite) TestRegularFilesAndDirs(c *check.C) { {src: os.DevNull, dst: "/dir1/dir2/dir3/.keep"}, {src: s.cp.hostOutputDir + "/dir1/foo", dst: "/dir1/foo", size: 3}, }) + c.Check(s.log.String(), check.Matches, `.* msg="Skipping unsupported file type \(mode 200000644\) in output dir: \\"/ctr/outdir/dir1/fifo\\""\n`) } func (s *copierSuite) TestSymlinkCycle(c *check.C) { @@ -128,7 +130,9 @@ func (s *copierSuite) TestSymlinkToMountedCollection(c *check.C) { PortableDataHash: arvadostest.FooCollectionPDH, Writable: true, } - s.cp.binds = append(s.cp.binds, bindtmp+":/mnt-w") + s.cp.bindmounts = map[string]bindmount{ + "/mnt-w": bindmount{HostPath: bindtmp, ReadOnly: false}, + } c.Assert(os.Symlink("../../mnt", s.cp.hostOutputDir+"/l_dir"), check.IsNil) c.Assert(os.Symlink("/mnt/foo", s.cp.hostOutputDir+"/l_file"), check.IsNil) @@ -220,3 +224,184 @@ func (s *copierSuite) writeFileInOutputDir(c *check.C, path, data string) { c.Assert(err, check.IsNil) c.Assert(f.Close(), check.IsNil) } + +// applyGlobsToFilesAndDirs uses the same glob-matching code as +// applyGlobsToCollectionFS, so we don't need to test all of the same +// glob-matching behavior covered in TestApplyGlobsToCollectionFS. We +// do need to check that (a) the glob is actually being used to filter +// out files, and (b) non-matching dirs still included if and only if +// they are ancestors of matching files. +func (s *copierSuite) TestApplyGlobsToFilesAndDirs(c *check.C) { + dirs := []string{"dir1", "dir1/dir11", "dir1/dir12", "dir2"} + files := []string{"dir1/file11", "dir1/dir11/file111", "dir2/file2"} + for _, trial := range []struct { + globs []string + dirs []string + files []string + }{ + { + globs: []string{}, + dirs: append([]string{}, dirs...), + files: append([]string{}, files...), + }, + { + globs: []string{"**"}, + dirs: append([]string{}, dirs...), + files: append([]string{}, files...), + }, + { + globs: []string{"**/file111"}, + dirs: []string{"dir1", "dir1/dir11"}, + files: []string{"dir1/dir11/file111"}, + }, + { + globs: []string{"nothing"}, + dirs: nil, + files: nil, + }, + { + globs: []string{"**/dir12"}, + dirs: []string{"dir1", "dir1/dir12"}, + files: nil, + }, + { + globs: []string{"**/file*"}, + dirs: []string{"dir1", "dir1/dir11", "dir2"}, + files: append([]string{}, files...), + }, + { + globs: []string{"**/dir1[12]"}, + dirs: []string{"dir1", "dir1/dir11", "dir1/dir12"}, + files: nil, + }, + { + globs: []string{"**/dir1[^2]"}, + dirs: []string{"dir1", "dir1/dir11"}, + files: nil, + }, + { + globs: []string{"dir1/**"}, + dirs: []string{"dir1", "dir1/dir11", "dir1/dir12"}, + files: []string{"dir1/file11", "dir1/dir11/file111"}, + }, + } { + c.Logf("=== globs: %q", trial.globs) + cp := copier{ + globs: trial.globs, + dirs: dirs, + } + for _, path := range files { + cp.files = append(cp.files, filetodo{dst: path}) + } + cp.applyGlobsToFilesAndDirs() + var gotFiles []string + for _, file := range cp.files { + gotFiles = append(gotFiles, file.dst) + } + c.Check(cp.dirs, check.DeepEquals, trial.dirs) + c.Check(gotFiles, check.DeepEquals, trial.files) + } +} + +func (s *copierSuite) TestApplyGlobsToCollectionFS(c *check.C) { + for _, trial := range []struct { + globs []string + expect []string + }{ + { + globs: nil, + expect: []string{"foo", "bar", "baz/quux", "baz/parent1/item1"}, + }, + { + globs: []string{"foo"}, + expect: []string{"foo"}, + }, + { + globs: []string{"baz/parent1/item1"}, + expect: []string{"baz/parent1/item1"}, + }, + { + globs: []string{"**"}, + expect: []string{"foo", "bar", "baz/quux", "baz/parent1/item1"}, + }, + { + globs: []string{"**/*"}, + expect: []string{"foo", "bar", "baz/quux", "baz/parent1/item1"}, + }, + { + globs: []string{"*"}, + expect: []string{"foo", "bar"}, + }, + { + globs: []string{"baz"}, + expect: nil, + }, + { + globs: []string{"b*/**"}, + expect: []string{"baz/quux", "baz/parent1/item1"}, + }, + { + globs: []string{"baz"}, + expect: nil, + }, + { + globs: []string{"baz/**"}, + expect: []string{"baz/quux", "baz/parent1/item1"}, + }, + { + globs: []string{"baz/*"}, + expect: []string{"baz/quux"}, + }, + { + globs: []string{"baz/**/*uu?"}, + expect: []string{"baz/quux"}, + }, + { + globs: []string{"**/*m1"}, + expect: []string{"baz/parent1/item1"}, + }, + { + globs: []string{"*/*/*/**/*1"}, + expect: nil, + }, + { + globs: []string{"f*", "**/q*"}, + expect: []string{"foo", "baz/quux"}, + }, + { + globs: []string{"\\"}, // invalid pattern matches nothing + expect: nil, + }, + { + globs: []string{"\\", "foo"}, + expect: []string{"foo"}, + }, + { + globs: []string{"foo/**"}, + expect: nil, + }, + { + globs: []string{"foo*/**"}, + expect: nil, + }, + } { + c.Logf("=== globs: %q", trial.globs) + collfs, err := (&arvados.Collection{ManifestText: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:bar 0:0:baz/quux 0:0:baz/parent1/item1\n"}).FileSystem(nil, nil) + c.Assert(err, check.IsNil) + cp := copier{globs: trial.globs} + err = cp.applyGlobsToCollectionFS(collfs) + if !c.Check(err, check.IsNil) { + continue + } + var got []string + fs.WalkDir(arvados.FS(collfs), "", func(path string, ent fs.DirEntry, err error) error { + if !ent.IsDir() { + got = append(got, path) + } + return nil + }) + sort.Strings(got) + sort.Strings(trial.expect) + c.Check(got, check.DeepEquals, trial.expect) + } +}