1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
12 "git.curoverse.com/arvados.git/sdk/go/arvados"
13 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
14 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
15 check "gopkg.in/check.v1"
18 var _ = check.Suite(&copierSuite{})
20 type copierSuite struct {
24 func (s *copierSuite) SetUpTest(c *check.C) {
25 tmpdir, err := ioutil.TempDir("", "crunch-run.test.")
26 c.Assert(err, check.IsNil)
27 api, err := arvadosclient.MakeArvadosClient()
28 c.Assert(err, check.IsNil)
30 client: arvados.NewClientFromEnv(),
32 hostOutputDir: tmpdir,
33 ctrOutputDir: "/ctr/outdir",
34 mounts: map[string]arvados.Mount{
35 "/ctr/outdir": {Kind: "tmp"},
37 secretMounts: map[string]arvados.Mount{
38 "/secret_text": {Kind: "text", Content: "xyzzy"},
43 func (s *copierSuite) TearDownTest(c *check.C) {
44 os.RemoveAll(s.cp.hostOutputDir)
47 func (s *copierSuite) TestEmptyOutput(c *check.C) {
48 err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
49 c.Check(err, check.IsNil)
50 c.Check(s.cp.dirs, check.DeepEquals, []string(nil))
51 c.Check(len(s.cp.files), check.Equals, 0)
54 func (s *copierSuite) TestRegularFilesAndDirs(c *check.C) {
55 err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
56 c.Assert(err, check.IsNil)
57 f, err := os.OpenFile(s.cp.hostOutputDir+"/dir1/foo", os.O_CREATE|os.O_WRONLY, 0644)
58 c.Assert(err, check.IsNil)
59 _, err = io.WriteString(f, "foo")
60 c.Assert(err, check.IsNil)
61 c.Assert(f.Close(), check.IsNil)
63 err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
64 c.Check(err, check.IsNil)
65 c.Check(s.cp.dirs, check.DeepEquals, []string{"/dir1", "/dir1/dir2", "/dir1/dir2/dir3"})
66 c.Check(s.cp.files, check.DeepEquals, []filetodo{
67 {src: os.DevNull, dst: "/dir1/dir2/dir3/.keep"},
68 {src: s.cp.hostOutputDir + "/dir1/foo", dst: "/dir1/foo", size: 3},
72 func (s *copierSuite) TestSymlinkCycle(c *check.C) {
73 c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir1", 0755), check.IsNil)
74 c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir2", 0755), check.IsNil)
75 c.Assert(os.Symlink("../dir2", s.cp.hostOutputDir+"/dir1/l_dir2"), check.IsNil)
76 c.Assert(os.Symlink("../dir1", s.cp.hostOutputDir+"/dir2/l_dir1"), check.IsNil)
77 err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
78 c.Check(err, check.ErrorMatches, `.*cycle.*`)
81 func (s *copierSuite) TestSymlinkTargetMissing(c *check.C) {
82 c.Assert(os.Symlink("./missing", s.cp.hostOutputDir+"/symlink"), check.IsNil)
83 err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
84 c.Check(err, check.ErrorMatches, `.*/ctr/outdir/missing.*`)
87 func (s *copierSuite) TestSymlinkTargetNotMounted(c *check.C) {
88 c.Assert(os.Symlink("../boop", s.cp.hostOutputDir+"/symlink"), check.IsNil)
89 err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
90 c.Check(err, check.ErrorMatches, `.*/ctr/boop.*`)
93 func (s *copierSuite) TestSymlinkToSecret(c *check.C) {
94 c.Assert(os.Symlink("/secret_text", s.cp.hostOutputDir+"/symlink"), check.IsNil)
95 err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
96 c.Check(err, check.IsNil)
97 c.Check(len(s.cp.dirs), check.Equals, 0)
98 c.Check(len(s.cp.files), check.Equals, 0)
101 func (s *copierSuite) TestSecretInOutputDir(c *check.C) {
102 s.cp.secretMounts["/ctr/outdir/secret_text"] = s.cp.secretMounts["/secret_text"]
103 s.writeFileInOutputDir(c, "secret_text", "xyzzy")
104 err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
105 c.Check(err, check.IsNil)
106 c.Check(len(s.cp.dirs), check.Equals, 0)
107 c.Check(len(s.cp.files), check.Equals, 0)
110 func (s *copierSuite) TestSymlinkToMountedCollection(c *check.C) {
111 s.cp.mounts["/mnt"] = arvados.Mount{
113 PortableDataHash: arvadostest.FooPdh,
115 c.Assert(os.Symlink("../../mnt", s.cp.hostOutputDir+"/l_dir"), check.IsNil)
116 c.Assert(os.Symlink("/mnt/foo", s.cp.hostOutputDir+"/l_file"), check.IsNil)
118 err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
119 c.Check(err, check.IsNil)
120 c.Check(s.cp.manifest, check.Matches, `(?ms)\./l_dir acbd\S+ 0:3:foo\n\. acbd\S+ 0:3:l_file\n`)
123 func (s *copierSuite) TestSymlink(c *check.C) {
124 hostfile := s.cp.hostOutputDir + "/dir1/file"
126 err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
127 c.Assert(err, check.IsNil)
128 s.writeFileInOutputDir(c, "dir1/file", "file")
129 for _, err := range []error{
130 os.Symlink(s.cp.ctrOutputDir+"/dir1/file", s.cp.hostOutputDir+"/l_abs_file"),
131 os.Symlink(s.cp.ctrOutputDir+"/dir1/dir2", s.cp.hostOutputDir+"/l_abs_dir2"),
132 os.Symlink("../../dir1/file", s.cp.hostOutputDir+"/dir1/dir2/l_rel_file"),
133 os.Symlink("dir1/file", s.cp.hostOutputDir+"/l_rel_file"),
134 os.MkdirAll(s.cp.hostOutputDir+"/morelinks", 0755),
135 os.Symlink("../dir1/dir2", s.cp.hostOutputDir+"/morelinks/l_rel_dir2"),
136 os.Symlink("dir1/dir2/dir3", s.cp.hostOutputDir+"/l_rel_dir3"),
138 c.Assert(err, check.IsNil)
141 err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
142 c.Check(err, check.IsNil)
143 c.Check(s.cp.dirs, check.DeepEquals, []string{
144 "/dir1", "/dir1/dir2", "/dir1/dir2/dir3",
145 "/l_abs_dir2", "/l_abs_dir2/dir3",
147 "/morelinks", "/morelinks/l_rel_dir2", "/morelinks/l_rel_dir2/dir3",
149 c.Check(s.cp.files, check.DeepEquals, []filetodo{
150 {dst: "/dir1/dir2/dir3/.keep", src: os.DevNull},
151 {dst: "/dir1/dir2/l_rel_file", src: hostfile, size: 4},
152 {dst: "/dir1/file", src: hostfile, size: 4},
153 {dst: "/l_abs_dir2/dir3/.keep", src: os.DevNull},
154 {dst: "/l_abs_dir2/l_rel_file", src: hostfile, size: 4},
155 {dst: "/l_abs_file", src: hostfile, size: 4},
156 {dst: "/l_rel_dir3/.keep", src: os.DevNull},
157 {dst: "/l_rel_file", src: hostfile, size: 4},
158 {dst: "/morelinks/l_rel_dir2/dir3/.keep", src: os.DevNull},
159 {dst: "/morelinks/l_rel_dir2/l_rel_file", src: hostfile, size: 4},
163 func (s *copierSuite) TestUnsupportedOutputMount(c *check.C) {
164 s.cp.mounts["/ctr/outdir"] = arvados.Mount{Kind: "waz"}
165 err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
166 c.Check(err, check.NotNil)
169 func (s *copierSuite) TestUnsupportedMountKindBelow(c *check.C) {
170 s.cp.mounts["/ctr/outdir/dirk"] = arvados.Mount{Kind: "waz"}
171 err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
172 c.Check(err, check.NotNil)
175 func (s *copierSuite) TestUnsupportedWritableMountBelow(c *check.C) {
176 s.cp.mounts["/ctr/outdir/dirk"] = arvados.Mount{
178 PortableDataHash: arvadostest.FooPdh,
181 err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
182 c.Check(err, check.NotNil)
185 func (s *copierSuite) writeFileInOutputDir(c *check.C, path, data string) {
186 f, err := os.OpenFile(s.cp.hostOutputDir+"/"+path, os.O_CREATE|os.O_WRONLY, 0644)
187 c.Assert(err, check.IsNil)
188 _, err = io.WriteString(f, data)
189 c.Assert(err, check.IsNil)
190 c.Assert(f.Close(), check.IsNil)