17417: Merge branch 'main' into 17417-add-arm64
[arvados.git] / lib / crunchrun / copier_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package crunchrun
6
7 import (
8         "bytes"
9         "io"
10         "io/ioutil"
11         "os"
12         "syscall"
13
14         "git.arvados.org/arvados.git/sdk/go/arvados"
15         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
16         "git.arvados.org/arvados.git/sdk/go/arvadostest"
17         "github.com/sirupsen/logrus"
18         check "gopkg.in/check.v1"
19 )
20
21 var _ = check.Suite(&copierSuite{})
22
23 type copierSuite struct {
24         cp  copier
25         log bytes.Buffer
26 }
27
28 func (s *copierSuite) SetUpTest(c *check.C) {
29         tmpdir := c.MkDir()
30         api, err := arvadosclient.MakeArvadosClient()
31         c.Assert(err, check.IsNil)
32         s.log = bytes.Buffer{}
33         s.cp = copier{
34                 client:        arvados.NewClientFromEnv(),
35                 arvClient:     api,
36                 hostOutputDir: tmpdir,
37                 ctrOutputDir:  "/ctr/outdir",
38                 mounts: map[string]arvados.Mount{
39                         "/ctr/outdir": {Kind: "tmp"},
40                 },
41                 secretMounts: map[string]arvados.Mount{
42                         "/secret_text": {Kind: "text", Content: "xyzzy"},
43                 },
44                 logger: &logrus.Logger{Out: &s.log, Formatter: &logrus.TextFormatter{}, Level: logrus.InfoLevel},
45         }
46 }
47
48 func (s *copierSuite) TestEmptyOutput(c *check.C) {
49         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
50         c.Check(err, check.IsNil)
51         c.Check(s.cp.dirs, check.DeepEquals, []string(nil))
52         c.Check(len(s.cp.files), check.Equals, 0)
53 }
54
55 func (s *copierSuite) TestRegularFilesAndDirs(c *check.C) {
56         err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
57         c.Assert(err, check.IsNil)
58         f, err := os.OpenFile(s.cp.hostOutputDir+"/dir1/foo", os.O_CREATE|os.O_WRONLY, 0644)
59         c.Assert(err, check.IsNil)
60         _, err = io.WriteString(f, "foo")
61         c.Assert(err, check.IsNil)
62         c.Assert(f.Close(), check.IsNil)
63         err = syscall.Mkfifo(s.cp.hostOutputDir+"/dir1/fifo", 0644)
64         c.Assert(err, check.IsNil)
65
66         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
67         c.Check(err, check.IsNil)
68         c.Check(s.cp.dirs, check.DeepEquals, []string{"/dir1", "/dir1/dir2", "/dir1/dir2/dir3"})
69         c.Check(s.cp.files, check.DeepEquals, []filetodo{
70                 {src: os.DevNull, dst: "/dir1/dir2/dir3/.keep"},
71                 {src: s.cp.hostOutputDir + "/dir1/foo", dst: "/dir1/foo", size: 3},
72         })
73         c.Check(s.log.String(), check.Matches, `.* msg="Skipping unsupported file type \(mode 200000644\) in output dir: \\"/ctr/outdir/dir1/fifo\\""\n`)
74 }
75
76 func (s *copierSuite) TestSymlinkCycle(c *check.C) {
77         c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir1", 0755), check.IsNil)
78         c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir2", 0755), check.IsNil)
79         c.Assert(os.Symlink("../dir2", s.cp.hostOutputDir+"/dir1/l_dir2"), check.IsNil)
80         c.Assert(os.Symlink("../dir1", s.cp.hostOutputDir+"/dir2/l_dir1"), check.IsNil)
81         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
82         c.Check(err, check.ErrorMatches, `.*cycle.*`)
83 }
84
85 func (s *copierSuite) TestSymlinkTargetMissing(c *check.C) {
86         c.Assert(os.Symlink("./missing", s.cp.hostOutputDir+"/symlink"), check.IsNil)
87         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
88         c.Check(err, check.ErrorMatches, `.*/ctr/outdir/missing.*`)
89 }
90
91 func (s *copierSuite) TestSymlinkTargetNotMounted(c *check.C) {
92         c.Assert(os.Symlink("../boop", s.cp.hostOutputDir+"/symlink"), check.IsNil)
93         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
94         c.Check(err, check.ErrorMatches, `.*/ctr/boop.*`)
95 }
96
97 func (s *copierSuite) TestSymlinkToSecret(c *check.C) {
98         c.Assert(os.Symlink("/secret_text", s.cp.hostOutputDir+"/symlink"), check.IsNil)
99         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
100         c.Check(err, check.IsNil)
101         c.Check(len(s.cp.dirs), check.Equals, 0)
102         c.Check(len(s.cp.files), check.Equals, 0)
103 }
104
105 func (s *copierSuite) TestSecretInOutputDir(c *check.C) {
106         s.cp.secretMounts["/ctr/outdir/secret_text"] = s.cp.secretMounts["/secret_text"]
107         s.writeFileInOutputDir(c, "secret_text", "xyzzy")
108         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
109         c.Check(err, check.IsNil)
110         c.Check(len(s.cp.dirs), check.Equals, 0)
111         c.Check(len(s.cp.files), check.Equals, 0)
112 }
113
114 func (s *copierSuite) TestSymlinkToMountedCollection(c *check.C) {
115         // simulate mounted read-only collection
116         s.cp.mounts["/mnt"] = arvados.Mount{
117                 Kind:             "collection",
118                 PortableDataHash: arvadostest.FooCollectionPDH,
119         }
120
121         // simulate mounted writable collection
122         bindtmp, err := ioutil.TempDir("", "crunch-run.test.")
123         c.Assert(err, check.IsNil)
124         defer os.RemoveAll(bindtmp)
125         f, err := os.OpenFile(bindtmp+"/.arvados#collection", os.O_CREATE|os.O_WRONLY, 0644)
126         c.Assert(err, check.IsNil)
127         _, err = io.WriteString(f, `{"manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"}`)
128         c.Assert(err, check.IsNil)
129         c.Assert(f.Close(), check.IsNil)
130         s.cp.mounts["/mnt-w"] = arvados.Mount{
131                 Kind:             "collection",
132                 PortableDataHash: arvadostest.FooCollectionPDH,
133                 Writable:         true,
134         }
135         s.cp.bindmounts = map[string]bindmount{
136                 "/mnt-w": bindmount{HostPath: bindtmp, ReadOnly: false},
137         }
138
139         c.Assert(os.Symlink("../../mnt", s.cp.hostOutputDir+"/l_dir"), check.IsNil)
140         c.Assert(os.Symlink("/mnt/foo", s.cp.hostOutputDir+"/l_file"), check.IsNil)
141         c.Assert(os.Symlink("/mnt-w/bar", s.cp.hostOutputDir+"/l_file_w"), check.IsNil)
142
143         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
144         c.Check(err, check.IsNil)
145         c.Check(s.cp.manifest, check.Matches, `(?ms)\./l_dir acbd\S+ 0:3:foo\n\. acbd\S+ 0:3:l_file\n\. 37b5\S+ 0:3:l_file_w\n`)
146 }
147
148 func (s *copierSuite) TestSymlink(c *check.C) {
149         hostfile := s.cp.hostOutputDir + "/dir1/file"
150
151         err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
152         c.Assert(err, check.IsNil)
153         s.writeFileInOutputDir(c, "dir1/file", "file")
154         for _, err := range []error{
155                 os.Symlink(s.cp.ctrOutputDir+"/dir1/file", s.cp.hostOutputDir+"/l_abs_file"),
156                 os.Symlink(s.cp.ctrOutputDir+"/dir1/dir2", s.cp.hostOutputDir+"/l_abs_dir2"),
157                 os.Symlink("../../dir1/file", s.cp.hostOutputDir+"/dir1/dir2/l_rel_file"),
158                 os.Symlink("dir1/file", s.cp.hostOutputDir+"/l_rel_file"),
159                 os.MkdirAll(s.cp.hostOutputDir+"/morelinks", 0755),
160                 os.Symlink("../dir1/dir2", s.cp.hostOutputDir+"/morelinks/l_rel_dir2"),
161                 os.Symlink("dir1/dir2/dir3", s.cp.hostOutputDir+"/l_rel_dir3"),
162                 // rel. symlink -> rel. symlink -> regular file
163                 os.Symlink("../dir1/dir2/l_rel_file", s.cp.hostOutputDir+"/morelinks/l_rel_l_rel_file"),
164         } {
165                 c.Assert(err, check.IsNil)
166         }
167
168         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
169         c.Check(err, check.IsNil)
170         c.Check(s.cp.dirs, check.DeepEquals, []string{
171                 "/dir1", "/dir1/dir2", "/dir1/dir2/dir3",
172                 "/l_abs_dir2", "/l_abs_dir2/dir3",
173                 "/l_rel_dir3",
174                 "/morelinks", "/morelinks/l_rel_dir2", "/morelinks/l_rel_dir2/dir3",
175         })
176         c.Check(s.cp.files, check.DeepEquals, []filetodo{
177                 {dst: "/dir1/dir2/dir3/.keep", src: os.DevNull},
178                 {dst: "/dir1/dir2/l_rel_file", src: hostfile, size: 4},
179                 {dst: "/dir1/file", src: hostfile, size: 4},
180                 {dst: "/l_abs_dir2/dir3/.keep", src: os.DevNull},
181                 {dst: "/l_abs_dir2/l_rel_file", src: hostfile, size: 4},
182                 {dst: "/l_abs_file", src: hostfile, size: 4},
183                 {dst: "/l_rel_dir3/.keep", src: os.DevNull},
184                 {dst: "/l_rel_file", src: hostfile, size: 4},
185                 {dst: "/morelinks/l_rel_dir2/dir3/.keep", src: os.DevNull},
186                 {dst: "/morelinks/l_rel_dir2/l_rel_file", src: hostfile, size: 4},
187                 {dst: "/morelinks/l_rel_l_rel_file", src: hostfile, size: 4},
188         })
189 }
190
191 func (s *copierSuite) TestUnsupportedOutputMount(c *check.C) {
192         s.cp.mounts["/ctr/outdir"] = arvados.Mount{Kind: "waz"}
193         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
194         c.Check(err, check.NotNil)
195 }
196
197 func (s *copierSuite) TestUnsupportedMountKindBelow(c *check.C) {
198         s.cp.mounts["/ctr/outdir/dirk"] = arvados.Mount{Kind: "waz"}
199         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
200         c.Check(err, check.NotNil)
201 }
202
203 func (s *copierSuite) TestWritableMountBelow(c *check.C) {
204         s.cp.mounts["/ctr/outdir/mount"] = arvados.Mount{
205                 Kind:             "collection",
206                 PortableDataHash: arvadostest.FooCollectionPDH,
207                 Writable:         true,
208         }
209         c.Assert(os.MkdirAll(s.cp.hostOutputDir+"/mount", 0755), check.IsNil)
210         s.writeFileInOutputDir(c, "file", "file")
211         s.writeFileInOutputDir(c, "mount/foo", "foo")
212
213         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
214         c.Check(err, check.IsNil)
215         c.Check(s.cp.dirs, check.DeepEquals, []string{"/mount"})
216         c.Check(s.cp.files, check.DeepEquals, []filetodo{
217                 {src: s.cp.hostOutputDir + "/file", dst: "/file", size: 4},
218                 {src: s.cp.hostOutputDir + "/mount/foo", dst: "/mount/foo", size: 3},
219         })
220 }
221
222 func (s *copierSuite) writeFileInOutputDir(c *check.C, path, data string) {
223         f, err := os.OpenFile(s.cp.hostOutputDir+"/"+path, os.O_CREATE|os.O_WRONLY, 0644)
224         c.Assert(err, check.IsNil)
225         _, err = io.WriteString(f, data)
226         c.Assert(err, check.IsNil)
227         c.Assert(f.Close(), check.IsNil)
228 }