1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
17 check "gopkg.in/check.v1"
21 // Importing arvadostest would be an import cycle, so these
22 // fixtures are duplicated here [until fs moves to a separate
24 fixtureActiveToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
25 fixtureAProjectUUID = "zzzzz-j7d0g-v955i6s2oi1cbso"
26 fixtureThisFilterGroupUUID = "zzzzz-j7d0g-thisfiltergroup"
27 fixtureAFilterGroupTwoUUID = "zzzzz-j7d0g-afiltergrouptwo"
28 fixtureAFilterGroupThreeUUID = "zzzzz-j7d0g-filtergroupthre"
29 fixtureAFilterGroupFourUUID = "zzzzz-j7d0g-filtergroupfour"
30 fixtureAFilterGroupFiveUUID = "zzzzz-j7d0g-filtergroupfive"
31 fixtureFooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
32 fixtureFooCollectionName = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
33 fixtureFooCollectionPDH = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
34 fixtureFooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
35 fixtureNonexistentCollection = "zzzzz-4zz18-totallynotexist"
36 fixtureStorageClassesDesiredArchive = "zzzzz-4zz18-3t236wr12769qqa"
37 fixtureBlobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
38 fixtureBlobSigningTTL = 336 * time.Hour
41 var _ = check.Suite(&SiteFSSuite{})
44 // Enable DebugLocksPanicMode sometimes. Don't enable it all
45 // the time, though -- it adds many calls to time.Sleep(),
46 // which could hide different bugs.
47 if time.Now().Second()&1 == 0 {
48 DebugLocksPanicMode = true
52 type SiteFSSuite struct {
58 func (s *SiteFSSuite) SetUpTest(c *check.C) {
60 APIHost: os.Getenv("ARVADOS_API_HOST"),
61 AuthToken: fixtureActiveToken,
64 s.kc = &keepClientStub{
65 blocks: map[string][]byte{
66 "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
68 sigkey: fixtureBlobSigningKey,
69 sigttl: fixtureBlobSigningTTL,
70 authToken: fixtureActiveToken,
72 s.fs = s.client.SiteFileSystem(s.kc)
75 func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
76 _, ok := s.fs.(http.FileSystem)
77 c.Check(ok, check.Equals, true)
80 func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
81 f, err := s.fs.Open("/by_id")
82 c.Assert(err, check.IsNil)
83 fis, err := f.Readdir(-1)
84 c.Check(err, check.IsNil)
85 c.Check(len(fis), check.Equals, 0)
88 func (s *SiteFSSuite) TestUpdateStorageClasses(c *check.C) {
89 f, err := s.fs.OpenFile("/by_id/"+fixtureStorageClassesDesiredArchive+"/newfile", os.O_CREATE|os.O_RDWR, 0777)
90 c.Assert(err, check.IsNil)
91 _, err = f.Write([]byte("nope"))
92 c.Assert(err, check.IsNil)
94 c.Assert(err, check.IsNil)
96 c.Assert(err, check.ErrorMatches, `.*stub does not write storage class "archive"`)
99 func (s *SiteFSSuite) TestByUUIDAndPDH(c *check.C) {
100 f, err := s.fs.Open("/by_id")
101 c.Assert(err, check.IsNil)
102 fis, err := f.Readdir(-1)
103 c.Check(err, check.IsNil)
104 c.Check(len(fis), check.Equals, 0)
106 err = s.fs.Mkdir("/by_id/"+fixtureFooCollection, 0755)
107 c.Check(err, check.Equals, os.ErrExist)
109 f, err = s.fs.Open("/by_id/" + fixtureNonexistentCollection)
110 c.Assert(err, check.Equals, os.ErrNotExist)
112 for _, path := range []string{
113 fixtureFooCollection,
114 fixtureFooCollectionPDH,
115 fixtureAProjectUUID + "/" + fixtureFooCollectionName,
117 f, err = s.fs.Open("/by_id/" + path)
118 c.Assert(err, check.IsNil)
119 fis, err = f.Readdir(-1)
120 c.Assert(err, check.IsNil)
122 for _, fi := range fis {
123 names = append(names, fi.Name())
125 c.Check(names, check.DeepEquals, []string{"foo"})
128 f, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file")
129 c.Assert(err, check.IsNil)
130 fis, err = f.Readdir(-1)
131 c.Assert(err, check.IsNil)
133 for _, fi := range fis {
134 names = append(names, fi.Name())
136 c.Check(names, check.DeepEquals, []string{"baz"})
138 _, err = s.fs.OpenFile("/by_id/"+fixtureNonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
139 c.Check(err, ErrorIs, ErrInvalidOperation)
140 err = s.fs.Rename("/by_id/"+fixtureFooCollection, "/by_id/beep")
141 c.Check(err, ErrorIs, ErrInvalidOperation)
142 err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/beep")
143 c.Check(err, ErrorIs, ErrInvalidOperation)
144 _, err = s.fs.Stat("/by_id/beep")
145 c.Check(err, check.Equals, os.ErrNotExist)
146 err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/"+fixtureFooCollection+"/bar")
147 c.Check(err, check.IsNil)
149 err = s.fs.Rename("/by_id", "/beep")
150 c.Check(err, ErrorIs, ErrInvalidOperation)
153 // Copy subtree from OS src to dst path inside fs. If src is a
154 // directory, dst must exist and be a directory.
155 func copyFromOS(fs FileSystem, dst, src string) error {
156 inf, err := os.Open(src)
161 dirents, err := inf.Readdir(-1)
162 if e, ok := err.(*os.PathError); ok {
163 if e, ok := e.Err.(syscall.Errno); ok {
164 if e == syscall.ENOTDIR {
165 err = syscall.ENOTDIR
169 if err == syscall.ENOTDIR {
170 outf, err := fs.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_TRUNC|os.O_WRONLY, 0700)
172 return fmt.Errorf("open %s: %s", dst, err)
175 _, err = io.Copy(outf, inf)
177 return fmt.Errorf("%s: copying data from %s: %s", dst, src, err)
183 } else if err != nil {
184 return fmt.Errorf("%s: readdir: %T %s", src, err, err)
187 d, err := fs.Open(dst)
189 return fmt.Errorf("opendir(%s): %s", dst, err)
193 for _, ent := range dirents {
194 if ent.Name() == "." || ent.Name() == ".." {
197 dstname := dst + "/" + ent.Name()
199 err = fs.Mkdir(dstname, 0700)
201 return fmt.Errorf("mkdir %s: %s", dstname, err)
204 err = copyFromOS(fs, dstname, src+"/"+ent.Name())
213 func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
214 s.fs.MountProject("home", "")
215 thisfile, err := ioutil.ReadFile("fs_site_test.go")
216 c.Assert(err, check.IsNil)
219 err = s.client.RequestAndDecode(&src1, "POST", "arvados/v1/collections", nil, map[string]interface{}{
220 "collection": map[string]string{
221 "name": "TestSnapshotSplice src1",
222 "owner_uuid": fixtureAProjectUUID,
225 c.Assert(err, check.IsNil)
226 defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src1.UUID, nil, nil)
228 c.Assert(err, check.IsNil)
229 err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src1", "..") // arvados.git/sdk/go
230 c.Assert(err, check.IsNil)
233 err = s.client.RequestAndDecode(&src2, "POST", "arvados/v1/collections", nil, map[string]interface{}{
234 "collection": map[string]string{
235 "name": "TestSnapshotSplice src2",
236 "owner_uuid": fixtureAProjectUUID,
239 c.Assert(err, check.IsNil)
240 defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src2.UUID, nil, nil)
242 c.Assert(err, check.IsNil)
243 err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src2", "..") // arvados.git/sdk/go
244 c.Assert(err, check.IsNil)
247 err = s.client.RequestAndDecode(&dst, "POST", "arvados/v1/collections", nil, map[string]interface{}{
248 "collection": map[string]string{
249 "name": "TestSnapshotSplice dst",
250 "owner_uuid": fixtureAProjectUUID,
253 c.Assert(err, check.IsNil)
254 defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+dst.UUID, nil, nil)
256 c.Assert(err, check.IsNil)
258 dstPath := "/home/A Project/TestSnapshotSplice dst"
259 err = copyFromOS(s.fs, dstPath, "..") // arvados.git/sdk/go
260 c.Assert(err, check.IsNil)
262 // Snapshot directory
263 snap1, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/ctxlog")
264 c.Check(err, check.IsNil)
265 // Attach same snapshot twice, at paths that didn't exist before
266 err = Splice(s.fs, dstPath+"/ctxlog-copy", snap1)
267 c.Check(err, check.IsNil)
268 err = Splice(s.fs, dstPath+"/ctxlog-copy2", snap1)
269 c.Check(err, check.IsNil)
270 // Splicing a snapshot twice results in two independent copies
271 err = s.fs.Rename(dstPath+"/ctxlog-copy2/log.go", dstPath+"/ctxlog-copy/log2.go")
272 c.Check(err, check.IsNil)
273 _, err = s.fs.Open(dstPath + "/ctxlog-copy2/log.go")
274 c.Check(err, check.Equals, os.ErrNotExist)
275 f, err := s.fs.Open(dstPath + "/ctxlog-copy/log.go")
276 if c.Check(err, check.IsNil) {
277 buf, err := ioutil.ReadAll(f)
278 c.Check(err, check.IsNil)
279 c.Check(string(buf), check.Not(check.Equals), "")
283 // Snapshot regular file
284 snapFile, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/arvados/fs_site_test.go")
285 c.Check(err, check.IsNil)
286 // Replace dir with file
287 err = Splice(s.fs, dstPath+"/ctxlog-copy2", snapFile)
288 c.Check(err, check.IsNil)
289 if f, err := s.fs.Open(dstPath + "/ctxlog-copy2"); c.Check(err, check.IsNil) {
290 buf, err := ioutil.ReadAll(f)
291 c.Check(err, check.IsNil)
292 c.Check(string(buf), check.Equals, string(thisfile))
295 // Cannot splice a file onto a collection root; cannot splice
296 // anything to a target outside a collection.
297 for _, badpath := range []string{
300 "/home/A Project/newnodename/",
301 "/home/A Project/newnodename",
304 "/home/newnodename/",
312 err = Splice(s.fs, badpath, snapFile)
313 c.Check(err, check.NotNil)
314 if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
315 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
317 c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
319 if strings.TrimSuffix(badpath, "/") == dstPath {
320 c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %q", badpath))
324 err = Splice(s.fs, badpath, snap1)
325 if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
326 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
328 c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
332 // Destination's parent must already exist
333 for _, badpath := range []string{
334 dstPath + "/newdirname/",
335 dstPath + "/newdirname/foobar",
338 err = Splice(s.fs, badpath, snap1)
339 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
340 err = Splice(s.fs, badpath, snapFile)
341 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
344 snap2, err := Snapshot(s.fs, dstPath+"/ctxlog-copy")
345 if c.Check(err, check.IsNil) {
346 err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
347 c.Check(err, check.IsNil)
350 // Snapshot entire collection, splice into same collection at
351 // a new path, remove file from original location, verify
352 // spliced content survives
353 snapDst, err := Snapshot(s.fs, dstPath+"")
354 c.Check(err, check.IsNil)
355 err = Splice(s.fs, dstPath+"", snapDst)
356 c.Check(err, check.IsNil)
357 err = Splice(s.fs, dstPath+"/copy1", snapDst)
358 c.Check(err, check.IsNil)
359 err = Splice(s.fs, dstPath+"/copy2", snapDst)
360 c.Check(err, check.IsNil)
361 err = s.fs.RemoveAll(dstPath + "/arvados/fs_site_test.go")
362 c.Check(err, check.IsNil)
363 err = s.fs.RemoveAll(dstPath + "/arvados")
364 c.Check(err, check.IsNil)
365 _, err = s.fs.Open(dstPath + "/arvados/fs_site_test.go")
366 c.Check(err, check.Equals, os.ErrNotExist)
367 f, err = s.fs.Open(dstPath + "/copy2/arvados/fs_site_test.go")
368 if c.Check(err, check.IsNil) {
370 buf, err := ioutil.ReadAll(f)
371 c.Check(err, check.IsNil)
372 c.Check(string(buf), check.Equals, string(thisfile))