sdk/go/arvados/fs_site_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "fmt"
   9         "io"
  10         "io/ioutil"
  11         "net/http"
  12         "os"
  13         "syscall"
  14         "time"
  15
  16         check "gopkg.in/check.v1"
  17 )
  18
  19 const (
  20         // Importing arvadostest would be an import cycle, so these
  21         // fixtures are duplicated here [until fs moves to a separate
  22         // package].
  23         fixtureActiveToken                  = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
  24         fixtureAProjectUUID                 = "zzzzz-j7d0g-v955i6s2oi1cbso"
  25         fixtureThisFilterGroupUUID          = "zzzzz-j7d0g-thisfiltergroup"
  26         fixtureAFilterGroupTwoUUID          = "zzzzz-j7d0g-afiltergrouptwo"
  27         fixtureAFilterGroupThreeUUID        = "zzzzz-j7d0g-filtergroupthre"
  28         fixtureAFilterGroupFourUUID         = "zzzzz-j7d0g-filtergroupfour"
  29         fixtureAFilterGroupFiveUUID         = "zzzzz-j7d0g-filtergroupfive"
  30         fixtureFooAndBarFilesInDirUUID      = "zzzzz-4zz18-foonbarfilesdir"
  31         fixtureFooCollectionName            = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
  32         fixtureFooCollectionPDH             = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
  33         fixtureFooCollection                = "zzzzz-4zz18-fy296fx3hot09f7"
  34         fixtureNonexistentCollection        = "zzzzz-4zz18-totallynotexist"
  35         fixtureStorageClassesDesiredArchive = "zzzzz-4zz18-3t236wr12769qqa"
  36         fixtureBlobSigningKey               = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
  37         fixtureBlobSigningTTL               = 336 * time.Hour
  38 )
  39
  40 var _ = check.Suite(&SiteFSSuite{})
  41
  42 func init() {
  43         // Enable DebugLocksPanicMode sometimes. Don't enable it all
  44         // the time, though -- it adds many calls to time.Sleep(),
  45         // which could hide different bugs.
  46         if time.Now().Second()&1 == 0 {
  47                 DebugLocksPanicMode = true
  48         }
  49 }
  50
  51 type SiteFSSuite struct {
  52         client *Client
  53         fs     CustomFileSystem
  54         kc     keepClient
  55 }
  56
  57 func (s *SiteFSSuite) SetUpTest(c *check.C) {
  58         s.client = &Client{
  59                 APIHost:   os.Getenv("ARVADOS_API_HOST"),
  60                 AuthToken: fixtureActiveToken,
  61                 Insecure:  true,
  62         }
  63         s.kc = &keepClientStub{
  64                 blocks: map[string][]byte{
  65                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  66                 },
  67                 sigkey:    fixtureBlobSigningKey,
  68                 sigttl:    fixtureBlobSigningTTL,
  69                 authToken: fixtureActiveToken,
  70         }
  71         s.fs = s.client.SiteFileSystem(s.kc)
  72 }
  73
  74 func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
  75         _, ok := s.fs.(http.FileSystem)
  76         c.Check(ok, check.Equals, true)
  77 }
  78
  79 func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
  80         f, err := s.fs.Open("/by_id")
  81         c.Assert(err, check.IsNil)
  82         fis, err := f.Readdir(-1)
  83         c.Check(err, check.IsNil)
  84         c.Check(len(fis), check.Equals, 0)
  85 }
  86
  87 func (s *SiteFSSuite) TestUpdateStorageClasses(c *check.C) {
  88         f, err := s.fs.OpenFile("/by_id/"+fixtureStorageClassesDesiredArchive+"/newfile", os.O_CREATE|os.O_RDWR, 0777)
  89         c.Assert(err, check.IsNil)
  90         _, err = f.Write([]byte("nope"))
  91         c.Assert(err, check.IsNil)
  92         err = f.Close()
  93         c.Assert(err, check.IsNil)
  94         err = s.fs.Sync()
  95         c.Assert(err, check.ErrorMatches, `.*stub does not write storage class "archive"`)
  96 }
  97
  98 func (s *SiteFSSuite) TestByUUIDAndPDH(c *check.C) {
  99         f, err := s.fs.Open("/by_id")
 100         c.Assert(err, check.IsNil)
 101         fis, err := f.Readdir(-1)
 102         c.Check(err, check.IsNil)
 103         c.Check(len(fis), check.Equals, 0)
 104
 105         err = s.fs.Mkdir("/by_id/"+fixtureFooCollection, 0755)
 106         c.Check(err, check.Equals, os.ErrExist)
 107
 108         f, err = s.fs.Open("/by_id/" + fixtureNonexistentCollection)
 109         c.Assert(err, check.Equals, os.ErrNotExist)
 110
 111         for _, path := range []string{
 112                 fixtureFooCollection,
 113                 fixtureFooCollectionPDH,
 114                 fixtureAProjectUUID + "/" + fixtureFooCollectionName,
 115         } {
 116                 f, err = s.fs.Open("/by_id/" + path)
 117                 c.Assert(err, check.IsNil)
 118                 fis, err = f.Readdir(-1)
 119                 c.Assert(err, check.IsNil)
 120                 var names []string
 121                 for _, fi := range fis {
 122                         names = append(names, fi.Name())
 123                 }
 124                 c.Check(names, check.DeepEquals, []string{"foo"})
 125         }
 126
 127         f, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file")
 128         c.Assert(err, check.IsNil)
 129         fis, err = f.Readdir(-1)
 130         c.Assert(err, check.IsNil)
 131         var names []string
 132         for _, fi := range fis {
 133                 names = append(names, fi.Name())
 134         }
 135         c.Check(names, check.DeepEquals, []string{"baz"})
 136
 137         _, err = s.fs.OpenFile("/by_id/"+fixtureNonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
 138         c.Check(err, ErrorIs, ErrInvalidOperation)
 139         err = s.fs.Rename("/by_id/"+fixtureFooCollection, "/by_id/beep")
 140         c.Check(err, ErrorIs, ErrInvalidOperation)
 141         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/beep")
 142         c.Check(err, ErrorIs, ErrInvalidOperation)
 143         _, err = s.fs.Stat("/by_id/beep")
 144         c.Check(err, check.Equals, os.ErrNotExist)
 145         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/"+fixtureFooCollection+"/bar")
 146         c.Check(err, check.IsNil)
 147
 148         err = s.fs.Rename("/by_id", "/beep")
 149         c.Check(err, ErrorIs, ErrInvalidOperation)
 150 }
 151
 152 // Copy subtree from OS src to dst path inside fs. If src is a
 153 // directory, dst must exist and be a directory.
 154 func copyFromOS(fs FileSystem, dst, src string) error {
 155         inf, err := os.Open(src)
 156         if err != nil {
 157                 return err
 158         }
 159         defer inf.Close()
 160         dirents, err := inf.Readdir(-1)
 161         if e, ok := err.(*os.PathError); ok {
 162                 if e, ok := e.Err.(syscall.Errno); ok {
 163                         if e == syscall.ENOTDIR {
 164                                 err = syscall.ENOTDIR
 165                         }
 166                 }
 167         }
 168         if err == syscall.ENOTDIR {
 169                 outf, err := fs.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_TRUNC|os.O_WRONLY, 0700)
 170                 if err != nil {
 171                         return fmt.Errorf("open %s: %s", dst, err)
 172                 }
 173                 defer outf.Close()
 174                 _, err = io.Copy(outf, inf)
 175                 if err != nil {
 176                         return fmt.Errorf("%s: copying data from %s: %s", dst, src, err)
 177                 }
 178                 err = outf.Close()
 179                 if err != nil {
 180                         return err
 181                 }
 182         } else if err != nil {
 183                 return fmt.Errorf("%s: readdir: %T %s", src, err, err)
 184         } else {
 185                 {
 186                         d, err := fs.Open(dst)
 187                         if err != nil {
 188                                 return fmt.Errorf("opendir(%s): %s", dst, err)
 189                         }
 190                         d.Close()
 191                 }
 192                 for _, ent := range dirents {
 193                         if ent.Name() == "." || ent.Name() == ".." {
 194                                 continue
 195                         }
 196                         dstname := dst + "/" + ent.Name()
 197                         if ent.IsDir() {
 198                                 err = fs.Mkdir(dstname, 0700)
 199                                 if err != nil {
 200                                         return fmt.Errorf("mkdir %s: %s", dstname, err)
 201                                 }
 202                         }
 203                         err = copyFromOS(fs, dstname, src+"/"+ent.Name())
 204                         if err != nil {
 205                                 return err
 206                         }
 207                 }
 208         }
 209         return nil
 210 }
 211
 212 func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
 213         s.fs.MountProject("home", "")
 214         thisfile, err := ioutil.ReadFile("fs_site_test.go")
 215         c.Assert(err, check.IsNil)
 216
 217         var src1 Collection
 218         err = s.client.RequestAndDecode(&src1, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 219                 "collection": map[string]string{
 220                         "name":       "TestSnapshotSplice src1",
 221                         "owner_uuid": fixtureAProjectUUID,
 222                 },
 223         })
 224         c.Assert(err, check.IsNil)
 225         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src1.UUID, nil, nil)
 226         err = s.fs.Sync()
 227         c.Assert(err, check.IsNil)
 228         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src1", "..") // arvados.git/sdk/go
 229         c.Assert(err, check.IsNil)
 230
 231         var src2 Collection
 232         err = s.client.RequestAndDecode(&src2, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 233                 "collection": map[string]string{
 234                         "name":       "TestSnapshotSplice src2",
 235                         "owner_uuid": fixtureAProjectUUID,
 236                 },
 237         })
 238         c.Assert(err, check.IsNil)
 239         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src2.UUID, nil, nil)
 240         err = s.fs.Sync()
 241         c.Assert(err, check.IsNil)
 242         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src2", "..") // arvados.git/sdk/go
 243         c.Assert(err, check.IsNil)
 244
 245         var dst Collection
 246         err = s.client.RequestAndDecode(&dst, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 247                 "collection": map[string]string{
 248                         "name":       "TestSnapshotSplice dst",
 249                         "owner_uuid": fixtureAProjectUUID,
 250                 },
 251         })
 252         c.Assert(err, check.IsNil)
 253         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+dst.UUID, nil, nil)
 254         err = s.fs.Sync()
 255         c.Assert(err, check.IsNil)
 256
 257         dstPath := "/home/A Project/TestSnapshotSplice dst"
 258         err = copyFromOS(s.fs, dstPath, "..") // arvados.git/sdk/go
 259         c.Assert(err, check.IsNil)
 260
 261         // Snapshot directory
 262         snap1, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/ctxlog")
 263         c.Check(err, check.IsNil)
 264         // Attach same snapshot twice, at paths that didn't exist before
 265         err = Splice(s.fs, dstPath+"/ctxlog-copy", snap1)
 266         c.Check(err, check.IsNil)
 267         err = Splice(s.fs, dstPath+"/ctxlog-copy2", snap1)
 268         c.Check(err, check.IsNil)
 269         // Splicing a snapshot twice results in two independent copies
 270         err = s.fs.Rename(dstPath+"/ctxlog-copy2/log.go", dstPath+"/ctxlog-copy/log2.go")
 271         c.Check(err, check.IsNil)
 272         _, err = s.fs.Open(dstPath + "/ctxlog-copy2/log.go")
 273         c.Check(err, check.Equals, os.ErrNotExist)
 274         f, err := s.fs.Open(dstPath + "/ctxlog-copy/log.go")
 275         if c.Check(err, check.IsNil) {
 276                 buf, err := ioutil.ReadAll(f)
 277                 c.Check(err, check.IsNil)
 278                 c.Check(string(buf), check.Not(check.Equals), "")
 279                 f.Close()
 280         }
 281
 282         // Snapshot regular file
 283         snapFile, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/arvados/fs_site_test.go")
 284         c.Check(err, check.IsNil)
 285         // Replace dir with file
 286         err = Splice(s.fs, dstPath+"/ctxlog-copy2", snapFile)
 287         c.Check(err, check.IsNil)
 288         if f, err := s.fs.Open(dstPath + "/ctxlog-copy2"); c.Check(err, check.IsNil) {
 289                 buf, err := ioutil.ReadAll(f)
 290                 c.Check(err, check.IsNil)
 291                 c.Check(string(buf), check.Equals, string(thisfile))
 292         }
 293
 294         // Cannot splice a file onto a collection root, or anywhere
 295         // outside a collection
 296         for _, badpath := range []string{
 297                 dstPath,
 298                 "/home/A Project/newnodename",
 299                 "/home/A Project",
 300                 "/home/newnodename",
 301                 "/home",
 302                 "/newnodename",
 303         } {
 304                 err = Splice(s.fs, badpath, snapFile)
 305                 c.Check(err, check.NotNil)
 306                 c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %s"))
 307                 if badpath == dstPath {
 308                         c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %s", badpath))
 309                         continue
 310                 }
 311                 err = Splice(s.fs, badpath, snap1)
 312                 c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %s"))
 313         }
 314
 315         // Destination cannot have trailing slash
 316         for _, badpath := range []string{
 317                 dstPath + "/ctxlog/",
 318                 dstPath + "/",
 319                 "/home/A Project/",
 320                 "/home/",
 321                 "/",
 322                 "",
 323         } {
 324                 err = Splice(s.fs, badpath, snap1)
 325                 c.Check(err, ErrorIs, ErrInvalidArgument, check.Commentf("badpath %s", badpath))
 326                 err = Splice(s.fs, badpath, snapFile)
 327                 c.Check(err, ErrorIs, ErrInvalidArgument, check.Commentf("badpath %s", badpath))
 328         }
 329
 330         // Destination's parent must already exist
 331         for _, badpath := range []string{
 332                 dstPath + "/newdirname/",
 333                 dstPath + "/newdirname/foobar",
 334                 "/foo/bar",
 335         } {
 336                 err = Splice(s.fs, badpath, snap1)
 337                 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
 338                 err = Splice(s.fs, badpath, snapFile)
 339                 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
 340         }
 341
 342         snap2, err := Snapshot(s.fs, dstPath+"/ctxlog-copy")
 343         c.Check(err, check.IsNil)
 344         err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
 345         c.Check(err, check.IsNil)
 346
 347         // Snapshot entire collection, splice into same collection at
 348         // a new path, remove file from original location, verify
 349         // spliced content survives
 350         snapDst, err := Snapshot(s.fs, dstPath+"")
 351         c.Check(err, check.IsNil)
 352         err = Splice(s.fs, dstPath+"", snapDst)
 353         c.Check(err, check.IsNil)
 354         err = Splice(s.fs, dstPath+"/copy1", snapDst)
 355         c.Check(err, check.IsNil)
 356         err = Splice(s.fs, dstPath+"/copy2", snapDst)
 357         c.Check(err, check.IsNil)
 358         err = s.fs.RemoveAll(dstPath + "/arvados/fs_site_test.go")
 359         c.Check(err, check.IsNil)
 360         err = s.fs.RemoveAll(dstPath + "/arvados")
 361         c.Check(err, check.IsNil)
 362         _, err = s.fs.Open(dstPath + "/arvados/fs_site_test.go")
 363         c.Check(err, check.Equals, os.ErrNotExist)
 364         f, err = s.fs.Open(dstPath + "/copy2/arvados/fs_site_test.go")
 365         c.Check(err, check.IsNil)
 366         defer f.Close()
 367         buf, err := ioutil.ReadAll(f)
 368         c.Check(err, check.IsNil)
 369         c.Check(string(buf), check.Equals, string(thisfile))
 370 }