sdk/go/arvados/fs_site_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "fmt"
   9         "io"
  10         "io/ioutil"
  11         "net/http"
  12         "os"
  13         "strings"
  14         "syscall"
  15         "time"
  16
  17         check "gopkg.in/check.v1"
  18 )
  19
  20 const (
  21         // Importing arvadostest would be an import cycle, so these
  22         // fixtures are duplicated here [until fs moves to a separate
  23         // package].
  24         fixtureActiveToken                  = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
  25         fixtureAProjectUUID                 = "zzzzz-j7d0g-v955i6s2oi1cbso"
  26         fixtureThisFilterGroupUUID          = "zzzzz-j7d0g-thisfiltergroup"
  27         fixtureAFilterGroupTwoUUID          = "zzzzz-j7d0g-afiltergrouptwo"
  28         fixtureAFilterGroupThreeUUID        = "zzzzz-j7d0g-filtergroupthre"
  29         fixtureAFilterGroupFourUUID         = "zzzzz-j7d0g-filtergroupfour"
  30         fixtureAFilterGroupFiveUUID         = "zzzzz-j7d0g-filtergroupfive"
  31         fixtureFooAndBarFilesInDirUUID      = "zzzzz-4zz18-foonbarfilesdir"
  32         fixtureFooCollectionName            = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
  33         fixtureFooCollectionPDH             = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
  34         fixtureFooCollection                = "zzzzz-4zz18-fy296fx3hot09f7"
  35         fixtureNonexistentCollection        = "zzzzz-4zz18-totallynotexist"
  36         fixtureStorageClassesDesiredArchive = "zzzzz-4zz18-3t236wr12769qqa"
  37         fixtureBlobSigningKey               = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
  38         fixtureBlobSigningTTL               = 336 * time.Hour
  39 )
  40
  41 var _ = check.Suite(&SiteFSSuite{})
  42
  43 func init() {
  44         // Enable DebugLocksPanicMode sometimes. Don't enable it all
  45         // the time, though -- it adds many calls to time.Sleep(),
  46         // which could hide different bugs.
  47         if time.Now().Second()&1 == 0 {
  48                 DebugLocksPanicMode = true
  49         }
  50 }
  51
  52 type SiteFSSuite struct {
  53         client *Client
  54         fs     CustomFileSystem
  55         kc     keepClient
  56 }
  57
  58 func (s *SiteFSSuite) SetUpTest(c *check.C) {
  59         s.client = &Client{
  60                 APIHost:   os.Getenv("ARVADOS_API_HOST"),
  61                 AuthToken: fixtureActiveToken,
  62                 Insecure:  true,
  63         }
  64         s.kc = &keepClientStub{
  65                 blocks: map[string][]byte{
  66                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  67                 },
  68                 sigkey:    fixtureBlobSigningKey,
  69                 sigttl:    fixtureBlobSigningTTL,
  70                 authToken: fixtureActiveToken,
  71         }
  72         s.fs = s.client.SiteFileSystem(s.kc)
  73 }
  74
  75 func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
  76         _, ok := s.fs.(http.FileSystem)
  77         c.Check(ok, check.Equals, true)
  78 }
  79
  80 func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
  81         f, err := s.fs.Open("/by_id")
  82         c.Assert(err, check.IsNil)
  83         fis, err := f.Readdir(-1)
  84         c.Check(err, check.IsNil)
  85         c.Check(len(fis), check.Equals, 0)
  86 }
  87
  88 func (s *SiteFSSuite) TestUpdateStorageClasses(c *check.C) {
  89         f, err := s.fs.OpenFile("/by_id/"+fixtureStorageClassesDesiredArchive+"/newfile", os.O_CREATE|os.O_RDWR, 0777)
  90         c.Assert(err, check.IsNil)
  91         _, err = f.Write([]byte("nope"))
  92         c.Assert(err, check.IsNil)
  93         err = f.Close()
  94         c.Assert(err, check.IsNil)
  95         err = s.fs.Sync()
  96         c.Assert(err, check.ErrorMatches, `.*stub does not write storage class "archive"`)
  97 }
  98
  99 func (s *SiteFSSuite) TestByUUIDAndPDH(c *check.C) {
 100         f, err := s.fs.Open("/by_id")
 101         c.Assert(err, check.IsNil)
 102         fis, err := f.Readdir(-1)
 103         c.Check(err, check.IsNil)
 104         c.Check(len(fis), check.Equals, 0)
 105
 106         err = s.fs.Mkdir("/by_id/"+fixtureFooCollection, 0755)
 107         c.Check(err, check.Equals, os.ErrExist)
 108
 109         f, err = s.fs.Open("/by_id/" + fixtureNonexistentCollection)
 110         c.Assert(err, check.Equals, os.ErrNotExist)
 111
 112         for _, path := range []string{
 113                 fixtureFooCollection,
 114                 fixtureFooCollectionPDH,
 115                 fixtureAProjectUUID + "/" + fixtureFooCollectionName,
 116         } {
 117                 f, err = s.fs.Open("/by_id/" + path)
 118                 c.Assert(err, check.IsNil)
 119                 fis, err = f.Readdir(-1)
 120                 c.Assert(err, check.IsNil)
 121                 var names []string
 122                 for _, fi := range fis {
 123                         names = append(names, fi.Name())
 124                 }
 125                 c.Check(names, check.DeepEquals, []string{"foo"})
 126         }
 127
 128         f, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file")
 129         c.Assert(err, check.IsNil)
 130         fis, err = f.Readdir(-1)
 131         c.Assert(err, check.IsNil)
 132         var names []string
 133         for _, fi := range fis {
 134                 names = append(names, fi.Name())
 135         }
 136         c.Check(names, check.DeepEquals, []string{"baz"})
 137
 138         _, err = s.fs.OpenFile("/by_id/"+fixtureNonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
 139         c.Check(err, ErrorIs, ErrInvalidOperation)
 140         err = s.fs.Rename("/by_id/"+fixtureFooCollection, "/by_id/beep")
 141         c.Check(err, ErrorIs, ErrInvalidOperation)
 142         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/beep")
 143         c.Check(err, ErrorIs, ErrInvalidOperation)
 144         _, err = s.fs.Stat("/by_id/beep")
 145         c.Check(err, check.Equals, os.ErrNotExist)
 146         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/"+fixtureFooCollection+"/bar")
 147         c.Check(err, check.IsNil)
 148
 149         err = s.fs.Rename("/by_id", "/beep")
 150         c.Check(err, ErrorIs, ErrInvalidOperation)
 151 }
 152
 153 // Copy subtree from OS src to dst path inside fs. If src is a
 154 // directory, dst must exist and be a directory.
 155 func copyFromOS(fs FileSystem, dst, src string) error {
 156         inf, err := os.Open(src)
 157         if err != nil {
 158                 return err
 159         }
 160         defer inf.Close()
 161         dirents, err := inf.Readdir(-1)
 162         if e, ok := err.(*os.PathError); ok {
 163                 if e, ok := e.Err.(syscall.Errno); ok {
 164                         if e == syscall.ENOTDIR {
 165                                 err = syscall.ENOTDIR
 166                         }
 167                 }
 168         }
 169         if err == syscall.ENOTDIR {
 170                 outf, err := fs.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_TRUNC|os.O_WRONLY, 0700)
 171                 if err != nil {
 172                         return fmt.Errorf("open %s: %s", dst, err)
 173                 }
 174                 defer outf.Close()
 175                 _, err = io.Copy(outf, inf)
 176                 if err != nil {
 177                         return fmt.Errorf("%s: copying data from %s: %s", dst, src, err)
 178                 }
 179                 err = outf.Close()
 180                 if err != nil {
 181                         return err
 182                 }
 183         } else if err != nil {
 184                 return fmt.Errorf("%s: readdir: %T %s", src, err, err)
 185         } else {
 186                 {
 187                         d, err := fs.Open(dst)
 188                         if err != nil {
 189                                 return fmt.Errorf("opendir(%s): %s", dst, err)
 190                         }
 191                         d.Close()
 192                 }
 193                 for _, ent := range dirents {
 194                         if ent.Name() == "." || ent.Name() == ".." {
 195                                 continue
 196                         }
 197                         dstname := dst + "/" + ent.Name()
 198                         if ent.IsDir() {
 199                                 err = fs.Mkdir(dstname, 0700)
 200                                 if err != nil {
 201                                         return fmt.Errorf("mkdir %s: %s", dstname, err)
 202                                 }
 203                         }
 204                         err = copyFromOS(fs, dstname, src+"/"+ent.Name())
 205                         if err != nil {
 206                                 return err
 207                         }
 208                 }
 209         }
 210         return nil
 211 }
 212
 213 func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
 214         s.fs.MountProject("home", "")
 215         thisfile, err := ioutil.ReadFile("fs_site_test.go")
 216         c.Assert(err, check.IsNil)
 217
 218         var src1 Collection
 219         err = s.client.RequestAndDecode(&src1, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 220                 "collection": map[string]string{
 221                         "name":       "TestSnapshotSplice src1",
 222                         "owner_uuid": fixtureAProjectUUID,
 223                 },
 224         })
 225         c.Assert(err, check.IsNil)
 226         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src1.UUID, nil, nil)
 227         err = s.fs.Sync()
 228         c.Assert(err, check.IsNil)
 229         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src1", "..") // arvados.git/sdk/go
 230         c.Assert(err, check.IsNil)
 231
 232         var src2 Collection
 233         err = s.client.RequestAndDecode(&src2, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 234                 "collection": map[string]string{
 235                         "name":       "TestSnapshotSplice src2",
 236                         "owner_uuid": fixtureAProjectUUID,
 237                 },
 238         })
 239         c.Assert(err, check.IsNil)
 240         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src2.UUID, nil, nil)
 241         err = s.fs.Sync()
 242         c.Assert(err, check.IsNil)
 243         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src2", "..") // arvados.git/sdk/go
 244         c.Assert(err, check.IsNil)
 245
 246         var dst Collection
 247         err = s.client.RequestAndDecode(&dst, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 248                 "collection": map[string]string{
 249                         "name":       "TestSnapshotSplice dst",
 250                         "owner_uuid": fixtureAProjectUUID,
 251                 },
 252         })
 253         c.Assert(err, check.IsNil)
 254         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+dst.UUID, nil, nil)
 255         err = s.fs.Sync()
 256         c.Assert(err, check.IsNil)
 257
 258         dstPath := "/home/A Project/TestSnapshotSplice dst"
 259         err = copyFromOS(s.fs, dstPath, "..") // arvados.git/sdk/go
 260         c.Assert(err, check.IsNil)
 261
 262         // Snapshot directory
 263         snap1, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/ctxlog")
 264         c.Check(err, check.IsNil)
 265         // Attach same snapshot twice, at paths that didn't exist before
 266         err = Splice(s.fs, dstPath+"/ctxlog-copy", snap1)
 267         c.Check(err, check.IsNil)
 268         err = Splice(s.fs, dstPath+"/ctxlog-copy2", snap1)
 269         c.Check(err, check.IsNil)
 270         // Splicing a snapshot twice results in two independent copies
 271         err = s.fs.Rename(dstPath+"/ctxlog-copy2/log.go", dstPath+"/ctxlog-copy/log2.go")
 272         c.Check(err, check.IsNil)
 273         _, err = s.fs.Open(dstPath + "/ctxlog-copy2/log.go")
 274         c.Check(err, check.Equals, os.ErrNotExist)
 275         f, err := s.fs.Open(dstPath + "/ctxlog-copy/log.go")
 276         if c.Check(err, check.IsNil) {
 277                 buf, err := ioutil.ReadAll(f)
 278                 c.Check(err, check.IsNil)
 279                 c.Check(string(buf), check.Not(check.Equals), "")
 280                 f.Close()
 281         }
 282
 283         // Snapshot regular file
 284         snapFile, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/arvados/fs_site_test.go")
 285         c.Check(err, check.IsNil)
 286         // Replace dir with file
 287         err = Splice(s.fs, dstPath+"/ctxlog-copy2", snapFile)
 288         c.Check(err, check.IsNil)
 289         if f, err := s.fs.Open(dstPath + "/ctxlog-copy2"); c.Check(err, check.IsNil) {
 290                 buf, err := ioutil.ReadAll(f)
 291                 c.Check(err, check.IsNil)
 292                 c.Check(string(buf), check.Equals, string(thisfile))
 293         }
 294
 295         // Cannot splice a file onto a collection root; cannot splice
 296         // anything to a target outside a collection.
 297         for _, badpath := range []string{
 298                 dstPath + "/",
 299                 dstPath,
 300                 "/home/A Project/newnodename/",
 301                 "/home/A Project/newnodename",
 302                 "/home/A Project/",
 303                 "/home/A Project",
 304                 "/home/newnodename/",
 305                 "/home/newnodename",
 306                 "/home/",
 307                 "/home",
 308                 "/newnodename/",
 309                 "/newnodename",
 310                 "/",
 311         } {
 312                 err = Splice(s.fs, badpath, snapFile)
 313                 c.Check(err, check.NotNil)
 314                 if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
 315                         c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
 316                 } else {
 317                         c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
 318                 }
 319                 if strings.TrimSuffix(badpath, "/") == dstPath {
 320                         c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %q", badpath))
 321                         continue
 322                 }
 323
 324                 err = Splice(s.fs, badpath, snap1)
 325                 if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
 326                         c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
 327                 } else {
 328                         c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
 329                 }
 330         }
 331
 332         // Destination's parent must already exist
 333         for _, badpath := range []string{
 334                 dstPath + "/newdirname/",
 335                 dstPath + "/newdirname/foobar",
 336                 "/foo/bar",
 337         } {
 338                 err = Splice(s.fs, badpath, snap1)
 339                 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
 340                 err = Splice(s.fs, badpath, snapFile)
 341                 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
 342         }
 343
 344         snap2, err := Snapshot(s.fs, dstPath+"/ctxlog-copy")
 345         if c.Check(err, check.IsNil) {
 346                 err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
 347                 c.Check(err, check.IsNil)
 348         }
 349
 350         // Snapshot entire collection, splice into same collection at
 351         // a new path, remove file from original location, verify
 352         // spliced content survives
 353         snapDst, err := Snapshot(s.fs, dstPath+"")
 354         c.Check(err, check.IsNil)
 355         err = Splice(s.fs, dstPath+"", snapDst)
 356         c.Check(err, check.IsNil)
 357         err = Splice(s.fs, dstPath+"/copy1", snapDst)
 358         c.Check(err, check.IsNil)
 359         err = Splice(s.fs, dstPath+"/copy2", snapDst)
 360         c.Check(err, check.IsNil)
 361         err = s.fs.RemoveAll(dstPath + "/arvados/fs_site_test.go")
 362         c.Check(err, check.IsNil)
 363         err = s.fs.RemoveAll(dstPath + "/arvados")
 364         c.Check(err, check.IsNil)
 365         _, err = s.fs.Open(dstPath + "/arvados/fs_site_test.go")
 366         c.Check(err, check.Equals, os.ErrNotExist)
 367         f, err = s.fs.Open(dstPath + "/copy2/arvados/fs_site_test.go")
 368         if c.Check(err, check.IsNil) {
 369                 defer f.Close()
 370                 buf, err := ioutil.ReadAll(f)
 371                 c.Check(err, check.IsNil)
 372                 c.Check(string(buf), check.Equals, string(thisfile))
 373         }
 374 }