sdk/go/arvados/fs_site_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "fmt"
   9         "io"
  10         "io/ioutil"
  11         "net/http"
  12         "os"
  13         "strings"
  14         "sync"
  15         "syscall"
  16         "time"
  17
  18         check "gopkg.in/check.v1"
  19 )
  20
  21 const (
  22         // Importing arvadostest would be an import cycle, so these
  23         // fixtures are duplicated here [until fs moves to a separate
  24         // package].
  25         fixtureActiveToken                  = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
  26         fixtureAProjectUUID                 = "zzzzz-j7d0g-v955i6s2oi1cbso"
  27         fixtureThisFilterGroupUUID          = "zzzzz-j7d0g-thisfiltergroup"
  28         fixtureAFilterGroupTwoUUID          = "zzzzz-j7d0g-afiltergrouptwo"
  29         fixtureAFilterGroupThreeUUID        = "zzzzz-j7d0g-filtergroupthre"
  30         fixtureAFilterGroupFourUUID         = "zzzzz-j7d0g-filtergroupfour"
  31         fixtureAFilterGroupFiveUUID         = "zzzzz-j7d0g-filtergroupfive"
  32         fixtureFooAndBarFilesInDirUUID      = "zzzzz-4zz18-foonbarfilesdir"
  33         fixtureFooCollectionName            = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
  34         fixtureFooCollectionPDH             = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
  35         fixtureFooCollection                = "zzzzz-4zz18-fy296fx3hot09f7"
  36         fixtureNonexistentCollection        = "zzzzz-4zz18-totallynotexist"
  37         fixtureStorageClassesDesiredArchive = "zzzzz-4zz18-3t236wr12769qqa"
  38         fixtureBlobSigningKey               = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
  39         fixtureBlobSigningTTL               = 336 * time.Hour
  40 )
  41
  42 var _ = check.Suite(&SiteFSSuite{})
  43
  44 func init() {
  45         // Enable DebugLocksPanicMode sometimes. Don't enable it all
  46         // the time, though -- it adds many calls to time.Sleep(),
  47         // which could hide different bugs.
  48         if time.Now().Second()&1 == 0 {
  49                 DebugLocksPanicMode = true
  50         }
  51 }
  52
  53 type SiteFSSuite struct {
  54         client *Client
  55         fs     CustomFileSystem
  56         kc     keepClient
  57 }
  58
  59 func (s *SiteFSSuite) SetUpTest(c *check.C) {
  60         s.client = &Client{
  61                 APIHost:   os.Getenv("ARVADOS_API_HOST"),
  62                 AuthToken: fixtureActiveToken,
  63                 Insecure:  true,
  64         }
  65         s.kc = &keepClientStub{
  66                 blocks: map[string][]byte{
  67                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  68                 },
  69                 sigkey:    fixtureBlobSigningKey,
  70                 sigttl:    fixtureBlobSigningTTL,
  71                 authToken: fixtureActiveToken,
  72         }
  73         s.fs = s.client.SiteFileSystem(s.kc)
  74 }
  75
  76 func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
  77         _, ok := s.fs.(http.FileSystem)
  78         c.Check(ok, check.Equals, true)
  79 }
  80
  81 func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
  82         f, err := s.fs.Open("/by_id")
  83         c.Assert(err, check.IsNil)
  84         fis, err := f.Readdir(-1)
  85         c.Check(err, check.IsNil)
  86         c.Check(len(fis), check.Equals, 0)
  87 }
  88
  89 func (s *SiteFSSuite) TestUpdateStorageClasses(c *check.C) {
  90         f, err := s.fs.OpenFile("/by_id/"+fixtureStorageClassesDesiredArchive+"/newfile", os.O_CREATE|os.O_RDWR, 0777)
  91         c.Assert(err, check.IsNil)
  92         _, err = f.Write([]byte("nope"))
  93         c.Assert(err, check.IsNil)
  94         err = f.Close()
  95         c.Assert(err, check.IsNil)
  96         err = s.fs.Sync()
  97         c.Assert(err, check.ErrorMatches, `.*stub does not write storage class "archive"`)
  98 }
  99
 100 func (s *SiteFSSuite) TestByUUIDAndPDH(c *check.C) {
 101         f, err := s.fs.Open("/by_id")
 102         c.Assert(err, check.IsNil)
 103         fis, err := f.Readdir(-1)
 104         c.Check(err, check.IsNil)
 105         c.Check(len(fis), check.Equals, 0)
 106
 107         err = s.fs.Mkdir("/by_id/"+fixtureFooCollection, 0755)
 108         c.Check(err, check.Equals, os.ErrExist)
 109
 110         f, err = s.fs.Open("/by_id/" + fixtureNonexistentCollection)
 111         c.Assert(err, check.Equals, os.ErrNotExist)
 112
 113         for _, path := range []string{
 114                 fixtureFooCollection,
 115                 fixtureFooCollectionPDH,
 116                 fixtureAProjectUUID + "/" + fixtureFooCollectionName,
 117         } {
 118                 f, err = s.fs.Open("/by_id/" + path)
 119                 c.Assert(err, check.IsNil)
 120                 fis, err = f.Readdir(-1)
 121                 c.Assert(err, check.IsNil)
 122                 var names []string
 123                 for _, fi := range fis {
 124                         names = append(names, fi.Name())
 125                 }
 126                 c.Check(names, check.DeepEquals, []string{"foo"})
 127         }
 128
 129         f, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file")
 130         c.Assert(err, check.IsNil)
 131         fis, err = f.Readdir(-1)
 132         c.Assert(err, check.IsNil)
 133         var names []string
 134         for _, fi := range fis {
 135                 names = append(names, fi.Name())
 136         }
 137         c.Check(names, check.DeepEquals, []string{"baz"})
 138
 139         _, err = s.fs.OpenFile("/by_id/"+fixtureNonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
 140         c.Check(err, ErrorIs, ErrInvalidOperation)
 141         err = s.fs.Rename("/by_id/"+fixtureFooCollection, "/by_id/beep")
 142         c.Check(err, ErrorIs, ErrInvalidOperation)
 143         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/beep")
 144         c.Check(err, ErrorIs, ErrInvalidOperation)
 145         _, err = s.fs.Stat("/by_id/beep")
 146         c.Check(err, check.Equals, os.ErrNotExist)
 147         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/"+fixtureFooCollection+"/bar")
 148         c.Check(err, check.IsNil)
 149
 150         err = s.fs.Rename("/by_id", "/beep")
 151         c.Check(err, ErrorIs, ErrInvalidOperation)
 152 }
 153
 154 // Copy subtree from OS src to dst path inside fs. If src is a
 155 // directory, dst must exist and be a directory.
 156 func copyFromOS(fs FileSystem, dst, src string) error {
 157         inf, err := os.Open(src)
 158         if err != nil {
 159                 return err
 160         }
 161         defer inf.Close()
 162         dirents, err := inf.Readdir(-1)
 163         if e, ok := err.(*os.PathError); ok {
 164                 if e, ok := e.Err.(syscall.Errno); ok {
 165                         if e == syscall.ENOTDIR {
 166                                 err = syscall.ENOTDIR
 167                         }
 168                 }
 169         }
 170         if err == syscall.ENOTDIR {
 171                 outf, err := fs.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_TRUNC|os.O_WRONLY, 0700)
 172                 if err != nil {
 173                         return fmt.Errorf("open %s: %s", dst, err)
 174                 }
 175                 defer outf.Close()
 176                 _, err = io.Copy(outf, inf)
 177                 if err != nil {
 178                         return fmt.Errorf("%s: copying data from %s: %s", dst, src, err)
 179                 }
 180                 err = outf.Close()
 181                 if err != nil {
 182                         return err
 183                 }
 184         } else if err != nil {
 185                 return fmt.Errorf("%s: readdir: %T %s", src, err, err)
 186         } else {
 187                 {
 188                         d, err := fs.Open(dst)
 189                         if err != nil {
 190                                 return fmt.Errorf("opendir(%s): %s", dst, err)
 191                         }
 192                         d.Close()
 193                 }
 194                 for _, ent := range dirents {
 195                         if ent.Name() == "." || ent.Name() == ".." {
 196                                 continue
 197                         }
 198                         dstname := dst + "/" + ent.Name()
 199                         if ent.IsDir() {
 200                                 err = fs.Mkdir(dstname, 0700)
 201                                 if err != nil {
 202                                         return fmt.Errorf("mkdir %s: %s", dstname, err)
 203                                 }
 204                         }
 205                         err = copyFromOS(fs, dstname, src+"/"+ent.Name())
 206                         if err != nil {
 207                                 return err
 208                         }
 209                 }
 210         }
 211         return nil
 212 }
 213
 214 func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
 215         s.fs.MountProject("home", "")
 216         thisfile, err := ioutil.ReadFile("fs_site_test.go")
 217         c.Assert(err, check.IsNil)
 218
 219         var src1 Collection
 220         err = s.client.RequestAndDecode(&src1, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 221                 "collection": map[string]string{
 222                         "name":       "TestSnapshotSplice src1",
 223                         "owner_uuid": fixtureAProjectUUID,
 224                 },
 225         })
 226         c.Assert(err, check.IsNil)
 227         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src1.UUID, nil, nil)
 228         err = s.fs.Sync()
 229         c.Assert(err, check.IsNil)
 230         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src1", "..") // arvados.git/sdk/go
 231         c.Assert(err, check.IsNil)
 232
 233         var src2 Collection
 234         err = s.client.RequestAndDecode(&src2, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 235                 "collection": map[string]string{
 236                         "name":       "TestSnapshotSplice src2",
 237                         "owner_uuid": fixtureAProjectUUID,
 238                 },
 239         })
 240         c.Assert(err, check.IsNil)
 241         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src2.UUID, nil, nil)
 242         err = s.fs.Sync()
 243         c.Assert(err, check.IsNil)
 244         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src2", "..") // arvados.git/sdk/go
 245         c.Assert(err, check.IsNil)
 246
 247         var dst Collection
 248         err = s.client.RequestAndDecode(&dst, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 249                 "collection": map[string]string{
 250                         "name":       "TestSnapshotSplice dst",
 251                         "owner_uuid": fixtureAProjectUUID,
 252                 },
 253         })
 254         c.Assert(err, check.IsNil)
 255         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+dst.UUID, nil, nil)
 256         err = s.fs.Sync()
 257         c.Assert(err, check.IsNil)
 258
 259         dstPath := "/home/A Project/TestSnapshotSplice dst"
 260         err = copyFromOS(s.fs, dstPath, "..") // arvados.git/sdk/go
 261         c.Assert(err, check.IsNil)
 262
 263         // Snapshot directory
 264         snap1, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/ctxlog")
 265         c.Check(err, check.IsNil)
 266         // Attach same snapshot twice, at paths that didn't exist before
 267         err = Splice(s.fs, dstPath+"/ctxlog-copy", snap1)
 268         c.Check(err, check.IsNil)
 269         err = Splice(s.fs, dstPath+"/ctxlog-copy2", snap1)
 270         c.Check(err, check.IsNil)
 271         // Splicing a snapshot twice results in two independent copies
 272         err = s.fs.Rename(dstPath+"/ctxlog-copy2/log.go", dstPath+"/ctxlog-copy/log2.go")
 273         c.Check(err, check.IsNil)
 274         _, err = s.fs.Open(dstPath + "/ctxlog-copy2/log.go")
 275         c.Check(err, check.Equals, os.ErrNotExist)
 276         f, err := s.fs.Open(dstPath + "/ctxlog-copy/log.go")
 277         if c.Check(err, check.IsNil) {
 278                 buf, err := ioutil.ReadAll(f)
 279                 c.Check(err, check.IsNil)
 280                 c.Check(string(buf), check.Not(check.Equals), "")
 281                 f.Close()
 282         }
 283
 284         // Snapshot regular file
 285         snapFile, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/arvados/fs_site_test.go")
 286         c.Check(err, check.IsNil)
 287         // Replace dir with file
 288         err = Splice(s.fs, dstPath+"/ctxlog-copy2", snapFile)
 289         c.Check(err, check.IsNil)
 290         if f, err := s.fs.Open(dstPath + "/ctxlog-copy2"); c.Check(err, check.IsNil) {
 291                 buf, err := ioutil.ReadAll(f)
 292                 c.Check(err, check.IsNil)
 293                 c.Check(string(buf), check.Equals, string(thisfile))
 294         }
 295
 296         // Cannot splice a file onto a collection root; cannot splice
 297         // anything to a target outside a collection.
 298         for _, badpath := range []string{
 299                 dstPath + "/",
 300                 dstPath,
 301                 "/home/A Project/newnodename/",
 302                 "/home/A Project/newnodename",
 303                 "/home/A Project/",
 304                 "/home/A Project",
 305                 "/home/newnodename/",
 306                 "/home/newnodename",
 307                 "/home/",
 308                 "/home",
 309                 "/newnodename/",
 310                 "/newnodename",
 311                 "/",
 312         } {
 313                 err = Splice(s.fs, badpath, snapFile)
 314                 c.Check(err, check.NotNil)
 315                 if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
 316                         c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
 317                 } else {
 318                         c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
 319                 }
 320                 if strings.TrimSuffix(badpath, "/") == dstPath {
 321                         c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %q", badpath))
 322                         continue
 323                 }
 324
 325                 err = Splice(s.fs, badpath, snap1)
 326                 if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
 327                         c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
 328                 } else {
 329                         c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
 330                 }
 331         }
 332
 333         // Destination's parent must already exist
 334         for _, badpath := range []string{
 335                 dstPath + "/newdirname/",
 336                 dstPath + "/newdirname/foobar",
 337                 "/foo/bar",
 338         } {
 339                 err = Splice(s.fs, badpath, snap1)
 340                 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
 341                 err = Splice(s.fs, badpath, snapFile)
 342                 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
 343         }
 344
 345         snap2, err := Snapshot(s.fs, dstPath+"/ctxlog-copy")
 346         if c.Check(err, check.IsNil) {
 347                 err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
 348                 c.Check(err, check.IsNil)
 349         }
 350
 351         // Snapshot entire collection, splice into same collection at
 352         // a new path, remove file from original location, verify
 353         // spliced content survives
 354         snapDst, err := Snapshot(s.fs, dstPath+"")
 355         c.Check(err, check.IsNil)
 356         err = Splice(s.fs, dstPath+"", snapDst)
 357         c.Check(err, check.IsNil)
 358         err = Splice(s.fs, dstPath+"/copy1", snapDst)
 359         c.Check(err, check.IsNil)
 360         err = Splice(s.fs, dstPath+"/copy2", snapDst)
 361         c.Check(err, check.IsNil)
 362         err = s.fs.RemoveAll(dstPath + "/arvados/fs_site_test.go")
 363         c.Check(err, check.IsNil)
 364         err = s.fs.RemoveAll(dstPath + "/arvados")
 365         c.Check(err, check.IsNil)
 366         _, err = s.fs.Open(dstPath + "/arvados/fs_site_test.go")
 367         c.Check(err, check.Equals, os.ErrNotExist)
 368         f, err = s.fs.Open(dstPath + "/copy2/arvados/fs_site_test.go")
 369         if c.Check(err, check.IsNil) {
 370                 defer f.Close()
 371                 buf, err := ioutil.ReadAll(f)
 372                 c.Check(err, check.IsNil)
 373                 c.Check(string(buf), check.Equals, string(thisfile))
 374         }
 375 }
 376
 377 func (s *SiteFSSuite) TestLocks(c *check.C) {
 378         DebugLocksPanicMode = false
 379         done := make(chan struct{})
 380         defer close(done)
 381         ticker := time.NewTicker(2 * time.Second)
 382         go func() {
 383                 for {
 384                         timeout := time.AfterFunc(5*time.Second, func() {
 385                                 // c.FailNow() doesn't break deadlock, but this sure does
 386                                 panic("timed out -- deadlock?")
 387                         })
 388                         select {
 389                         case <-done:
 390                                 timeout.Stop()
 391                                 return
 392                         case <-ticker.C:
 393                                 c.Logf("MemorySize == %d", s.fs.MemorySize())
 394                         }
 395                         timeout.Stop()
 396                 }
 397         }()
 398         ncolls := 5
 399         ndirs := 3
 400         nfiles := 5
 401         projects := make([]Group, 5)
 402         for pnum := range projects {
 403                 c.Logf("make project %d", pnum)
 404                 err := s.client.RequestAndDecode(&projects[pnum], "POST", "arvados/v1/groups", nil, map[string]interface{}{
 405                         "group": map[string]string{
 406                                 "name":        fmt.Sprintf("TestLocks project %d", pnum),
 407                                 "owner_uuid":  fixtureAProjectUUID,
 408                                 "group_class": "project",
 409                         },
 410                         "ensure_unique_name": true,
 411                 })
 412                 c.Assert(err, check.IsNil)
 413                 for cnum := 0; cnum < ncolls; cnum++ {
 414                         c.Logf("make project %d collection %d", pnum, cnum)
 415                         var coll Collection
 416                         err = s.client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 417                                 "collection": map[string]string{
 418                                         "name":       fmt.Sprintf("TestLocks collection %d", cnum),
 419                                         "owner_uuid": projects[pnum].UUID,
 420                                 },
 421                         })
 422                         c.Assert(err, check.IsNil)
 423                         for d1num := 0; d1num < ndirs; d1num++ {
 424                                 s.fs.Mkdir(fmt.Sprintf("/by_id/%s/dir1-%d", coll.UUID, d1num), 0777)
 425                                 for d2num := 0; d2num < ndirs; d2num++ {
 426                                         s.fs.Mkdir(fmt.Sprintf("/by_id/%s/dir1-%d/dir2-%d", coll.UUID, d1num, d2num), 0777)
 427                                         for fnum := 0; fnum < nfiles; fnum++ {
 428                                                 f, err := s.fs.OpenFile(fmt.Sprintf("/by_id/%s/dir1-%d/dir2-%d/file-%d", coll.UUID, d1num, d2num, fnum), os.O_CREATE|os.O_RDWR, 0755)
 429                                                 c.Assert(err, check.IsNil)
 430                                                 f.Close()
 431                                                 f, err = s.fs.OpenFile(fmt.Sprintf("/by_id/%s/dir1-%d/file-%d", coll.UUID, d1num, fnum), os.O_CREATE|os.O_RDWR, 0755)
 432                                                 c.Assert(err, check.IsNil)
 433                                                 f.Close()
 434                                         }
 435                                 }
 436                         }
 437                 }
 438         }
 439         c.Log("sync")
 440         s.fs.Sync()
 441         var wg sync.WaitGroup
 442         for n := 0; n < 100; n++ {
 443                 wg.Add(1)
 444                 go func() {
 445                         defer wg.Done()
 446                         for pnum, project := range projects {
 447                                 c.Logf("read project %d", pnum)
 448                                 if pnum%2 == 0 {
 449                                         f, err := s.fs.Open(fmt.Sprintf("/by_id/%s", project.UUID))
 450                                         c.Assert(err, check.IsNil)
 451                                         f.Readdir(-1)
 452                                         f.Close()
 453                                 }
 454                                 for cnum := 0; cnum < ncolls; cnum++ {
 455                                         c.Logf("read project %d collection %d", pnum, cnum)
 456                                         if pnum%2 == 0 {
 457                                                 f, err := s.fs.Open(fmt.Sprintf("/by_id/%s/TestLocks collection %d", project.UUID, cnum))
 458                                                 c.Assert(err, check.IsNil)
 459                                                 _, err = f.Readdir(-1)
 460                                                 c.Assert(err, check.IsNil)
 461                                                 f.Close()
 462                                         }
 463                                         if pnum%3 == 0 {
 464                                                 for d1num := 0; d1num < ndirs; d1num++ {
 465                                                         f, err := s.fs.Open(fmt.Sprintf("/by_id/%s/TestLocks collection %d/dir1-%d", project.UUID, cnum, d1num))
 466                                                         c.Assert(err, check.IsNil)
 467                                                         fis, err := f.Readdir(-1)
 468                                                         c.Assert(err, check.IsNil)
 469                                                         c.Assert(fis, check.HasLen, ndirs+nfiles)
 470                                                         f.Close()
 471                                                 }
 472                                         }
 473                                         for d1num := 0; d1num < ndirs; d1num++ {
 474                                                 for d2num := 0; d2num < ndirs; d2num++ {
 475                                                         f, err := s.fs.Open(fmt.Sprintf("/by_id/%s/TestLocks collection %d/dir1-%d/dir2-%d", project.UUID, cnum, d1num, d2num))
 476                                                         c.Assert(err, check.IsNil)
 477                                                         fis, err := f.Readdir(-1)
 478                                                         c.Assert(err, check.IsNil)
 479                                                         c.Assert(fis, check.HasLen, nfiles)
 480                                                         f.Close()
 481                                                 }
 482                                         }
 483                                 }
 484                         }
 485                 }()
 486         }
 487         wg.Wait()
 488         c.Logf("MemorySize == %d", s.fs.MemorySize())
 489 }