sdk/go/arvados/fs_site_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "fmt"
   9         "io"
  10         "io/ioutil"
  11         "net/http"
  12         "os"
  13         "strings"
  14         "sync"
  15         "syscall"
  16         "time"
  17
  18         check "gopkg.in/check.v1"
  19 )
  20
  21 const (
  22         // Importing arvadostest would be an import cycle, so these
  23         // fixtures are duplicated here [until fs moves to a separate
  24         // package].
  25         fixtureActiveUserUUID               = "zzzzz-tpzed-xurymjxw79nv3jz"
  26         fixtureActiveToken                  = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
  27         fixtureAProjectUUID                 = "zzzzz-j7d0g-v955i6s2oi1cbso"
  28         fixtureThisFilterGroupUUID          = "zzzzz-j7d0g-thisfiltergroup"
  29         fixtureAFilterGroupTwoUUID          = "zzzzz-j7d0g-afiltergrouptwo"
  30         fixtureAFilterGroupThreeUUID        = "zzzzz-j7d0g-filtergroupthre"
  31         fixtureAFilterGroupFourUUID         = "zzzzz-j7d0g-filtergroupfour"
  32         fixtureAFilterGroupFiveUUID         = "zzzzz-j7d0g-filtergroupfive"
  33         fixtureFooAndBarFilesInDirUUID      = "zzzzz-4zz18-foonbarfilesdir"
  34         fixtureFooCollectionName            = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
  35         fixtureFooCollectionPDH             = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
  36         fixtureFooCollection                = "zzzzz-4zz18-fy296fx3hot09f7"
  37         fixtureNonexistentCollection        = "zzzzz-4zz18-totallynotexist"
  38         fixtureStorageClassesDesiredArchive = "zzzzz-4zz18-3t236wr12769qqa"
  39         fixtureBlobSigningKey               = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
  40         fixtureBlobSigningTTL               = 336 * time.Hour
  41 )
  42
  43 var _ = check.Suite(&SiteFSSuite{})
  44
  45 func init() {
  46         // Enable DebugLocksPanicMode sometimes. Don't enable it all
  47         // the time, though -- it adds many calls to time.Sleep(),
  48         // which could hide different bugs.
  49         if time.Now().Second()&1 == 0 {
  50                 DebugLocksPanicMode = true
  51         }
  52 }
  53
  54 type SiteFSSuite struct {
  55         client *Client
  56         fs     CustomFileSystem
  57         kc     keepClient
  58 }
  59
  60 func (s *SiteFSSuite) SetUpTest(c *check.C) {
  61         s.client = &Client{
  62                 APIHost:   os.Getenv("ARVADOS_API_HOST"),
  63                 AuthToken: fixtureActiveToken,
  64                 Insecure:  true,
  65         }
  66         s.kc = &keepClientStub{
  67                 blocks: map[string][]byte{
  68                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  69                 },
  70                 sigkey:    fixtureBlobSigningKey,
  71                 sigttl:    fixtureBlobSigningTTL,
  72                 authToken: fixtureActiveToken,
  73         }
  74         s.fs = s.client.SiteFileSystem(s.kc)
  75 }
  76
  77 func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
  78         _, ok := s.fs.(http.FileSystem)
  79         c.Check(ok, check.Equals, true)
  80 }
  81
  82 func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
  83         f, err := s.fs.Open("/by_id")
  84         c.Assert(err, check.IsNil)
  85         fis, err := f.Readdir(-1)
  86         c.Check(err, check.IsNil)
  87         c.Check(len(fis), check.Equals, 0)
  88 }
  89
  90 func (s *SiteFSSuite) TestUpdateStorageClasses(c *check.C) {
  91         f, err := s.fs.OpenFile("/by_id/"+fixtureStorageClassesDesiredArchive+"/newfile", os.O_CREATE|os.O_RDWR, 0777)
  92         c.Assert(err, check.IsNil)
  93         _, err = f.Write([]byte("nope"))
  94         c.Assert(err, check.IsNil)
  95         err = f.Close()
  96         c.Assert(err, check.IsNil)
  97         err = s.fs.Sync()
  98         c.Assert(err, check.ErrorMatches, `.*stub does not write storage class "archive"`)
  99 }
 100
 101 func (s *SiteFSSuite) TestSameCollectionDifferentPaths(c *check.C) {
 102         s.fs.MountProject("home", "")
 103         var coll Collection
 104         err := s.client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 105                 "collection": map[string]interface{}{
 106                         "owner_uuid": fixtureAProjectUUID,
 107                         "name":       fmt.Sprintf("test collection %d", time.Now().UnixNano()),
 108                 },
 109         })
 110         c.Assert(err, check.IsNil)
 111
 112         viaProjID := "by_id/" + fixtureAProjectUUID + "/" + coll.Name
 113         viaProjName := "home/A Project/" + coll.Name
 114         viaCollID := "by_id/" + coll.UUID
 115         for n, dirs := range [][]string{
 116                 {viaCollID, viaProjID, viaProjName},
 117                 {viaCollID, viaProjName, viaProjID},
 118                 {viaProjID, viaProjName, viaCollID},
 119                 {viaProjID, viaCollID, viaProjName},
 120                 {viaProjName, viaCollID, viaProjID},
 121                 {viaProjName, viaProjID, viaCollID},
 122         } {
 123                 filename := fmt.Sprintf("file %d", n)
 124                 f := make([]File, 3)
 125                 for i, dir := range dirs {
 126                         path := dir + "/" + filename
 127                         mode := os.O_RDWR
 128                         if i == 0 {
 129                                 mode |= os.O_CREATE
 130                                 c.Logf("create %s", path)
 131                         } else {
 132                                 c.Logf("open %s", path)
 133                         }
 134                         f[i], err = s.fs.OpenFile(path, mode, 0777)
 135                         c.Assert(err, check.IsNil, check.Commentf("n=%d i=%d path=%s", n, i, path))
 136                         defer f[i].Close()
 137                 }
 138                 _, err = io.WriteString(f[0], filename)
 139                 c.Assert(err, check.IsNil)
 140                 _, err = f[1].Seek(0, io.SeekEnd)
 141                 c.Assert(err, check.IsNil)
 142                 _, err = io.WriteString(f[1], filename)
 143                 c.Assert(err, check.IsNil)
 144                 buf, err := io.ReadAll(f[2])
 145                 c.Assert(err, check.IsNil)
 146                 c.Check(string(buf), check.Equals, filename+filename)
 147         }
 148 }
 149
 150 func (s *SiteFSSuite) TestByUUIDAndPDH(c *check.C) {
 151         f, err := s.fs.Open("/by_id")
 152         c.Assert(err, check.IsNil)
 153         fis, err := f.Readdir(-1)
 154         c.Check(err, check.IsNil)
 155         c.Check(len(fis), check.Equals, 0)
 156
 157         err = s.fs.Mkdir("/by_id/"+fixtureFooCollection, 0755)
 158         c.Check(err, check.Equals, os.ErrExist)
 159
 160         f, err = s.fs.Open("/by_id/" + fixtureNonexistentCollection)
 161         c.Assert(err, check.Equals, os.ErrNotExist)
 162
 163         for _, path := range []string{
 164                 fixtureFooCollection,
 165                 fixtureFooCollectionPDH,
 166                 fixtureAProjectUUID + "/" + fixtureFooCollectionName,
 167         } {
 168                 f, err = s.fs.Open("/by_id/" + path)
 169                 c.Assert(err, check.IsNil)
 170                 fis, err = f.Readdir(-1)
 171                 c.Assert(err, check.IsNil)
 172                 var names []string
 173                 for _, fi := range fis {
 174                         names = append(names, fi.Name())
 175                 }
 176                 c.Check(names, check.DeepEquals, []string{"foo"})
 177         }
 178
 179         f, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file")
 180         c.Assert(err, check.IsNil)
 181         fis, err = f.Readdir(-1)
 182         c.Assert(err, check.IsNil)
 183         var names []string
 184         for _, fi := range fis {
 185                 names = append(names, fi.Name())
 186         }
 187         c.Check(names, check.DeepEquals, []string{"baz"})
 188         f, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file/baz")
 189         c.Assert(err, check.IsNil)
 190         err = f.Close()
 191         c.Assert(err, check.IsNil)
 192         _, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file/baz/")
 193         c.Assert(err, check.Equals, ErrNotADirectory)
 194         _, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file/baz/z")
 195         c.Assert(err, check.Equals, ErrNotADirectory)
 196         _, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file/baz/..")
 197         c.Assert(err, check.Equals, ErrNotADirectory)
 198
 199         _, err = s.fs.OpenFile("/by_id/"+fixtureNonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
 200         c.Check(err, ErrorIs, ErrInvalidOperation)
 201         err = s.fs.Rename("/by_id/"+fixtureFooCollection, "/by_id/beep")
 202         c.Check(err, ErrorIs, ErrInvalidOperation)
 203         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/beep")
 204         c.Check(err, ErrorIs, ErrInvalidOperation)
 205         _, err = s.fs.Stat("/by_id/beep")
 206         c.Check(err, check.Equals, os.ErrNotExist)
 207         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/"+fixtureFooCollection+"/bar")
 208         c.Check(err, check.IsNil)
 209
 210         err = s.fs.Rename("/by_id", "/beep")
 211         c.Check(err, ErrorIs, ErrInvalidOperation)
 212 }
 213
 214 // Copy subtree from OS src to dst path inside fs. If src is a
 215 // directory, dst must exist and be a directory.
 216 func copyFromOS(fs FileSystem, dst, src string) error {
 217         inf, err := os.Open(src)
 218         if err != nil {
 219                 return err
 220         }
 221         defer inf.Close()
 222         dirents, err := inf.Readdir(-1)
 223         if e, ok := err.(*os.PathError); ok {
 224                 if e, ok := e.Err.(syscall.Errno); ok {
 225                         if e == syscall.ENOTDIR {
 226                                 err = syscall.ENOTDIR
 227                         }
 228                 }
 229         }
 230         if err == syscall.ENOTDIR {
 231                 outf, err := fs.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_TRUNC|os.O_WRONLY, 0700)
 232                 if err != nil {
 233                         return fmt.Errorf("open %s: %s", dst, err)
 234                 }
 235                 defer outf.Close()
 236                 _, err = io.Copy(outf, inf)
 237                 if err != nil {
 238                         return fmt.Errorf("%s: copying data from %s: %s", dst, src, err)
 239                 }
 240                 err = outf.Close()
 241                 if err != nil {
 242                         return err
 243                 }
 244         } else if err != nil {
 245                 return fmt.Errorf("%s: readdir: %T %s", src, err, err)
 246         } else {
 247                 {
 248                         d, err := fs.Open(dst)
 249                         if err != nil {
 250                                 return fmt.Errorf("opendir(%s): %s", dst, err)
 251                         }
 252                         d.Close()
 253                 }
 254                 for _, ent := range dirents {
 255                         if ent.Name() == "." || ent.Name() == ".." {
 256                                 continue
 257                         }
 258                         dstname := dst + "/" + ent.Name()
 259                         if ent.IsDir() {
 260                                 err = fs.Mkdir(dstname, 0700)
 261                                 if err != nil {
 262                                         return fmt.Errorf("mkdir %s: %s", dstname, err)
 263                                 }
 264                         }
 265                         err = copyFromOS(fs, dstname, src+"/"+ent.Name())
 266                         if err != nil {
 267                                 return err
 268                         }
 269                 }
 270         }
 271         return nil
 272 }
 273
 274 func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
 275         s.fs.MountProject("home", "")
 276         thisfile, err := ioutil.ReadFile("fs_site_test.go")
 277         c.Assert(err, check.IsNil)
 278
 279         var src1 Collection
 280         err = s.client.RequestAndDecode(&src1, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 281                 "collection": map[string]string{
 282                         "name":       "TestSnapshotSplice src1",
 283                         "owner_uuid": fixtureAProjectUUID,
 284                 },
 285         })
 286         c.Assert(err, check.IsNil)
 287         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src1.UUID, nil, nil)
 288         err = s.fs.Sync()
 289         c.Assert(err, check.IsNil)
 290         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src1", "..") // arvados.git/sdk/go
 291         c.Assert(err, check.IsNil)
 292
 293         var src2 Collection
 294         err = s.client.RequestAndDecode(&src2, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 295                 "collection": map[string]string{
 296                         "name":       "TestSnapshotSplice src2",
 297                         "owner_uuid": fixtureAProjectUUID,
 298                 },
 299         })
 300         c.Assert(err, check.IsNil)
 301         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src2.UUID, nil, nil)
 302         err = s.fs.Sync()
 303         c.Assert(err, check.IsNil)
 304         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src2", "..") // arvados.git/sdk/go
 305         c.Assert(err, check.IsNil)
 306
 307         var dst Collection
 308         err = s.client.RequestAndDecode(&dst, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 309                 "collection": map[string]string{
 310                         "name":       "TestSnapshotSplice dst",
 311                         "owner_uuid": fixtureAProjectUUID,
 312                 },
 313         })
 314         c.Assert(err, check.IsNil)
 315         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+dst.UUID, nil, nil)
 316         err = s.fs.Sync()
 317         c.Assert(err, check.IsNil)
 318
 319         dstPath := "/home/A Project/TestSnapshotSplice dst"
 320         err = copyFromOS(s.fs, dstPath, "..") // arvados.git/sdk/go
 321         c.Assert(err, check.IsNil)
 322
 323         // Snapshot directory
 324         snap1, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/ctxlog")
 325         c.Check(err, check.IsNil)
 326         // Attach same snapshot twice, at paths that didn't exist before
 327         err = Splice(s.fs, dstPath+"/ctxlog-copy", snap1)
 328         c.Check(err, check.IsNil)
 329         err = Splice(s.fs, dstPath+"/ctxlog-copy2", snap1)
 330         c.Check(err, check.IsNil)
 331         // Splicing a snapshot twice results in two independent copies
 332         err = s.fs.Rename(dstPath+"/ctxlog-copy2/log.go", dstPath+"/ctxlog-copy/log2.go")
 333         c.Check(err, check.IsNil)
 334         _, err = s.fs.Open(dstPath + "/ctxlog-copy2/log.go")
 335         c.Check(err, check.Equals, os.ErrNotExist)
 336         f, err := s.fs.Open(dstPath + "/ctxlog-copy/log.go")
 337         if c.Check(err, check.IsNil) {
 338                 buf, err := ioutil.ReadAll(f)
 339                 c.Check(err, check.IsNil)
 340                 c.Check(string(buf), check.Not(check.Equals), "")
 341                 f.Close()
 342         }
 343
 344         // Snapshot regular file
 345         snapFile, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/arvados/fs_site_test.go")
 346         c.Check(err, check.IsNil)
 347         // Replace dir with file
 348         err = Splice(s.fs, dstPath+"/ctxlog-copy2", snapFile)
 349         c.Check(err, check.IsNil)
 350         if f, err := s.fs.Open(dstPath + "/ctxlog-copy2"); c.Check(err, check.IsNil) {
 351                 buf, err := ioutil.ReadAll(f)
 352                 c.Check(err, check.IsNil)
 353                 c.Check(string(buf), check.Equals, string(thisfile))
 354         }
 355
 356         // Cannot splice a file onto a collection root; cannot splice
 357         // anything to a target outside a collection.
 358         for _, badpath := range []string{
 359                 dstPath + "/",
 360                 dstPath,
 361                 "/home/A Project/newnodename/",
 362                 "/home/A Project/newnodename",
 363                 "/home/A Project/",
 364                 "/home/A Project",
 365                 "/home/newnodename/",
 366                 "/home/newnodename",
 367                 "/home/",
 368                 "/home",
 369                 "/newnodename/",
 370                 "/newnodename",
 371                 "/",
 372         } {
 373                 err = Splice(s.fs, badpath, snapFile)
 374                 c.Check(err, check.NotNil)
 375                 if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
 376                         c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
 377                 } else {
 378                         c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
 379                 }
 380                 if strings.TrimSuffix(badpath, "/") == dstPath {
 381                         c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %q", badpath))
 382                         continue
 383                 }
 384
 385                 err = Splice(s.fs, badpath, snap1)
 386                 if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
 387                         c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
 388                 } else {
 389                         c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
 390                 }
 391         }
 392
 393         // Destination's parent must already exist
 394         for _, badpath := range []string{
 395                 dstPath + "/newdirname/",
 396                 dstPath + "/newdirname/foobar",
 397                 "/foo/bar",
 398         } {
 399                 err = Splice(s.fs, badpath, snap1)
 400                 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
 401                 err = Splice(s.fs, badpath, snapFile)
 402                 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
 403         }
 404
 405         snap2, err := Snapshot(s.fs, dstPath+"/ctxlog-copy")
 406         if c.Check(err, check.IsNil) {
 407                 err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
 408                 c.Check(err, check.IsNil)
 409         }
 410
 411         // Snapshot entire collection, splice into same collection at
 412         // a new path, remove file from original location, verify
 413         // spliced content survives
 414         snapDst, err := Snapshot(s.fs, dstPath+"")
 415         c.Check(err, check.IsNil)
 416         err = Splice(s.fs, dstPath+"", snapDst)
 417         c.Check(err, check.IsNil)
 418         err = Splice(s.fs, dstPath+"/copy1", snapDst)
 419         c.Check(err, check.IsNil)
 420         err = Splice(s.fs, dstPath+"/copy2", snapDst)
 421         c.Check(err, check.IsNil)
 422         err = s.fs.RemoveAll(dstPath + "/arvados/fs_site_test.go")
 423         c.Check(err, check.IsNil)
 424         err = s.fs.RemoveAll(dstPath + "/arvados")
 425         c.Check(err, check.IsNil)
 426         _, err = s.fs.Open(dstPath + "/arvados/fs_site_test.go")
 427         c.Check(err, check.Equals, os.ErrNotExist)
 428         f, err = s.fs.Open(dstPath + "/copy2/arvados/fs_site_test.go")
 429         if c.Check(err, check.IsNil) {
 430                 defer f.Close()
 431                 buf, err := ioutil.ReadAll(f)
 432                 c.Check(err, check.IsNil)
 433                 c.Check(string(buf), check.Equals, string(thisfile))
 434         }
 435 }
 436
 437 func (s *SiteFSSuite) TestLocks(c *check.C) {
 438         DebugLocksPanicMode = false
 439         done := make(chan struct{})
 440         defer close(done)
 441         ticker := time.NewTicker(2 * time.Second)
 442         go func() {
 443                 for {
 444                         timeout := time.AfterFunc(5*time.Second, func() {
 445                                 // c.FailNow() doesn't break deadlock, but this sure does
 446                                 panic("timed out -- deadlock?")
 447                         })
 448                         select {
 449                         case <-done:
 450                                 timeout.Stop()
 451                                 return
 452                         case <-ticker.C:
 453                                 c.Logf("MemorySize == %d", s.fs.MemorySize())
 454                         }
 455                         timeout.Stop()
 456                 }
 457         }()
 458         ncolls := 5
 459         ndirs := 3
 460         nfiles := 5
 461         projects := make([]Group, 5)
 462         for pnum := range projects {
 463                 c.Logf("make project %d", pnum)
 464                 err := s.client.RequestAndDecode(&projects[pnum], "POST", "arvados/v1/groups", nil, map[string]interface{}{
 465                         "group": map[string]string{
 466                                 "name":        fmt.Sprintf("TestLocks project %d", pnum),
 467                                 "owner_uuid":  fixtureAProjectUUID,
 468                                 "group_class": "project",
 469                         },
 470                         "ensure_unique_name": true,
 471                 })
 472                 c.Assert(err, check.IsNil)
 473                 for cnum := 0; cnum < ncolls; cnum++ {
 474                         c.Logf("make project %d collection %d", pnum, cnum)
 475                         var coll Collection
 476                         err = s.client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 477                                 "collection": map[string]string{
 478                                         "name":       fmt.Sprintf("TestLocks collection %d", cnum),
 479                                         "owner_uuid": projects[pnum].UUID,
 480                                 },
 481                         })
 482                         c.Assert(err, check.IsNil)
 483                         for d1num := 0; d1num < ndirs; d1num++ {
 484                                 s.fs.Mkdir(fmt.Sprintf("/by_id/%s/dir1-%d", coll.UUID, d1num), 0777)
 485                                 for d2num := 0; d2num < ndirs; d2num++ {
 486                                         s.fs.Mkdir(fmt.Sprintf("/by_id/%s/dir1-%d/dir2-%d", coll.UUID, d1num, d2num), 0777)
 487                                         for fnum := 0; fnum < nfiles; fnum++ {
 488                                                 f, err := s.fs.OpenFile(fmt.Sprintf("/by_id/%s/dir1-%d/dir2-%d/file-%d", coll.UUID, d1num, d2num, fnum), os.O_CREATE|os.O_RDWR, 0755)
 489                                                 c.Assert(err, check.IsNil)
 490                                                 f.Close()
 491                                                 f, err = s.fs.OpenFile(fmt.Sprintf("/by_id/%s/dir1-%d/file-%d", coll.UUID, d1num, fnum), os.O_CREATE|os.O_RDWR, 0755)
 492                                                 c.Assert(err, check.IsNil)
 493                                                 f.Close()
 494                                         }
 495                                 }
 496                         }
 497                 }
 498         }
 499         c.Log("sync")
 500         s.fs.Sync()
 501         var wg sync.WaitGroup
 502         for n := 0; n < 100; n++ {
 503                 wg.Add(1)
 504                 go func() {
 505                         defer wg.Done()
 506                         for pnum, project := range projects {
 507                                 c.Logf("read project %d", pnum)
 508                                 if pnum%2 == 0 {
 509                                         f, err := s.fs.Open(fmt.Sprintf("/by_id/%s", project.UUID))
 510                                         c.Assert(err, check.IsNil)
 511                                         f.Readdir(-1)
 512                                         f.Close()
 513                                 }
 514                                 for cnum := 0; cnum < ncolls; cnum++ {
 515                                         c.Logf("read project %d collection %d", pnum, cnum)
 516                                         if pnum%2 == 0 {
 517                                                 f, err := s.fs.Open(fmt.Sprintf("/by_id/%s/TestLocks collection %d", project.UUID, cnum))
 518                                                 c.Assert(err, check.IsNil)
 519                                                 _, err = f.Readdir(-1)
 520                                                 c.Assert(err, check.IsNil)
 521                                                 f.Close()
 522                                         }
 523                                         if pnum%3 == 0 {
 524                                                 for d1num := 0; d1num < ndirs; d1num++ {
 525                                                         f, err := s.fs.Open(fmt.Sprintf("/by_id/%s/TestLocks collection %d/dir1-%d", project.UUID, cnum, d1num))
 526                                                         c.Assert(err, check.IsNil)
 527                                                         fis, err := f.Readdir(-1)
 528                                                         c.Assert(err, check.IsNil)
 529                                                         c.Assert(fis, check.HasLen, ndirs+nfiles)
 530                                                         f.Close()
 531                                                 }
 532                                         }
 533                                         for d1num := 0; d1num < ndirs; d1num++ {
 534                                                 for d2num := 0; d2num < ndirs; d2num++ {
 535                                                         f, err := s.fs.Open(fmt.Sprintf("/by_id/%s/TestLocks collection %d/dir1-%d/dir2-%d", project.UUID, cnum, d1num, d2num))
 536                                                         c.Assert(err, check.IsNil)
 537                                                         fis, err := f.Readdir(-1)
 538                                                         c.Assert(err, check.IsNil)
 539                                                         c.Assert(fis, check.HasLen, nfiles)
 540                                                         f.Close()
 541                                                 }
 542                                         }
 543                                 }
 544                         }
 545                 }()
 546         }
 547         wg.Wait()
 548         c.Logf("MemorySize == %d", s.fs.MemorySize())
 549 }