sdk/go/arvados/fs_site_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "fmt"
   9         "io"
  10         "io/ioutil"
  11         "net/http"
  12         "os"
  13         "strings"
  14         "sync"
  15         "syscall"
  16         "time"
  17
  18         check "gopkg.in/check.v1"
  19 )
  20
  21 const (
  22         // Importing arvadostest would be an import cycle, so these
  23         // fixtures are duplicated here [until fs moves to a separate
  24         // package].
  25         fixtureActiveUserUUID               = "zzzzz-tpzed-xurymjxw79nv3jz"
  26         fixtureActiveToken                  = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
  27         fixtureAProjectUUID                 = "zzzzz-j7d0g-v955i6s2oi1cbso"
  28         fixtureThisFilterGroupUUID          = "zzzzz-j7d0g-thisfiltergroup"
  29         fixtureAFilterGroupTwoUUID          = "zzzzz-j7d0g-afiltergrouptwo"
  30         fixtureAFilterGroupThreeUUID        = "zzzzz-j7d0g-filtergroupthre"
  31         fixtureAFilterGroupFourUUID         = "zzzzz-j7d0g-filtergroupfour"
  32         fixtureAFilterGroupFiveUUID         = "zzzzz-j7d0g-filtergroupfive"
  33         fixtureFooAndBarFilesInDirUUID      = "zzzzz-4zz18-foonbarfilesdir"
  34         fixtureFooCollectionName            = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
  35         fixtureFooCollectionPDH             = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
  36         fixtureFooCollection                = "zzzzz-4zz18-fy296fx3hot09f7"
  37         fixtureNonexistentCollection        = "zzzzz-4zz18-totallynotexist"
  38         fixtureStorageClassesDesiredArchive = "zzzzz-4zz18-3t236wr12769qqa"
  39         fixtureBlobSigningKey               = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
  40         fixtureBlobSigningTTL               = 336 * time.Hour
  41 )
  42
  43 var _ = check.Suite(&SiteFSSuite{})
  44
  45 func init() {
  46         // Enable DebugLocksPanicMode sometimes. Don't enable it all
  47         // the time, though -- it adds many calls to time.Sleep(),
  48         // which could hide different bugs.
  49         if time.Now().Second()&1 == 0 {
  50                 DebugLocksPanicMode = true
  51         }
  52 }
  53
  54 type SiteFSSuite struct {
  55         client *Client
  56         fs     CustomFileSystem
  57         kc     keepClient
  58 }
  59
  60 func (s *SiteFSSuite) SetUpTest(c *check.C) {
  61         s.client = &Client{
  62                 APIHost:   os.Getenv("ARVADOS_API_HOST"),
  63                 AuthToken: fixtureActiveToken,
  64                 Insecure:  true,
  65         }
  66         s.kc = &keepClientStub{
  67                 blocks: map[string][]byte{
  68                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  69                 },
  70                 sigkey:    fixtureBlobSigningKey,
  71                 sigttl:    fixtureBlobSigningTTL,
  72                 authToken: fixtureActiveToken,
  73         }
  74         s.fs = s.client.SiteFileSystem(s.kc)
  75 }
  76
  77 func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
  78         _, ok := s.fs.(http.FileSystem)
  79         c.Check(ok, check.Equals, true)
  80 }
  81
  82 func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
  83         f, err := s.fs.Open("/by_id")
  84         c.Assert(err, check.IsNil)
  85         fis, err := f.Readdir(-1)
  86         c.Check(err, check.IsNil)
  87         c.Check(len(fis), check.Equals, 0)
  88 }
  89
  90 func (s *SiteFSSuite) TestUpdateStorageClasses(c *check.C) {
  91         f, err := s.fs.OpenFile("/by_id/"+fixtureStorageClassesDesiredArchive+"/newfile", os.O_CREATE|os.O_RDWR, 0777)
  92         c.Assert(err, check.IsNil)
  93         _, err = f.Write([]byte("nope"))
  94         c.Assert(err, check.IsNil)
  95         err = f.Close()
  96         c.Assert(err, check.IsNil)
  97         err = s.fs.Sync()
  98         c.Assert(err, check.ErrorMatches, `.*stub does not write storage class "archive"`)
  99 }
 100
 101 func (s *SiteFSSuite) TestSameCollectionDifferentPaths(c *check.C) {
 102         s.fs.MountProject("home", "")
 103         var coll Collection
 104         err := s.client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 105                 "collection": map[string]interface{}{
 106                         "owner_uuid": fixtureAProjectUUID,
 107                         "name":       fmt.Sprintf("test collection %d", time.Now().UnixNano()),
 108                 },
 109         })
 110         c.Assert(err, check.IsNil)
 111
 112         viaProjID := "by_id/" + fixtureAProjectUUID + "/" + coll.Name
 113         viaProjName := "home/A Project/" + coll.Name
 114         viaCollID := "by_id/" + coll.UUID
 115         for n, dirs := range [][]string{
 116                 {viaCollID, viaProjID, viaProjName},
 117                 {viaCollID, viaProjName, viaProjID},
 118                 {viaProjID, viaProjName, viaCollID},
 119                 {viaProjID, viaCollID, viaProjName},
 120                 {viaProjName, viaCollID, viaProjID},
 121                 {viaProjName, viaProjID, viaCollID},
 122         } {
 123                 filename := fmt.Sprintf("file %d", n)
 124                 f := make([]File, 3)
 125                 for i, dir := range dirs {
 126                         path := dir + "/" + filename
 127                         mode := os.O_RDWR
 128                         if i == 0 {
 129                                 mode |= os.O_CREATE
 130                                 c.Logf("create %s", path)
 131                         } else {
 132                                 c.Logf("open %s", path)
 133                         }
 134                         f[i], err = s.fs.OpenFile(path, mode, 0777)
 135                         c.Assert(err, check.IsNil, check.Commentf("n=%d i=%d path=%s", n, i, path))
 136                         defer f[i].Close()
 137                 }
 138                 _, err = io.WriteString(f[0], filename)
 139                 c.Assert(err, check.IsNil)
 140                 _, err = f[1].Seek(0, io.SeekEnd)
 141                 c.Assert(err, check.IsNil)
 142                 _, err = io.WriteString(f[1], filename)
 143                 c.Assert(err, check.IsNil)
 144                 buf, err := io.ReadAll(f[2])
 145                 c.Assert(err, check.IsNil)
 146                 c.Check(string(buf), check.Equals, filename+filename)
 147         }
 148 }
 149
 150 func (s *SiteFSSuite) TestByUUIDAndPDH(c *check.C) {
 151         f, err := s.fs.Open("/by_id")
 152         c.Assert(err, check.IsNil)
 153         fis, err := f.Readdir(-1)
 154         c.Check(err, check.IsNil)
 155         c.Check(len(fis), check.Equals, 0)
 156
 157         err = s.fs.Mkdir("/by_id/"+fixtureFooCollection, 0755)
 158         c.Check(err, check.Equals, os.ErrExist)
 159
 160         f, err = s.fs.Open("/by_id/" + fixtureNonexistentCollection)
 161         c.Assert(err, check.Equals, os.ErrNotExist)
 162
 163         for _, path := range []string{
 164                 fixtureFooCollection,
 165                 fixtureFooCollectionPDH,
 166                 fixtureAProjectUUID + "/" + fixtureFooCollectionName,
 167         } {
 168                 f, err = s.fs.Open("/by_id/" + path)
 169                 c.Assert(err, check.IsNil)
 170                 fis, err = f.Readdir(-1)
 171                 c.Assert(err, check.IsNil)
 172                 var names []string
 173                 for _, fi := range fis {
 174                         names = append(names, fi.Name())
 175                 }
 176                 c.Check(names, check.DeepEquals, []string{"foo"})
 177         }
 178
 179         f, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file")
 180         c.Assert(err, check.IsNil)
 181         fis, err = f.Readdir(-1)
 182         c.Assert(err, check.IsNil)
 183         var names []string
 184         for _, fi := range fis {
 185                 names = append(names, fi.Name())
 186         }
 187         c.Check(names, check.DeepEquals, []string{"baz"})
 188
 189         _, err = s.fs.OpenFile("/by_id/"+fixtureNonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
 190         c.Check(err, ErrorIs, ErrInvalidOperation)
 191         err = s.fs.Rename("/by_id/"+fixtureFooCollection, "/by_id/beep")
 192         c.Check(err, ErrorIs, ErrInvalidOperation)
 193         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/beep")
 194         c.Check(err, ErrorIs, ErrInvalidOperation)
 195         _, err = s.fs.Stat("/by_id/beep")
 196         c.Check(err, check.Equals, os.ErrNotExist)
 197         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/"+fixtureFooCollection+"/bar")
 198         c.Check(err, check.IsNil)
 199
 200         err = s.fs.Rename("/by_id", "/beep")
 201         c.Check(err, ErrorIs, ErrInvalidOperation)
 202 }
 203
 204 // Copy subtree from OS src to dst path inside fs. If src is a
 205 // directory, dst must exist and be a directory.
 206 func copyFromOS(fs FileSystem, dst, src string) error {
 207         inf, err := os.Open(src)
 208         if err != nil {
 209                 return err
 210         }
 211         defer inf.Close()
 212         dirents, err := inf.Readdir(-1)
 213         if e, ok := err.(*os.PathError); ok {
 214                 if e, ok := e.Err.(syscall.Errno); ok {
 215                         if e == syscall.ENOTDIR {
 216                                 err = syscall.ENOTDIR
 217                         }
 218                 }
 219         }
 220         if err == syscall.ENOTDIR {
 221                 outf, err := fs.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_TRUNC|os.O_WRONLY, 0700)
 222                 if err != nil {
 223                         return fmt.Errorf("open %s: %s", dst, err)
 224                 }
 225                 defer outf.Close()
 226                 _, err = io.Copy(outf, inf)
 227                 if err != nil {
 228                         return fmt.Errorf("%s: copying data from %s: %s", dst, src, err)
 229                 }
 230                 err = outf.Close()
 231                 if err != nil {
 232                         return err
 233                 }
 234         } else if err != nil {
 235                 return fmt.Errorf("%s: readdir: %T %s", src, err, err)
 236         } else {
 237                 {
 238                         d, err := fs.Open(dst)
 239                         if err != nil {
 240                                 return fmt.Errorf("opendir(%s): %s", dst, err)
 241                         }
 242                         d.Close()
 243                 }
 244                 for _, ent := range dirents {
 245                         if ent.Name() == "." || ent.Name() == ".." {
 246                                 continue
 247                         }
 248                         dstname := dst + "/" + ent.Name()
 249                         if ent.IsDir() {
 250                                 err = fs.Mkdir(dstname, 0700)
 251                                 if err != nil {
 252                                         return fmt.Errorf("mkdir %s: %s", dstname, err)
 253                                 }
 254                         }
 255                         err = copyFromOS(fs, dstname, src+"/"+ent.Name())
 256                         if err != nil {
 257                                 return err
 258                         }
 259                 }
 260         }
 261         return nil
 262 }
 263
 264 func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
 265         s.fs.MountProject("home", "")
 266         thisfile, err := ioutil.ReadFile("fs_site_test.go")
 267         c.Assert(err, check.IsNil)
 268
 269         var src1 Collection
 270         err = s.client.RequestAndDecode(&src1, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 271                 "collection": map[string]string{
 272                         "name":       "TestSnapshotSplice src1",
 273                         "owner_uuid": fixtureAProjectUUID,
 274                 },
 275         })
 276         c.Assert(err, check.IsNil)
 277         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src1.UUID, nil, nil)
 278         err = s.fs.Sync()
 279         c.Assert(err, check.IsNil)
 280         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src1", "..") // arvados.git/sdk/go
 281         c.Assert(err, check.IsNil)
 282
 283         var src2 Collection
 284         err = s.client.RequestAndDecode(&src2, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 285                 "collection": map[string]string{
 286                         "name":       "TestSnapshotSplice src2",
 287                         "owner_uuid": fixtureAProjectUUID,
 288                 },
 289         })
 290         c.Assert(err, check.IsNil)
 291         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src2.UUID, nil, nil)
 292         err = s.fs.Sync()
 293         c.Assert(err, check.IsNil)
 294         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src2", "..") // arvados.git/sdk/go
 295         c.Assert(err, check.IsNil)
 296
 297         var dst Collection
 298         err = s.client.RequestAndDecode(&dst, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 299                 "collection": map[string]string{
 300                         "name":       "TestSnapshotSplice dst",
 301                         "owner_uuid": fixtureAProjectUUID,
 302                 },
 303         })
 304         c.Assert(err, check.IsNil)
 305         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+dst.UUID, nil, nil)
 306         err = s.fs.Sync()
 307         c.Assert(err, check.IsNil)
 308
 309         dstPath := "/home/A Project/TestSnapshotSplice dst"
 310         err = copyFromOS(s.fs, dstPath, "..") // arvados.git/sdk/go
 311         c.Assert(err, check.IsNil)
 312
 313         // Snapshot directory
 314         snap1, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/ctxlog")
 315         c.Check(err, check.IsNil)
 316         // Attach same snapshot twice, at paths that didn't exist before
 317         err = Splice(s.fs, dstPath+"/ctxlog-copy", snap1)
 318         c.Check(err, check.IsNil)
 319         err = Splice(s.fs, dstPath+"/ctxlog-copy2", snap1)
 320         c.Check(err, check.IsNil)
 321         // Splicing a snapshot twice results in two independent copies
 322         err = s.fs.Rename(dstPath+"/ctxlog-copy2/log.go", dstPath+"/ctxlog-copy/log2.go")
 323         c.Check(err, check.IsNil)
 324         _, err = s.fs.Open(dstPath + "/ctxlog-copy2/log.go")
 325         c.Check(err, check.Equals, os.ErrNotExist)
 326         f, err := s.fs.Open(dstPath + "/ctxlog-copy/log.go")
 327         if c.Check(err, check.IsNil) {
 328                 buf, err := ioutil.ReadAll(f)
 329                 c.Check(err, check.IsNil)
 330                 c.Check(string(buf), check.Not(check.Equals), "")
 331                 f.Close()
 332         }
 333
 334         // Snapshot regular file
 335         snapFile, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/arvados/fs_site_test.go")
 336         c.Check(err, check.IsNil)
 337         // Replace dir with file
 338         err = Splice(s.fs, dstPath+"/ctxlog-copy2", snapFile)
 339         c.Check(err, check.IsNil)
 340         if f, err := s.fs.Open(dstPath + "/ctxlog-copy2"); c.Check(err, check.IsNil) {
 341                 buf, err := ioutil.ReadAll(f)
 342                 c.Check(err, check.IsNil)
 343                 c.Check(string(buf), check.Equals, string(thisfile))
 344         }
 345
 346         // Cannot splice a file onto a collection root; cannot splice
 347         // anything to a target outside a collection.
 348         for _, badpath := range []string{
 349                 dstPath + "/",
 350                 dstPath,
 351                 "/home/A Project/newnodename/",
 352                 "/home/A Project/newnodename",
 353                 "/home/A Project/",
 354                 "/home/A Project",
 355                 "/home/newnodename/",
 356                 "/home/newnodename",
 357                 "/home/",
 358                 "/home",
 359                 "/newnodename/",
 360                 "/newnodename",
 361                 "/",
 362         } {
 363                 err = Splice(s.fs, badpath, snapFile)
 364                 c.Check(err, check.NotNil)
 365                 if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
 366                         c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
 367                 } else {
 368                         c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
 369                 }
 370                 if strings.TrimSuffix(badpath, "/") == dstPath {
 371                         c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %q", badpath))
 372                         continue
 373                 }
 374
 375                 err = Splice(s.fs, badpath, snap1)
 376                 if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
 377                         c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
 378                 } else {
 379                         c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
 380                 }
 381         }
 382
 383         // Destination's parent must already exist
 384         for _, badpath := range []string{
 385                 dstPath + "/newdirname/",
 386                 dstPath + "/newdirname/foobar",
 387                 "/foo/bar",
 388         } {
 389                 err = Splice(s.fs, badpath, snap1)
 390                 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
 391                 err = Splice(s.fs, badpath, snapFile)
 392                 c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
 393         }
 394
 395         snap2, err := Snapshot(s.fs, dstPath+"/ctxlog-copy")
 396         if c.Check(err, check.IsNil) {
 397                 err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
 398                 c.Check(err, check.IsNil)
 399         }
 400
 401         // Snapshot entire collection, splice into same collection at
 402         // a new path, remove file from original location, verify
 403         // spliced content survives
 404         snapDst, err := Snapshot(s.fs, dstPath+"")
 405         c.Check(err, check.IsNil)
 406         err = Splice(s.fs, dstPath+"", snapDst)
 407         c.Check(err, check.IsNil)
 408         err = Splice(s.fs, dstPath+"/copy1", snapDst)
 409         c.Check(err, check.IsNil)
 410         err = Splice(s.fs, dstPath+"/copy2", snapDst)
 411         c.Check(err, check.IsNil)
 412         err = s.fs.RemoveAll(dstPath + "/arvados/fs_site_test.go")
 413         c.Check(err, check.IsNil)
 414         err = s.fs.RemoveAll(dstPath + "/arvados")
 415         c.Check(err, check.IsNil)
 416         _, err = s.fs.Open(dstPath + "/arvados/fs_site_test.go")
 417         c.Check(err, check.Equals, os.ErrNotExist)
 418         f, err = s.fs.Open(dstPath + "/copy2/arvados/fs_site_test.go")
 419         if c.Check(err, check.IsNil) {
 420                 defer f.Close()
 421                 buf, err := ioutil.ReadAll(f)
 422                 c.Check(err, check.IsNil)
 423                 c.Check(string(buf), check.Equals, string(thisfile))
 424         }
 425 }
 426
 427 func (s *SiteFSSuite) TestLocks(c *check.C) {
 428         DebugLocksPanicMode = false
 429         done := make(chan struct{})
 430         defer close(done)
 431         ticker := time.NewTicker(2 * time.Second)
 432         go func() {
 433                 for {
 434                         timeout := time.AfterFunc(5*time.Second, func() {
 435                                 // c.FailNow() doesn't break deadlock, but this sure does
 436                                 panic("timed out -- deadlock?")
 437                         })
 438                         select {
 439                         case <-done:
 440                                 timeout.Stop()
 441                                 return
 442                         case <-ticker.C:
 443                                 c.Logf("MemorySize == %d", s.fs.MemorySize())
 444                         }
 445                         timeout.Stop()
 446                 }
 447         }()
 448         ncolls := 5
 449         ndirs := 3
 450         nfiles := 5
 451         projects := make([]Group, 5)
 452         for pnum := range projects {
 453                 c.Logf("make project %d", pnum)
 454                 err := s.client.RequestAndDecode(&projects[pnum], "POST", "arvados/v1/groups", nil, map[string]interface{}{
 455                         "group": map[string]string{
 456                                 "name":        fmt.Sprintf("TestLocks project %d", pnum),
 457                                 "owner_uuid":  fixtureAProjectUUID,
 458                                 "group_class": "project",
 459                         },
 460                         "ensure_unique_name": true,
 461                 })
 462                 c.Assert(err, check.IsNil)
 463                 for cnum := 0; cnum < ncolls; cnum++ {
 464                         c.Logf("make project %d collection %d", pnum, cnum)
 465                         var coll Collection
 466                         err = s.client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 467                                 "collection": map[string]string{
 468                                         "name":       fmt.Sprintf("TestLocks collection %d", cnum),
 469                                         "owner_uuid": projects[pnum].UUID,
 470                                 },
 471                         })
 472                         c.Assert(err, check.IsNil)
 473                         for d1num := 0; d1num < ndirs; d1num++ {
 474                                 s.fs.Mkdir(fmt.Sprintf("/by_id/%s/dir1-%d", coll.UUID, d1num), 0777)
 475                                 for d2num := 0; d2num < ndirs; d2num++ {
 476                                         s.fs.Mkdir(fmt.Sprintf("/by_id/%s/dir1-%d/dir2-%d", coll.UUID, d1num, d2num), 0777)
 477                                         for fnum := 0; fnum < nfiles; fnum++ {
 478                                                 f, err := s.fs.OpenFile(fmt.Sprintf("/by_id/%s/dir1-%d/dir2-%d/file-%d", coll.UUID, d1num, d2num, fnum), os.O_CREATE|os.O_RDWR, 0755)
 479                                                 c.Assert(err, check.IsNil)
 480                                                 f.Close()
 481                                                 f, err = s.fs.OpenFile(fmt.Sprintf("/by_id/%s/dir1-%d/file-%d", coll.UUID, d1num, fnum), os.O_CREATE|os.O_RDWR, 0755)
 482                                                 c.Assert(err, check.IsNil)
 483                                                 f.Close()
 484                                         }
 485                                 }
 486                         }
 487                 }
 488         }
 489         c.Log("sync")
 490         s.fs.Sync()
 491         var wg sync.WaitGroup
 492         for n := 0; n < 100; n++ {
 493                 wg.Add(1)
 494                 go func() {
 495                         defer wg.Done()
 496                         for pnum, project := range projects {
 497                                 c.Logf("read project %d", pnum)
 498                                 if pnum%2 == 0 {
 499                                         f, err := s.fs.Open(fmt.Sprintf("/by_id/%s", project.UUID))
 500                                         c.Assert(err, check.IsNil)
 501                                         f.Readdir(-1)
 502                                         f.Close()
 503                                 }
 504                                 for cnum := 0; cnum < ncolls; cnum++ {
 505                                         c.Logf("read project %d collection %d", pnum, cnum)
 506                                         if pnum%2 == 0 {
 507                                                 f, err := s.fs.Open(fmt.Sprintf("/by_id/%s/TestLocks collection %d", project.UUID, cnum))
 508                                                 c.Assert(err, check.IsNil)
 509                                                 _, err = f.Readdir(-1)
 510                                                 c.Assert(err, check.IsNil)
 511                                                 f.Close()
 512                                         }
 513                                         if pnum%3 == 0 {
 514                                                 for d1num := 0; d1num < ndirs; d1num++ {
 515                                                         f, err := s.fs.Open(fmt.Sprintf("/by_id/%s/TestLocks collection %d/dir1-%d", project.UUID, cnum, d1num))
 516                                                         c.Assert(err, check.IsNil)
 517                                                         fis, err := f.Readdir(-1)
 518                                                         c.Assert(err, check.IsNil)
 519                                                         c.Assert(fis, check.HasLen, ndirs+nfiles)
 520                                                         f.Close()
 521                                                 }
 522                                         }
 523                                         for d1num := 0; d1num < ndirs; d1num++ {
 524                                                 for d2num := 0; d2num < ndirs; d2num++ {
 525                                                         f, err := s.fs.Open(fmt.Sprintf("/by_id/%s/TestLocks collection %d/dir1-%d/dir2-%d", project.UUID, cnum, d1num, d2num))
 526                                                         c.Assert(err, check.IsNil)
 527                                                         fis, err := f.Readdir(-1)
 528                                                         c.Assert(err, check.IsNil)
 529                                                         c.Assert(fis, check.HasLen, nfiles)
 530                                                         f.Close()
 531                                                 }
 532                                         }
 533                                 }
 534                         }
 535                 }()
 536         }
 537         wg.Wait()
 538         c.Logf("MemorySize == %d", s.fs.MemorySize())
 539 }