sdk/go/arvados/fs_collection_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "crypto/sha1"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "math/rand"
  16         "net/http"
  17         "os"
  18         "regexp"
  19         "runtime"
  20         "strings"
  21         "sync"
  22         "testing"
  23         "time"
  24
  25         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  26         check "gopkg.in/check.v1"
  27 )
  28
  29 var _ = check.Suite(&CollectionFSSuite{})
  30
  31 type keepClientStub struct {
  32         blocks      map[string][]byte
  33         refreshable map[string]bool
  34         sync.RWMutex
  35 }
  36
  37 var errStub404 = errors.New("404 block not found")
  38
  39 func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error) {
  40         kcs.RLock()
  41         defer kcs.RUnlock()
  42         buf := kcs.blocks[locator[:32]]
  43         if buf == nil {
  44                 return 0, errStub404
  45         }
  46         return copy(p, buf[off:]), nil
  47 }
  48
  49 func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
  50         locator := fmt.Sprintf("%x+%d+A12345@abcde", md5.Sum(p), len(p))
  51         buf := make([]byte, len(p))
  52         copy(buf, p)
  53         kcs.Lock()
  54         defer kcs.Unlock()
  55         kcs.blocks[locator[:32]] = buf
  56         return locator, 1, nil
  57 }
  58
  59 var localOrRemoteSignature = regexp.MustCompile(`\+[AR][^+]*`)
  60
  61 func (kcs *keepClientStub) LocalLocator(locator string) (string, error) {
  62         kcs.Lock()
  63         defer kcs.Unlock()
  64         if strings.Contains(locator, "+R") {
  65                 if len(locator) < 32 {
  66                         return "", fmt.Errorf("bad locator: %q", locator)
  67                 }
  68                 if _, ok := kcs.blocks[locator[:32]]; !ok && !kcs.refreshable[locator[:32]] {
  69                         return "", fmt.Errorf("kcs.refreshable[%q]==false", locator)
  70                 }
  71         }
  72         fakeSig := fmt.Sprintf("+A%x@%x", sha1.Sum(nil), time.Now().Add(time.Hour*24*14).Unix())
  73         return localOrRemoteSignature.ReplaceAllLiteralString(locator, fakeSig), nil
  74 }
  75
  76 type CollectionFSSuite struct {
  77         client *Client
  78         coll   Collection
  79         fs     CollectionFileSystem
  80         kc     *keepClientStub
  81 }
  82
  83 func (s *CollectionFSSuite) SetUpTest(c *check.C) {
  84         s.client = NewClientFromEnv()
  85         err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
  86         c.Assert(err, check.IsNil)
  87         s.kc = &keepClientStub{
  88                 blocks: map[string][]byte{
  89                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  90                 }}
  91         s.fs, err = s.coll.FileSystem(s.client, s.kc)
  92         c.Assert(err, check.IsNil)
  93 }
  94
  95 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
  96         _, ok := s.fs.(http.FileSystem)
  97         c.Check(ok, check.Equals, true)
  98 }
  99
 100 func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
 101         fs, err := (&Collection{
 102                 ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
 103         }).FileSystem(s.client, s.kc)
 104         c.Assert(err, check.IsNil)
 105
 106         f, err := fs.Open("/foo:foo")
 107         c.Assert(err, check.IsNil)
 108
 109         fis, err := f.Readdir(0)
 110         c.Check(err, check.IsNil)
 111         c.Check(len(fis), check.Equals, 1)
 112         c.Check(fis[0].Name(), check.Equals, "bar:bar")
 113 }
 114
 115 func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
 116         f, err := s.fs.Open("/dir1")
 117         c.Assert(err, check.IsNil)
 118
 119         st, err := f.Stat()
 120         c.Assert(err, check.IsNil)
 121         c.Check(st.Size(), check.Equals, int64(2))
 122         c.Check(st.IsDir(), check.Equals, true)
 123
 124         fis, err := f.Readdir(0)
 125         c.Check(err, check.IsNil)
 126         c.Check(len(fis), check.Equals, 2)
 127         if len(fis) > 0 {
 128                 c.Check(fis[0].Size(), check.Equals, int64(3))
 129         }
 130 }
 131
 132 func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
 133         f, err := s.fs.Open("./dir1")
 134         c.Assert(err, check.IsNil)
 135
 136         fis, err := f.Readdir(1)
 137         c.Check(err, check.IsNil)
 138         c.Check(len(fis), check.Equals, 1)
 139         if len(fis) > 0 {
 140                 c.Check(fis[0].Size(), check.Equals, int64(3))
 141         }
 142
 143         fis, err = f.Readdir(1)
 144         c.Check(err, check.IsNil)
 145         c.Check(len(fis), check.Equals, 1)
 146         if len(fis) > 0 {
 147                 c.Check(fis[0].Size(), check.Equals, int64(3))
 148         }
 149
 150         fis, err = f.Readdir(1)
 151         c.Check(len(fis), check.Equals, 0)
 152         c.Check(err, check.NotNil)
 153         c.Check(err, check.Equals, io.EOF)
 154
 155         f, err = s.fs.Open("dir1")
 156         c.Assert(err, check.IsNil)
 157         fis, err = f.Readdir(1)
 158         c.Check(len(fis), check.Equals, 1)
 159         c.Assert(err, check.IsNil)
 160         fis, err = f.Readdir(2)
 161         c.Check(len(fis), check.Equals, 1)
 162         c.Assert(err, check.IsNil)
 163         fis, err = f.Readdir(2)
 164         c.Check(len(fis), check.Equals, 0)
 165         c.Assert(err, check.Equals, io.EOF)
 166 }
 167
 168 func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
 169         for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
 170                 f, err := s.fs.Open(path)
 171                 c.Assert(err, check.IsNil)
 172
 173                 st, err := f.Stat()
 174                 c.Assert(err, check.IsNil)
 175                 c.Check(st.Size(), check.Equals, int64(1))
 176                 c.Check(st.IsDir(), check.Equals, true)
 177         }
 178         for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
 179                 c.Logf("%q", path)
 180                 f, err := s.fs.Open(path)
 181                 c.Assert(err, check.IsNil)
 182
 183                 st, err := f.Stat()
 184                 c.Assert(err, check.IsNil)
 185                 c.Check(st.Size(), check.Equals, int64(2))
 186                 c.Check(st.IsDir(), check.Equals, true)
 187         }
 188 }
 189
 190 func (s *CollectionFSSuite) TestNotExist(c *check.C) {
 191         for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
 192                 f, err := s.fs.Open(path)
 193                 c.Assert(f, check.IsNil)
 194                 c.Assert(err, check.NotNil)
 195                 c.Assert(os.IsNotExist(err), check.Equals, true)
 196         }
 197 }
 198
 199 func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
 200         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDONLY, 0)
 201         c.Assert(err, check.IsNil)
 202         st, err := f.Stat()
 203         c.Assert(err, check.IsNil)
 204         c.Check(st.Size(), check.Equals, int64(3))
 205         n, err := f.Write([]byte("bar"))
 206         c.Check(n, check.Equals, 0)
 207         c.Check(err, check.Equals, ErrReadOnlyFile)
 208 }
 209
 210 func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
 211         f, err := s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE, 0)
 212         c.Assert(err, check.IsNil)
 213         st, err := f.Stat()
 214         c.Assert(err, check.IsNil)
 215         c.Check(st.Size(), check.Equals, int64(0))
 216
 217         n, err := f.Write([]byte("bar"))
 218         c.Check(n, check.Equals, 3)
 219         c.Check(err, check.IsNil)
 220
 221         c.Check(f.Close(), check.IsNil)
 222
 223         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 224         c.Check(f, check.IsNil)
 225         c.Assert(err, check.NotNil)
 226
 227         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR, 0)
 228         c.Assert(err, check.IsNil)
 229         st, err = f.Stat()
 230         c.Assert(err, check.IsNil)
 231         c.Check(st.Size(), check.Equals, int64(3))
 232
 233         c.Check(f.Close(), check.IsNil)
 234
 235         m, err := s.fs.MarshalManifest(".")
 236         c.Assert(err, check.IsNil)
 237         c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
 238 }
 239
 240 func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
 241         maxBlockSize = 8
 242         defer func() { maxBlockSize = 2 << 26 }()
 243
 244         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 245         c.Assert(err, check.IsNil)
 246         defer f.Close()
 247         st, err := f.Stat()
 248         c.Assert(err, check.IsNil)
 249         c.Check(st.Size(), check.Equals, int64(3))
 250
 251         f2, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 252         c.Assert(err, check.IsNil)
 253         defer f2.Close()
 254
 255         buf := make([]byte, 64)
 256         n, err := f.Read(buf)
 257         c.Check(n, check.Equals, 3)
 258         c.Check(err, check.Equals, io.EOF)
 259         c.Check(string(buf[:3]), check.DeepEquals, "foo")
 260
 261         pos, err := f.Seek(-2, io.SeekCurrent)
 262         c.Check(pos, check.Equals, int64(1))
 263         c.Check(err, check.IsNil)
 264
 265         // Split a storedExtent in two, and insert a memExtent
 266         n, err = f.Write([]byte("*"))
 267         c.Check(n, check.Equals, 1)
 268         c.Check(err, check.IsNil)
 269
 270         pos, err = f.Seek(0, io.SeekCurrent)
 271         c.Check(pos, check.Equals, int64(2))
 272         c.Check(err, check.IsNil)
 273
 274         pos, err = f.Seek(0, io.SeekStart)
 275         c.Check(pos, check.Equals, int64(0))
 276         c.Check(err, check.IsNil)
 277
 278         rbuf, err := ioutil.ReadAll(f)
 279         c.Check(len(rbuf), check.Equals, 3)
 280         c.Check(err, check.IsNil)
 281         c.Check(string(rbuf), check.Equals, "f*o")
 282
 283         // Write multiple blocks in one call
 284         f.Seek(1, io.SeekStart)
 285         n, err = f.Write([]byte("0123456789abcdefg"))
 286         c.Check(n, check.Equals, 17)
 287         c.Check(err, check.IsNil)
 288         pos, err = f.Seek(0, io.SeekCurrent)
 289         c.Check(pos, check.Equals, int64(18))
 290         c.Check(err, check.IsNil)
 291         pos, err = f.Seek(-18, io.SeekCurrent)
 292         c.Check(pos, check.Equals, int64(0))
 293         c.Check(err, check.IsNil)
 294         n, err = io.ReadFull(f, buf)
 295         c.Check(n, check.Equals, 18)
 296         c.Check(err, check.Equals, io.ErrUnexpectedEOF)
 297         c.Check(string(buf[:n]), check.Equals, "f0123456789abcdefg")
 298
 299         buf2, err := ioutil.ReadAll(f2)
 300         c.Check(err, check.IsNil)
 301         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 302
 303         // truncate to current size
 304         err = f.Truncate(18)
 305         c.Check(err, check.IsNil)
 306         f2.Seek(0, io.SeekStart)
 307         buf2, err = ioutil.ReadAll(f2)
 308         c.Check(err, check.IsNil)
 309         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 310
 311         // shrink to zero some data
 312         f.Truncate(15)
 313         f2.Seek(0, io.SeekStart)
 314         buf2, err = ioutil.ReadAll(f2)
 315         c.Check(err, check.IsNil)
 316         c.Check(string(buf2), check.Equals, "f0123456789abcd")
 317
 318         // grow to partial block/extent
 319         f.Truncate(20)
 320         f2.Seek(0, io.SeekStart)
 321         buf2, err = ioutil.ReadAll(f2)
 322         c.Check(err, check.IsNil)
 323         c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
 324
 325         f.Truncate(0)
 326         f2.Seek(0, io.SeekStart)
 327         f2.Write([]byte("12345678abcdefghijkl"))
 328
 329         // grow to block/extent boundary
 330         f.Truncate(64)
 331         f2.Seek(0, io.SeekStart)
 332         buf2, err = ioutil.ReadAll(f2)
 333         c.Check(err, check.IsNil)
 334         c.Check(len(buf2), check.Equals, 64)
 335         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 8)
 336
 337         // shrink to block/extent boundary
 338         err = f.Truncate(32)
 339         c.Check(err, check.IsNil)
 340         f2.Seek(0, io.SeekStart)
 341         buf2, err = ioutil.ReadAll(f2)
 342         c.Check(err, check.IsNil)
 343         c.Check(len(buf2), check.Equals, 32)
 344         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 4)
 345
 346         // shrink to partial block/extent
 347         err = f.Truncate(15)
 348         c.Check(err, check.IsNil)
 349         f2.Seek(0, io.SeekStart)
 350         buf2, err = ioutil.ReadAll(f2)
 351         c.Check(err, check.IsNil)
 352         c.Check(string(buf2), check.Equals, "12345678abcdefg")
 353         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 2)
 354
 355         // Force flush to ensure the block "12345678" gets stored, so
 356         // we know what to expect in the final manifest below.
 357         _, err = s.fs.MarshalManifest(".")
 358         c.Check(err, check.IsNil)
 359
 360         // Truncate to size=3 while f2's ptr is at 15
 361         err = f.Truncate(3)
 362         c.Check(err, check.IsNil)
 363         buf2, err = ioutil.ReadAll(f2)
 364         c.Check(err, check.IsNil)
 365         c.Check(string(buf2), check.Equals, "")
 366         f2.Seek(0, io.SeekStart)
 367         buf2, err = ioutil.ReadAll(f2)
 368         c.Check(err, check.IsNil)
 369         c.Check(string(buf2), check.Equals, "123")
 370         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 1)
 371
 372         m, err := s.fs.MarshalManifest(".")
 373         c.Check(err, check.IsNil)
 374         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 375         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 25d55ad283aa400af464c76d713c07ad+8 3:3:bar 6:3:foo\n")
 376         c.Check(s.fs.Size(), check.Equals, int64(6))
 377 }
 378
 379 func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
 380         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 381         c.Assert(err, check.IsNil)
 382         f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
 383         c.Assert(err, check.IsNil)
 384         defer f.Close()
 385
 386         checkSize := func(size int64) {
 387                 fi, err := f.Stat()
 388                 c.Assert(err, check.IsNil)
 389                 c.Check(fi.Size(), check.Equals, size)
 390
 391                 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
 392                 c.Assert(err, check.IsNil)
 393                 defer f.Close()
 394                 fi, err = f.Stat()
 395                 c.Check(err, check.IsNil)
 396                 c.Check(fi.Size(), check.Equals, size)
 397                 pos, err := f.Seek(0, io.SeekEnd)
 398                 c.Check(err, check.IsNil)
 399                 c.Check(pos, check.Equals, size)
 400         }
 401
 402         f.Seek(2, io.SeekEnd)
 403         checkSize(0)
 404         f.Write([]byte{1})
 405         checkSize(3)
 406
 407         f.Seek(2, io.SeekCurrent)
 408         checkSize(3)
 409         f.Write([]byte{})
 410         checkSize(5)
 411
 412         f.Seek(8, io.SeekStart)
 413         checkSize(5)
 414         n, err := f.Read(make([]byte, 1))
 415         c.Check(n, check.Equals, 0)
 416         c.Check(err, check.Equals, io.EOF)
 417         checkSize(5)
 418         f.Write([]byte{1, 2, 3})
 419         checkSize(11)
 420 }
 421
 422 func (s *CollectionFSSuite) TestMarshalCopiesRemoteBlocks(c *check.C) {
 423         foo := "foo"
 424         bar := "bar"
 425         hash := map[string]string{
 426                 foo: fmt.Sprintf("%x", md5.Sum([]byte(foo))),
 427                 bar: fmt.Sprintf("%x", md5.Sum([]byte(bar))),
 428         }
 429
 430         fs, err := (&Collection{
 431                 ManifestText: ". " + hash[foo] + "+3+Rzaaaa-foo@bab " + hash[bar] + "+3+A12345@ffffff 0:2:fo.txt 2:4:obar.txt\n",
 432         }).FileSystem(s.client, s.kc)
 433         c.Assert(err, check.IsNil)
 434         manifest, err := fs.MarshalManifest(".")
 435         c.Check(manifest, check.Equals, "")
 436         c.Check(err, check.NotNil)
 437
 438         s.kc.refreshable = map[string]bool{hash[bar]: true}
 439
 440         for _, sigIn := range []string{"Rzaaaa-foo@bab", "A12345@abcde"} {
 441                 fs, err = (&Collection{
 442                         ManifestText: ". " + hash[foo] + "+3+A12345@fffff " + hash[bar] + "+3+" + sigIn + " 0:2:fo.txt 2:4:obar.txt\n",
 443                 }).FileSystem(s.client, s.kc)
 444                 c.Assert(err, check.IsNil)
 445                 manifest, err := fs.MarshalManifest(".")
 446                 c.Check(err, check.IsNil)
 447                 // Both blocks should now have +A signatures.
 448                 c.Check(manifest, check.Matches, `.*\+A.* .*\+A.*\n`)
 449                 c.Check(manifest, check.Not(check.Matches), `.*\+R.*\n`)
 450         }
 451 }
 452
 453 func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
 454         maxBlockSize = 8
 455         defer func() { maxBlockSize = 2 << 26 }()
 456
 457         var err error
 458         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 459         c.Assert(err, check.IsNil)
 460         for _, name := range []string{"foo", "bar", "baz"} {
 461                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 462                 c.Assert(err, check.IsNil)
 463                 f.Write([]byte(name))
 464                 f.Close()
 465         }
 466
 467         m, err := s.fs.MarshalManifest(".")
 468         c.Check(err, check.IsNil)
 469         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 470         c.Check(m, check.Equals, ". c3c23db5285662ef7172373df0003206+6 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar 3:3:baz 6:3:foo\n")
 471 }
 472
 473 func (s *CollectionFSSuite) TestMkdir(c *check.C) {
 474         err := s.fs.Mkdir("foo/bar", 0755)
 475         c.Check(err, check.Equals, os.ErrNotExist)
 476
 477         f, err := s.fs.OpenFile("foo/bar", os.O_CREATE, 0)
 478         c.Check(err, check.Equals, os.ErrNotExist)
 479
 480         err = s.fs.Mkdir("foo", 0755)
 481         c.Check(err, check.IsNil)
 482
 483         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_WRONLY, 0)
 484         c.Check(err, check.IsNil)
 485         if err == nil {
 486                 defer f.Close()
 487                 f.Write([]byte("foo"))
 488         }
 489
 490         // mkdir fails if a file already exists with that name
 491         err = s.fs.Mkdir("foo/bar", 0755)
 492         c.Check(err, check.NotNil)
 493
 494         err = s.fs.Remove("foo/bar")
 495         c.Check(err, check.IsNil)
 496
 497         // mkdir succeeds after the file is deleted
 498         err = s.fs.Mkdir("foo/bar", 0755)
 499         c.Check(err, check.IsNil)
 500
 501         // creating a file in a nonexistent subdir should still fail
 502         f, err = s.fs.OpenFile("foo/bar/baz/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
 503         c.Check(err, check.Equals, os.ErrNotExist)
 504
 505         f, err = s.fs.OpenFile("foo/bar/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
 506         c.Check(err, check.IsNil)
 507         if err == nil {
 508                 defer f.Close()
 509                 f.Write([]byte("foo"))
 510         }
 511
 512         // creating foo/bar as a regular file should fail
 513         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, 0)
 514         c.Check(err, check.NotNil)
 515
 516         // creating foo/bar as a directory should fail
 517         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, os.ModeDir)
 518         c.Check(err, check.NotNil)
 519         err = s.fs.Mkdir("foo/bar", 0755)
 520         c.Check(err, check.NotNil)
 521
 522         m, err := s.fs.MarshalManifest(".")
 523         c.Check(err, check.IsNil)
 524         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 525         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 3:3:bar 0:3:foo\n./foo/bar acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 526 }
 527
 528 func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
 529         if testing.Short() {
 530                 c.Skip("slow")
 531         }
 532
 533         maxBlockSize = 8
 534         defer func() { maxBlockSize = 2 << 26 }()
 535
 536         var wg sync.WaitGroup
 537         for n := 0; n < 128; n++ {
 538                 wg.Add(1)
 539                 go func() {
 540                         defer wg.Done()
 541                         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 542                         c.Assert(err, check.IsNil)
 543                         defer f.Close()
 544                         for i := 0; i < 1024; i++ {
 545                                 r := rand.Uint32()
 546                                 switch {
 547                                 case r%11 == 0:
 548                                         _, err := s.fs.MarshalManifest(".")
 549                                         c.Check(err, check.IsNil)
 550                                 case r&3 == 0:
 551                                         f.Truncate(int64(rand.Intn(64)))
 552                                 case r&3 == 1:
 553                                         f.Seek(int64(rand.Intn(64)), io.SeekStart)
 554                                 case r&3 == 2:
 555                                         _, err := f.Write([]byte("beep boop"))
 556                                         c.Check(err, check.IsNil)
 557                                 case r&3 == 3:
 558                                         _, err := ioutil.ReadAll(f)
 559                                         c.Check(err, check.IsNil)
 560                                 }
 561                         }
 562                 }()
 563         }
 564         wg.Wait()
 565
 566         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 567         c.Assert(err, check.IsNil)
 568         defer f.Close()
 569         buf, err := ioutil.ReadAll(f)
 570         c.Check(err, check.IsNil)
 571         c.Logf("after lots of random r/w/seek/trunc, buf is %q", buf)
 572 }
 573
 574 func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
 575         maxBlockSize = 40
 576         defer func() { maxBlockSize = 2 << 26 }()
 577
 578         var err error
 579         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 580         c.Assert(err, check.IsNil)
 581
 582         const nfiles = 256
 583         const ngoroutines = 256
 584
 585         var wg sync.WaitGroup
 586         for n := 0; n < ngoroutines; n++ {
 587                 wg.Add(1)
 588                 go func(n int) {
 589                         defer wg.Done()
 590                         expect := make([]byte, 0, 64)
 591                         wbytes := []byte("there's no simple explanation for anything important that any of us do")
 592                         f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 593                         c.Assert(err, check.IsNil)
 594                         defer f.Close()
 595                         for i := 0; i < nfiles; i++ {
 596                                 trunc := rand.Intn(65)
 597                                 woff := rand.Intn(trunc + 1)
 598                                 wbytes = wbytes[:rand.Intn(64-woff+1)]
 599                                 for buf, i := expect[:cap(expect)], len(expect); i < trunc; i++ {
 600                                         buf[i] = 0
 601                                 }
 602                                 expect = expect[:trunc]
 603                                 if trunc < woff+len(wbytes) {
 604                                         expect = expect[:woff+len(wbytes)]
 605                                 }
 606                                 copy(expect[woff:], wbytes)
 607                                 f.Truncate(int64(trunc))
 608                                 pos, err := f.Seek(int64(woff), io.SeekStart)
 609                                 c.Check(pos, check.Equals, int64(woff))
 610                                 c.Check(err, check.IsNil)
 611                                 n, err := f.Write(wbytes)
 612                                 c.Check(n, check.Equals, len(wbytes))
 613                                 c.Check(err, check.IsNil)
 614                                 pos, err = f.Seek(0, io.SeekStart)
 615                                 c.Check(pos, check.Equals, int64(0))
 616                                 c.Check(err, check.IsNil)
 617                                 buf, err := ioutil.ReadAll(f)
 618                                 c.Check(string(buf), check.Equals, string(expect))
 619                                 c.Check(err, check.IsNil)
 620                         }
 621                         s.checkMemSize(c, f)
 622                 }(n)
 623         }
 624         wg.Wait()
 625
 626         root, err := s.fs.Open("/")
 627         c.Assert(err, check.IsNil)
 628         defer root.Close()
 629         fi, err := root.Readdir(-1)
 630         c.Check(err, check.IsNil)
 631         c.Check(len(fi), check.Equals, nfiles)
 632
 633         _, err = s.fs.MarshalManifest(".")
 634         c.Check(err, check.IsNil)
 635         // TODO: check manifest content
 636 }
 637
 638 func (s *CollectionFSSuite) TestRemove(c *check.C) {
 639         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 640         c.Assert(err, check.IsNil)
 641         err = fs.Mkdir("dir0", 0755)
 642         c.Assert(err, check.IsNil)
 643         err = fs.Mkdir("dir1", 0755)
 644         c.Assert(err, check.IsNil)
 645         err = fs.Mkdir("dir1/dir2", 0755)
 646         c.Assert(err, check.IsNil)
 647         err = fs.Mkdir("dir1/dir3", 0755)
 648         c.Assert(err, check.IsNil)
 649
 650         err = fs.Remove("dir0")
 651         c.Check(err, check.IsNil)
 652         err = fs.Remove("dir0")
 653         c.Check(err, check.Equals, os.ErrNotExist)
 654
 655         err = fs.Remove("dir1/dir2/.")
 656         c.Check(err, check.Equals, ErrInvalidArgument)
 657         err = fs.Remove("dir1/dir2/..")
 658         c.Check(err, check.Equals, ErrInvalidArgument)
 659         err = fs.Remove("dir1")
 660         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
 661         err = fs.Remove("dir1/dir2/../../../dir1")
 662         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
 663         err = fs.Remove("dir1/dir3/")
 664         c.Check(err, check.IsNil)
 665         err = fs.RemoveAll("dir1")
 666         c.Check(err, check.IsNil)
 667         err = fs.RemoveAll("dir1")
 668         c.Check(err, check.IsNil)
 669 }
 670
 671 func (s *CollectionFSSuite) TestRenameError(c *check.C) {
 672         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 673         c.Assert(err, check.IsNil)
 674         err = fs.Mkdir("first", 0755)
 675         c.Assert(err, check.IsNil)
 676         err = fs.Mkdir("first/second", 0755)
 677         c.Assert(err, check.IsNil)
 678         f, err := fs.OpenFile("first/second/file", os.O_CREATE|os.O_WRONLY, 0755)
 679         c.Assert(err, check.IsNil)
 680         f.Write([]byte{1, 2, 3, 4, 5})
 681         f.Close()
 682         err = fs.Rename("first", "first/second/third")
 683         c.Check(err, check.Equals, ErrInvalidArgument)
 684         err = fs.Rename("first", "first/third")
 685         c.Check(err, check.Equals, ErrInvalidArgument)
 686         err = fs.Rename("first/second", "second")
 687         c.Check(err, check.IsNil)
 688         f, err = fs.OpenFile("second/file", 0, 0)
 689         c.Assert(err, check.IsNil)
 690         data, err := ioutil.ReadAll(f)
 691         c.Check(err, check.IsNil)
 692         c.Check(data, check.DeepEquals, []byte{1, 2, 3, 4, 5})
 693 }
 694
 695 func (s *CollectionFSSuite) TestRenameDirectory(c *check.C) {
 696         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 697         c.Assert(err, check.IsNil)
 698         err = fs.Mkdir("foo", 0755)
 699         c.Assert(err, check.IsNil)
 700         err = fs.Mkdir("bar", 0755)
 701         c.Assert(err, check.IsNil)
 702         err = fs.Rename("bar", "baz")
 703         c.Check(err, check.IsNil)
 704         err = fs.Rename("foo", "baz")
 705         c.Check(err, check.NotNil)
 706         err = fs.Rename("foo", "baz/")
 707         c.Check(err, check.IsNil)
 708         err = fs.Rename("baz/foo", ".")
 709         c.Check(err, check.Equals, ErrInvalidArgument)
 710         err = fs.Rename("baz/foo/", ".")
 711         c.Check(err, check.Equals, ErrInvalidArgument)
 712 }
 713
 714 func (s *CollectionFSSuite) TestRename(c *check.C) {
 715         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 716         c.Assert(err, check.IsNil)
 717         const (
 718                 outer = 16
 719                 inner = 16
 720         )
 721         for i := 0; i < outer; i++ {
 722                 err = fs.Mkdir(fmt.Sprintf("dir%d", i), 0755)
 723                 c.Assert(err, check.IsNil)
 724                 for j := 0; j < inner; j++ {
 725                         err = fs.Mkdir(fmt.Sprintf("dir%d/dir%d", i, j), 0755)
 726                         c.Assert(err, check.IsNil)
 727                         for _, fnm := range []string{
 728                                 fmt.Sprintf("dir%d/file%d", i, j),
 729                                 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
 730                         } {
 731                                 f, err := fs.OpenFile(fnm, os.O_CREATE|os.O_WRONLY, 0755)
 732                                 c.Assert(err, check.IsNil)
 733                                 _, err = f.Write([]byte("beep"))
 734                                 c.Assert(err, check.IsNil)
 735                                 f.Close()
 736                         }
 737                 }
 738         }
 739         var wg sync.WaitGroup
 740         for i := 0; i < outer; i++ {
 741                 for j := 0; j < inner; j++ {
 742                         wg.Add(1)
 743                         go func(i, j int) {
 744                                 defer wg.Done()
 745                                 oldname := fmt.Sprintf("dir%d/dir%d/file%d", i, j, j)
 746                                 newname := fmt.Sprintf("dir%d/newfile%d", i, inner-j-1)
 747                                 _, err := fs.Open(newname)
 748                                 c.Check(err, check.Equals, os.ErrNotExist)
 749                                 err = fs.Rename(oldname, newname)
 750                                 c.Check(err, check.IsNil)
 751                                 f, err := fs.Open(newname)
 752                                 c.Check(err, check.IsNil)
 753                                 f.Close()
 754                         }(i, j)
 755
 756                         wg.Add(1)
 757                         go func(i, j int) {
 758                                 defer wg.Done()
 759                                 // oldname does not exist
 760                                 err := fs.Rename(
 761                                         fmt.Sprintf("dir%d/dir%d/missing", i, j),
 762                                         fmt.Sprintf("dir%d/dir%d/file%d", outer-i-1, j, j))
 763                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 764
 765                                 // newname parent dir does not exist
 766                                 err = fs.Rename(
 767                                         fmt.Sprintf("dir%d/dir%d", i, j),
 768                                         fmt.Sprintf("dir%d/missing/irrelevant", outer-i-1))
 769                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 770
 771                                 // oldname parent dir is a file
 772                                 err = fs.Rename(
 773                                         fmt.Sprintf("dir%d/file%d/patherror", i, j),
 774                                         fmt.Sprintf("dir%d/irrelevant", i))
 775                                 c.Check(err, check.ErrorMatches, `.*not a directory`)
 776
 777                                 // newname parent dir is a file
 778                                 err = fs.Rename(
 779                                         fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
 780                                         fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
 781                                 c.Check(err, check.ErrorMatches, `.*not a directory`)
 782                         }(i, j)
 783                 }
 784         }
 785         wg.Wait()
 786
 787         f, err := fs.OpenFile("dir1/newfile3", 0, 0)
 788         c.Assert(err, check.IsNil)
 789         c.Check(f.Size(), check.Equals, int64(4))
 790         buf, err := ioutil.ReadAll(f)
 791         c.Check(buf, check.DeepEquals, []byte("beep"))
 792         c.Check(err, check.IsNil)
 793         _, err = fs.Open("dir1/dir1/file1")
 794         c.Check(err, check.Equals, os.ErrNotExist)
 795 }
 796
 797 func (s *CollectionFSSuite) TestPersist(c *check.C) {
 798         maxBlockSize = 1024
 799         defer func() { maxBlockSize = 2 << 26 }()
 800
 801         var err error
 802         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 803         c.Assert(err, check.IsNil)
 804         err = s.fs.Mkdir("d:r", 0755)
 805         c.Assert(err, check.IsNil)
 806
 807         expect := map[string][]byte{}
 808
 809         var wg sync.WaitGroup
 810         for _, name := range []string{"random 1", "random:2", "random\\3", "d:r/random4"} {
 811                 buf := make([]byte, 500)
 812                 rand.Read(buf)
 813                 expect[name] = buf
 814
 815                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 816                 c.Assert(err, check.IsNil)
 817                 // Note: we don't close the file until after the test
 818                 // is done. Writes to unclosed files should persist.
 819                 defer f.Close()
 820
 821                 wg.Add(1)
 822                 go func() {
 823                         defer wg.Done()
 824                         for i := 0; i < len(buf); i += 5 {
 825                                 _, err := f.Write(buf[i : i+5])
 826                                 c.Assert(err, check.IsNil)
 827                         }
 828                 }()
 829         }
 830         wg.Wait()
 831
 832         m, err := s.fs.MarshalManifest(".")
 833         c.Check(err, check.IsNil)
 834         c.Logf("%q", m)
 835
 836         root, err := s.fs.Open("/")
 837         c.Assert(err, check.IsNil)
 838         defer root.Close()
 839         fi, err := root.Readdir(-1)
 840         c.Check(err, check.IsNil)
 841         c.Check(len(fi), check.Equals, 4)
 842
 843         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
 844         c.Assert(err, check.IsNil)
 845
 846         root, err = persisted.Open("/")
 847         c.Assert(err, check.IsNil)
 848         defer root.Close()
 849         fi, err = root.Readdir(-1)
 850         c.Check(err, check.IsNil)
 851         c.Check(len(fi), check.Equals, 4)
 852
 853         for name, content := range expect {
 854                 c.Logf("read %q", name)
 855                 f, err := persisted.Open(name)
 856                 c.Assert(err, check.IsNil)
 857                 defer f.Close()
 858                 buf, err := ioutil.ReadAll(f)
 859                 c.Check(err, check.IsNil)
 860                 c.Check(buf, check.DeepEquals, content)
 861         }
 862 }
 863
 864 func (s *CollectionFSSuite) TestPersistEmptyFilesAndDirs(c *check.C) {
 865         var err error
 866         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 867         c.Assert(err, check.IsNil)
 868         for _, name := range []string{"dir", "dir/zerodir", "empty", "not empty", "not empty/empty", "zero", "zero/zero"} {
 869                 err = s.fs.Mkdir(name, 0755)
 870                 c.Assert(err, check.IsNil)
 871         }
 872
 873         expect := map[string][]byte{
 874                 "0":                nil,
 875                 "00":               {},
 876                 "one":              {1},
 877                 "dir/0":            nil,
 878                 "dir/two":          {1, 2},
 879                 "dir/zero":         nil,
 880                 "dir/zerodir/zero": nil,
 881                 "zero/zero/zero":   nil,
 882         }
 883         for name, data := range expect {
 884                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 885                 c.Assert(err, check.IsNil)
 886                 if data != nil {
 887                         _, err := f.Write(data)
 888                         c.Assert(err, check.IsNil)
 889                 }
 890                 f.Close()
 891         }
 892
 893         m, err := s.fs.MarshalManifest(".")
 894         c.Check(err, check.IsNil)
 895         c.Logf("%q", m)
 896
 897         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
 898         c.Assert(err, check.IsNil)
 899
 900         for name, data := range expect {
 901                 _, err = persisted.Open("bogus-" + name)
 902                 c.Check(err, check.NotNil)
 903
 904                 f, err := persisted.Open(name)
 905                 c.Assert(err, check.IsNil)
 906
 907                 if data == nil {
 908                         data = []byte{}
 909                 }
 910                 buf, err := ioutil.ReadAll(f)
 911                 c.Check(err, check.IsNil)
 912                 c.Check(buf, check.DeepEquals, data)
 913         }
 914
 915         expectDir := map[string]int{
 916                 "empty":           0,
 917                 "not empty":       1,
 918                 "not empty/empty": 0,
 919         }
 920         for name, expectLen := range expectDir {
 921                 _, err := persisted.Open(name + "/bogus")
 922                 c.Check(err, check.NotNil)
 923
 924                 d, err := persisted.Open(name)
 925                 defer d.Close()
 926                 c.Check(err, check.IsNil)
 927                 fi, err := d.Readdir(-1)
 928                 c.Check(err, check.IsNil)
 929                 c.Check(fi, check.HasLen, expectLen)
 930         }
 931 }
 932
 933 func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
 934         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 935         c.Assert(err, check.IsNil)
 936
 937         f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
 938         c.Check(f, check.IsNil)
 939         c.Check(err, check.ErrorMatches, `file does not exist`)
 940
 941         f, err = fs.OpenFile("new", os.O_CREATE|os.O_RDONLY, 0)
 942         c.Assert(err, check.IsNil)
 943         defer f.Close()
 944         n, err := f.Write([]byte{1, 2, 3})
 945         c.Check(n, check.Equals, 0)
 946         c.Check(err, check.ErrorMatches, `read-only file`)
 947         n, err = f.Read(make([]byte, 1))
 948         c.Check(n, check.Equals, 0)
 949         c.Check(err, check.Equals, io.EOF)
 950         f, err = fs.OpenFile("new", os.O_RDWR, 0)
 951         c.Assert(err, check.IsNil)
 952         defer f.Close()
 953         _, err = f.Write([]byte{4, 5, 6})
 954         c.Check(err, check.IsNil)
 955         fi, err := f.Stat()
 956         c.Assert(err, check.IsNil)
 957         c.Check(fi.Size(), check.Equals, int64(3))
 958
 959         f, err = fs.OpenFile("new", os.O_TRUNC|os.O_RDWR, 0)
 960         c.Assert(err, check.IsNil)
 961         defer f.Close()
 962         pos, err := f.Seek(0, io.SeekEnd)
 963         c.Check(pos, check.Equals, int64(0))
 964         c.Check(err, check.IsNil)
 965         fi, err = f.Stat()
 966         c.Assert(err, check.IsNil)
 967         c.Check(fi.Size(), check.Equals, int64(0))
 968         fs.Remove("new")
 969
 970         buf := make([]byte, 64)
 971         f, err = fs.OpenFile("append", os.O_EXCL|os.O_CREATE|os.O_RDWR|os.O_APPEND, 0)
 972         c.Assert(err, check.IsNil)
 973         f.Write([]byte{1, 2, 3})
 974         f.Seek(0, io.SeekStart)
 975         n, _ = f.Read(buf[:1])
 976         c.Check(n, check.Equals, 1)
 977         c.Check(buf[:1], check.DeepEquals, []byte{1})
 978         pos, err = f.Seek(0, io.SeekCurrent)
 979         c.Assert(err, check.IsNil)
 980         c.Check(pos, check.Equals, int64(1))
 981         f.Write([]byte{4, 5, 6})
 982         pos, err = f.Seek(0, io.SeekCurrent)
 983         c.Assert(err, check.IsNil)
 984         c.Check(pos, check.Equals, int64(6))
 985         f.Seek(0, io.SeekStart)
 986         n, err = f.Read(buf)
 987         c.Check(buf[:n], check.DeepEquals, []byte{1, 2, 3, 4, 5, 6})
 988         c.Check(err, check.Equals, io.EOF)
 989         f.Close()
 990
 991         f, err = fs.OpenFile("append", os.O_RDWR|os.O_APPEND, 0)
 992         c.Assert(err, check.IsNil)
 993         pos, err = f.Seek(0, io.SeekCurrent)
 994         c.Check(pos, check.Equals, int64(0))
 995         c.Check(err, check.IsNil)
 996         f.Read(buf[:3])
 997         pos, _ = f.Seek(0, io.SeekCurrent)
 998         c.Check(pos, check.Equals, int64(3))
 999         f.Write([]byte{7, 8, 9})
1000         pos, err = f.Seek(0, io.SeekCurrent)
1001         c.Check(err, check.IsNil)
1002         c.Check(pos, check.Equals, int64(9))
1003         f.Close()
1004
1005         f, err = fs.OpenFile("wronly", os.O_CREATE|os.O_WRONLY, 0)
1006         c.Assert(err, check.IsNil)
1007         n, err = f.Write([]byte{3, 2, 1})
1008         c.Check(n, check.Equals, 3)
1009         c.Check(err, check.IsNil)
1010         pos, _ = f.Seek(0, io.SeekCurrent)
1011         c.Check(pos, check.Equals, int64(3))
1012         pos, _ = f.Seek(0, io.SeekStart)
1013         c.Check(pos, check.Equals, int64(0))
1014         n, err = f.Read(buf)
1015         c.Check(n, check.Equals, 0)
1016         c.Check(err, check.ErrorMatches, `.*O_WRONLY.*`)
1017         f, err = fs.OpenFile("wronly", os.O_RDONLY, 0)
1018         c.Assert(err, check.IsNil)
1019         n, _ = f.Read(buf)
1020         c.Check(buf[:n], check.DeepEquals, []byte{3, 2, 1})
1021
1022         f, err = fs.OpenFile("unsupported", os.O_CREATE|os.O_SYNC, 0)
1023         c.Check(f, check.IsNil)
1024         c.Check(err, check.NotNil)
1025
1026         f, err = fs.OpenFile("append", os.O_RDWR|os.O_WRONLY, 0)
1027         c.Check(f, check.IsNil)
1028         c.Check(err, check.ErrorMatches, `invalid flag.*`)
1029 }
1030
1031 func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
1032         maxBlockSize = 1024
1033         defer func() { maxBlockSize = 2 << 26 }()
1034
1035         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
1036         c.Assert(err, check.IsNil)
1037         f, err := fs.OpenFile("50K", os.O_WRONLY|os.O_CREATE, 0)
1038         c.Assert(err, check.IsNil)
1039         defer f.Close()
1040
1041         data := make([]byte, 500)
1042         rand.Read(data)
1043
1044         for i := 0; i < 100; i++ {
1045                 n, err := f.Write(data)
1046                 c.Assert(n, check.Equals, len(data))
1047                 c.Assert(err, check.IsNil)
1048         }
1049
1050         currentMemExtents := func() (memExtents []int) {
1051                 for idx, e := range f.(*filehandle).inode.(*filenode).segments {
1052                         switch e.(type) {
1053                         case *memSegment:
1054                                 memExtents = append(memExtents, idx)
1055                         }
1056                 }
1057                 return
1058         }
1059
1060         c.Check(len(currentMemExtents()) <= concurrentWriters, check.Equals, true)
1061         f.(*filehandle).inode.(*filenode).waitPrune()
1062         c.Check(currentMemExtents(), check.HasLen, 1)
1063
1064         m, err := fs.MarshalManifest(".")
1065         c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
1066         c.Check(err, check.IsNil)
1067         c.Check(currentMemExtents(), check.HasLen, 0)
1068 }
1069
1070 func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
1071         for _, txt := range []string{
1072                 "\n",
1073                 ".\n",
1074                 ". \n",
1075                 ". d41d8cd98f00b204e9800998ecf8427e+0\n",
1076                 ". d41d8cd98f00b204e9800998ecf8427e+0 \n",
1077                 ". 0:0:foo\n",
1078                 ".  0:0:foo\n",
1079                 ". 0:0:foo 0:0:bar\n",
1080                 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo\n",
1081                 ". d41d8cd98f00b204e9800998ecf8427e+0 :0:0:foo\n",
1082                 ". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
1083                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
1084                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
1085                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:\\056\n",
1086                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:\\056\\057\\056\n",
1087                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:.\n",
1088                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:..\n",
1089                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:..\n",
1090                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/..\n",
1091                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n",
1092                 "./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
1093         } {
1094                 c.Logf("<-%q", txt)
1095                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
1096                 c.Check(fs, check.IsNil)
1097                 c.Logf("-> %s", err)
1098                 c.Check(err, check.NotNil)
1099         }
1100 }
1101
1102 func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
1103         for _, txt := range []string{
1104                 "",
1105                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
1106                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:...\n",
1107                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:. 0:0:. 0:0:\\056 0:0:\\056\n",
1108                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/. 0:0:. 0:0:foo\\057bar\\057\\056\n",
1109                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
1110                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
1111         } {
1112                 c.Logf("<-%q", txt)
1113                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
1114                 c.Check(err, check.IsNil)
1115                 c.Check(fs, check.NotNil)
1116         }
1117 }
1118
1119 func (s *CollectionFSSuite) checkMemSize(c *check.C, f File) {
1120         fn := f.(*filehandle).inode.(*filenode)
1121         var memsize int64
1122         for _, seg := range fn.segments {
1123                 if e, ok := seg.(*memSegment); ok {
1124                         memsize += int64(len(e.buf))
1125                 }
1126         }
1127         c.Check(fn.memsize, check.Equals, memsize)
1128 }
1129
1130 type CollectionFSUnitSuite struct{}
1131
1132 var _ = check.Suite(&CollectionFSUnitSuite{})
1133
1134 // expect ~2 seconds to load a manifest with 256K files
1135 func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
1136         if testing.Short() {
1137                 c.Skip("slow")
1138         }
1139
1140         const (
1141                 dirCount  = 512
1142                 fileCount = 512
1143         )
1144
1145         mb := bytes.NewBuffer(make([]byte, 0, 40000000))
1146         for i := 0; i < dirCount; i++ {
1147                 fmt.Fprintf(mb, "./dir%d", i)
1148                 for j := 0; j <= fileCount; j++ {
1149                         fmt.Fprintf(mb, " %032x+42+A%040x@%08x", j, j, j)
1150                 }
1151                 for j := 0; j < fileCount; j++ {
1152                         fmt.Fprintf(mb, " %d:%d:dir%d/file%d", j*42+21, 42, j, j)
1153                 }
1154                 mb.Write([]byte{'\n'})
1155         }
1156         coll := Collection{ManifestText: mb.String()}
1157         c.Logf("%s built", time.Now())
1158
1159         var memstats runtime.MemStats
1160         runtime.ReadMemStats(&memstats)
1161         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1162
1163         f, err := coll.FileSystem(nil, nil)
1164         c.Check(err, check.IsNil)
1165         c.Logf("%s loaded", time.Now())
1166         c.Check(f.Size(), check.Equals, int64(42*dirCount*fileCount))
1167
1168         for i := 0; i < dirCount; i++ {
1169                 for j := 0; j < fileCount; j++ {
1170                         f.Stat(fmt.Sprintf("./dir%d/dir%d/file%d", i, j, j))
1171                 }
1172         }
1173         c.Logf("%s Stat() x %d", time.Now(), dirCount*fileCount)
1174
1175         runtime.ReadMemStats(&memstats)
1176         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1177 }
1178
1179 // Gocheck boilerplate
1180 func Test(t *testing.T) {
1181         check.TestingT(t)
1182 }