sdk/go/arvados/collection_fs_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "math/rand"
  15         "net/http"
  16         "os"
  17         "regexp"
  18         "runtime"
  19         "sync"
  20         "testing"
  21         "time"
  22
  23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  24         check "gopkg.in/check.v1"
  25 )
  26
  27 var _ = check.Suite(&CollectionFSSuite{})
  28
  29 type keepClientStub struct {
  30         blocks map[string][]byte
  31         sync.RWMutex
  32 }
  33
  34 var errStub404 = errors.New("404 block not found")
  35
  36 func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error) {
  37         kcs.RLock()
  38         defer kcs.RUnlock()
  39         buf := kcs.blocks[locator[:32]]
  40         if buf == nil {
  41                 return 0, errStub404
  42         }
  43         return copy(p, buf[off:]), nil
  44 }
  45
  46 func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
  47         locator := fmt.Sprintf("%x+%d+A12345@abcde", md5.Sum(p), len(p))
  48         buf := make([]byte, len(p))
  49         copy(buf, p)
  50         kcs.Lock()
  51         defer kcs.Unlock()
  52         kcs.blocks[locator[:32]] = buf
  53         return locator, 1, nil
  54 }
  55
  56 type CollectionFSSuite struct {
  57         client *Client
  58         coll   Collection
  59         fs     CollectionFileSystem
  60         kc     keepClient
  61 }
  62
  63 func (s *CollectionFSSuite) SetUpTest(c *check.C) {
  64         s.client = NewClientFromEnv()
  65         err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
  66         c.Assert(err, check.IsNil)
  67         s.kc = &keepClientStub{
  68                 blocks: map[string][]byte{
  69                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  70                 }}
  71         s.fs, err = s.coll.FileSystem(s.client, s.kc)
  72         c.Assert(err, check.IsNil)
  73 }
  74
  75 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
  76         _, ok := s.fs.(http.FileSystem)
  77         c.Check(ok, check.Equals, true)
  78 }
  79
  80 func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
  81         fs, err := (&Collection{
  82                 ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
  83         }).FileSystem(s.client, s.kc)
  84         c.Assert(err, check.IsNil)
  85
  86         f, err := fs.Open("/foo:foo")
  87         c.Assert(err, check.IsNil)
  88
  89         fis, err := f.Readdir(0)
  90         c.Check(err, check.IsNil)
  91         c.Check(len(fis), check.Equals, 1)
  92         c.Check(fis[0].Name(), check.Equals, "bar:bar")
  93 }
  94
  95 func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
  96         f, err := s.fs.Open("/dir1")
  97         c.Assert(err, check.IsNil)
  98
  99         st, err := f.Stat()
 100         c.Assert(err, check.IsNil)
 101         c.Check(st.Size(), check.Equals, int64(2))
 102         c.Check(st.IsDir(), check.Equals, true)
 103
 104         fis, err := f.Readdir(0)
 105         c.Check(err, check.IsNil)
 106         c.Check(len(fis), check.Equals, 2)
 107         if len(fis) > 0 {
 108                 c.Check(fis[0].Size(), check.Equals, int64(3))
 109         }
 110 }
 111
 112 func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
 113         f, err := s.fs.Open("./dir1")
 114         c.Assert(err, check.IsNil)
 115
 116         fis, err := f.Readdir(1)
 117         c.Check(err, check.IsNil)
 118         c.Check(len(fis), check.Equals, 1)
 119         if len(fis) > 0 {
 120                 c.Check(fis[0].Size(), check.Equals, int64(3))
 121         }
 122
 123         fis, err = f.Readdir(1)
 124         c.Check(err, check.IsNil)
 125         c.Check(len(fis), check.Equals, 1)
 126         if len(fis) > 0 {
 127                 c.Check(fis[0].Size(), check.Equals, int64(3))
 128         }
 129
 130         fis, err = f.Readdir(1)
 131         c.Check(len(fis), check.Equals, 0)
 132         c.Check(err, check.NotNil)
 133         c.Check(err, check.Equals, io.EOF)
 134
 135         f, err = s.fs.Open("dir1")
 136         c.Assert(err, check.IsNil)
 137         fis, err = f.Readdir(1)
 138         c.Check(len(fis), check.Equals, 1)
 139         c.Assert(err, check.IsNil)
 140         fis, err = f.Readdir(2)
 141         c.Check(len(fis), check.Equals, 1)
 142         c.Assert(err, check.IsNil)
 143         fis, err = f.Readdir(2)
 144         c.Check(len(fis), check.Equals, 0)
 145         c.Assert(err, check.Equals, io.EOF)
 146 }
 147
 148 func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
 149         for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
 150                 f, err := s.fs.Open(path)
 151                 c.Assert(err, check.IsNil)
 152
 153                 st, err := f.Stat()
 154                 c.Assert(err, check.IsNil)
 155                 c.Check(st.Size(), check.Equals, int64(1))
 156                 c.Check(st.IsDir(), check.Equals, true)
 157         }
 158         for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
 159                 c.Logf("%q", path)
 160                 f, err := s.fs.Open(path)
 161                 c.Assert(err, check.IsNil)
 162
 163                 st, err := f.Stat()
 164                 c.Assert(err, check.IsNil)
 165                 c.Check(st.Size(), check.Equals, int64(2))
 166                 c.Check(st.IsDir(), check.Equals, true)
 167         }
 168 }
 169
 170 func (s *CollectionFSSuite) TestNotExist(c *check.C) {
 171         for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
 172                 f, err := s.fs.Open(path)
 173                 c.Assert(f, check.IsNil)
 174                 c.Assert(err, check.NotNil)
 175                 c.Assert(os.IsNotExist(err), check.Equals, true)
 176         }
 177 }
 178
 179 func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
 180         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDONLY, 0)
 181         c.Assert(err, check.IsNil)
 182         st, err := f.Stat()
 183         c.Assert(err, check.IsNil)
 184         c.Check(st.Size(), check.Equals, int64(3))
 185         n, err := f.Write([]byte("bar"))
 186         c.Check(n, check.Equals, 0)
 187         c.Check(err, check.Equals, ErrReadOnlyFile)
 188 }
 189
 190 func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
 191         f, err := s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE, 0)
 192         c.Assert(err, check.IsNil)
 193         st, err := f.Stat()
 194         c.Assert(err, check.IsNil)
 195         c.Check(st.Size(), check.Equals, int64(0))
 196
 197         n, err := f.Write([]byte("bar"))
 198         c.Check(n, check.Equals, 3)
 199         c.Check(err, check.IsNil)
 200
 201         c.Check(f.Close(), check.IsNil)
 202
 203         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 204         c.Check(f, check.IsNil)
 205         c.Assert(err, check.NotNil)
 206
 207         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR, 0)
 208         c.Assert(err, check.IsNil)
 209         st, err = f.Stat()
 210         c.Assert(err, check.IsNil)
 211         c.Check(st.Size(), check.Equals, int64(3))
 212
 213         c.Check(f.Close(), check.IsNil)
 214
 215         m, err := s.fs.MarshalManifest(".")
 216         c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
 217 }
 218
 219 func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
 220         maxBlockSize = 8
 221         defer func() { maxBlockSize = 2 << 26 }()
 222
 223         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 224         c.Assert(err, check.IsNil)
 225         defer f.Close()
 226         st, err := f.Stat()
 227         c.Assert(err, check.IsNil)
 228         c.Check(st.Size(), check.Equals, int64(3))
 229
 230         f2, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 231         c.Assert(err, check.IsNil)
 232         defer f2.Close()
 233
 234         buf := make([]byte, 64)
 235         n, err := f.Read(buf)
 236         c.Check(n, check.Equals, 3)
 237         c.Check(err, check.Equals, io.EOF)
 238         c.Check(string(buf[:3]), check.DeepEquals, "foo")
 239
 240         pos, err := f.Seek(-2, io.SeekCurrent)
 241         c.Check(pos, check.Equals, int64(1))
 242         c.Check(err, check.IsNil)
 243
 244         // Split a storedExtent in two, and insert a memExtent
 245         n, err = f.Write([]byte("*"))
 246         c.Check(n, check.Equals, 1)
 247         c.Check(err, check.IsNil)
 248
 249         pos, err = f.Seek(0, io.SeekCurrent)
 250         c.Check(pos, check.Equals, int64(2))
 251         c.Check(err, check.IsNil)
 252
 253         pos, err = f.Seek(0, io.SeekStart)
 254         c.Check(pos, check.Equals, int64(0))
 255         c.Check(err, check.IsNil)
 256
 257         rbuf, err := ioutil.ReadAll(f)
 258         c.Check(len(rbuf), check.Equals, 3)
 259         c.Check(err, check.IsNil)
 260         c.Check(string(rbuf), check.Equals, "f*o")
 261
 262         // Write multiple blocks in one call
 263         f.Seek(1, io.SeekStart)
 264         n, err = f.Write([]byte("0123456789abcdefg"))
 265         c.Check(n, check.Equals, 17)
 266         c.Check(err, check.IsNil)
 267         pos, err = f.Seek(0, io.SeekCurrent)
 268         c.Check(pos, check.Equals, int64(18))
 269         pos, err = f.Seek(-18, io.SeekCurrent)
 270         c.Check(err, check.IsNil)
 271         n, err = io.ReadFull(f, buf)
 272         c.Check(n, check.Equals, 18)
 273         c.Check(err, check.Equals, io.ErrUnexpectedEOF)
 274         c.Check(string(buf[:n]), check.Equals, "f0123456789abcdefg")
 275
 276         buf2, err := ioutil.ReadAll(f2)
 277         c.Check(err, check.IsNil)
 278         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 279
 280         // truncate to current size
 281         err = f.Truncate(18)
 282         f2.Seek(0, io.SeekStart)
 283         buf2, err = ioutil.ReadAll(f2)
 284         c.Check(err, check.IsNil)
 285         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 286
 287         // shrink to zero some data
 288         f.Truncate(15)
 289         f2.Seek(0, io.SeekStart)
 290         buf2, err = ioutil.ReadAll(f2)
 291         c.Check(err, check.IsNil)
 292         c.Check(string(buf2), check.Equals, "f0123456789abcd")
 293
 294         // grow to partial block/extent
 295         f.Truncate(20)
 296         f2.Seek(0, io.SeekStart)
 297         buf2, err = ioutil.ReadAll(f2)
 298         c.Check(err, check.IsNil)
 299         c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
 300
 301         f.Truncate(0)
 302         f2.Seek(0, io.SeekStart)
 303         f2.Write([]byte("12345678abcdefghijkl"))
 304
 305         // grow to block/extent boundary
 306         f.Truncate(64)
 307         f2.Seek(0, io.SeekStart)
 308         buf2, err = ioutil.ReadAll(f2)
 309         c.Check(err, check.IsNil)
 310         c.Check(len(buf2), check.Equals, 64)
 311         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 8)
 312
 313         // shrink to block/extent boundary
 314         err = f.Truncate(32)
 315         f2.Seek(0, io.SeekStart)
 316         buf2, err = ioutil.ReadAll(f2)
 317         c.Check(err, check.IsNil)
 318         c.Check(len(buf2), check.Equals, 32)
 319         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 4)
 320
 321         // shrink to partial block/extent
 322         err = f.Truncate(15)
 323         f2.Seek(0, io.SeekStart)
 324         buf2, err = ioutil.ReadAll(f2)
 325         c.Check(err, check.IsNil)
 326         c.Check(string(buf2), check.Equals, "12345678abcdefg")
 327         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 2)
 328
 329         // Force flush to ensure the block "12345678" gets stored, so
 330         // we know what to expect in the final manifest below.
 331         _, err = s.fs.MarshalManifest(".")
 332         c.Check(err, check.IsNil)
 333
 334         // Truncate to size=3 while f2's ptr is at 15
 335         err = f.Truncate(3)
 336         c.Check(err, check.IsNil)
 337         buf2, err = ioutil.ReadAll(f2)
 338         c.Check(err, check.IsNil)
 339         c.Check(string(buf2), check.Equals, "")
 340         f2.Seek(0, io.SeekStart)
 341         buf2, err = ioutil.ReadAll(f2)
 342         c.Check(err, check.IsNil)
 343         c.Check(string(buf2), check.Equals, "123")
 344         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 1)
 345
 346         m, err := s.fs.MarshalManifest(".")
 347         c.Check(err, check.IsNil)
 348         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 349         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 25d55ad283aa400af464c76d713c07ad+8 3:3:bar 6:3:foo\n")
 350 }
 351
 352 func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
 353         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 354         c.Assert(err, check.IsNil)
 355         f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
 356         c.Assert(err, check.IsNil)
 357         defer f.Close()
 358
 359         checkSize := func(size int64) {
 360                 fi, err := f.Stat()
 361                 c.Check(fi.Size(), check.Equals, size)
 362
 363                 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
 364                 c.Assert(err, check.IsNil)
 365                 defer f.Close()
 366                 fi, err = f.Stat()
 367                 c.Check(fi.Size(), check.Equals, size)
 368                 pos, err := f.Seek(0, io.SeekEnd)
 369                 c.Check(pos, check.Equals, size)
 370         }
 371
 372         f.Seek(2, io.SeekEnd)
 373         checkSize(0)
 374         f.Write([]byte{1})
 375         checkSize(3)
 376
 377         f.Seek(2, io.SeekCurrent)
 378         checkSize(3)
 379         f.Write([]byte{})
 380         checkSize(5)
 381
 382         f.Seek(8, io.SeekStart)
 383         checkSize(5)
 384         n, err := f.Read(make([]byte, 1))
 385         c.Check(n, check.Equals, 0)
 386         c.Check(err, check.Equals, io.EOF)
 387         checkSize(5)
 388         f.Write([]byte{1, 2, 3})
 389         checkSize(11)
 390 }
 391
 392 func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
 393         maxBlockSize = 8
 394         defer func() { maxBlockSize = 2 << 26 }()
 395
 396         var err error
 397         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 398         c.Assert(err, check.IsNil)
 399         for _, name := range []string{"foo", "bar", "baz"} {
 400                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 401                 c.Assert(err, check.IsNil)
 402                 f.Write([]byte(name))
 403                 f.Close()
 404         }
 405
 406         m, err := s.fs.MarshalManifest(".")
 407         c.Check(err, check.IsNil)
 408         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 409         c.Check(m, check.Equals, ". c3c23db5285662ef7172373df0003206+6 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar 3:3:baz 6:3:foo\n")
 410 }
 411
 412 func (s *CollectionFSSuite) TestMkdir(c *check.C) {
 413         err := s.fs.Mkdir("foo/bar", 0755)
 414         c.Check(err, check.Equals, os.ErrNotExist)
 415
 416         f, err := s.fs.OpenFile("foo/bar", os.O_CREATE, 0)
 417         c.Check(err, check.Equals, os.ErrNotExist)
 418
 419         err = s.fs.Mkdir("foo", 0755)
 420         c.Check(err, check.IsNil)
 421
 422         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_WRONLY, 0)
 423         c.Check(err, check.IsNil)
 424         if err == nil {
 425                 defer f.Close()
 426                 f.Write([]byte("foo"))
 427         }
 428
 429         // mkdir fails if a file already exists with that name
 430         err = s.fs.Mkdir("foo/bar", 0755)
 431         c.Check(err, check.NotNil)
 432
 433         err = s.fs.Remove("foo/bar")
 434         c.Check(err, check.IsNil)
 435
 436         // mkdir succeds after the file is deleted
 437         err = s.fs.Mkdir("foo/bar", 0755)
 438         c.Check(err, check.IsNil)
 439
 440         // creating a file in a nonexistent subdir should still fail
 441         f, err = s.fs.OpenFile("foo/bar/baz/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
 442         c.Check(err, check.Equals, os.ErrNotExist)
 443
 444         f, err = s.fs.OpenFile("foo/bar/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
 445         c.Check(err, check.IsNil)
 446         if err == nil {
 447                 defer f.Close()
 448                 f.Write([]byte("foo"))
 449         }
 450
 451         // creating foo/bar as a regular file should fail
 452         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, 0)
 453         c.Check(err, check.NotNil)
 454
 455         // creating foo/bar as a directory should fail
 456         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, os.ModeDir)
 457         c.Check(err, check.NotNil)
 458         err = s.fs.Mkdir("foo/bar", 0755)
 459         c.Check(err, check.NotNil)
 460
 461         m, err := s.fs.MarshalManifest(".")
 462         c.Check(err, check.IsNil)
 463         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 464         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 3:3:bar 0:3:foo\n./foo/bar acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 465 }
 466
 467 func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
 468         maxBlockSize = 8
 469         defer func() { maxBlockSize = 2 << 26 }()
 470
 471         var wg sync.WaitGroup
 472         for n := 0; n < 128; n++ {
 473                 wg.Add(1)
 474                 go func() {
 475                         defer wg.Done()
 476                         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 477                         c.Assert(err, check.IsNil)
 478                         defer f.Close()
 479                         for i := 0; i < 6502; i++ {
 480                                 switch rand.Int() & 3 {
 481                                 case 0:
 482                                         f.Truncate(int64(rand.Intn(64)))
 483                                 case 1:
 484                                         f.Seek(int64(rand.Intn(64)), io.SeekStart)
 485                                 case 2:
 486                                         _, err := f.Write([]byte("beep boop"))
 487                                         c.Check(err, check.IsNil)
 488                                 case 3:
 489                                         _, err := ioutil.ReadAll(f)
 490                                         c.Check(err, check.IsNil)
 491                                 }
 492                         }
 493                 }()
 494         }
 495         wg.Wait()
 496
 497         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 498         c.Assert(err, check.IsNil)
 499         defer f.Close()
 500         buf, err := ioutil.ReadAll(f)
 501         c.Check(err, check.IsNil)
 502         c.Logf("after lots of random r/w/seek/trunc, buf is %q", buf)
 503 }
 504
 505 func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
 506         maxBlockSize = 40
 507         defer func() { maxBlockSize = 2 << 26 }()
 508
 509         var err error
 510         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 511         c.Assert(err, check.IsNil)
 512
 513         const nfiles = 256
 514         const ngoroutines = 256
 515
 516         var wg sync.WaitGroup
 517         for n := 0; n < nfiles; n++ {
 518                 wg.Add(1)
 519                 go func(n int) {
 520                         defer wg.Done()
 521                         expect := make([]byte, 0, 64)
 522                         wbytes := []byte("there's no simple explanation for anything important that any of us do")
 523                         f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 524                         c.Assert(err, check.IsNil)
 525                         defer f.Close()
 526                         for i := 0; i < ngoroutines; i++ {
 527                                 trunc := rand.Intn(65)
 528                                 woff := rand.Intn(trunc + 1)
 529                                 wbytes = wbytes[:rand.Intn(64-woff+1)]
 530                                 for buf, i := expect[:cap(expect)], len(expect); i < trunc; i++ {
 531                                         buf[i] = 0
 532                                 }
 533                                 expect = expect[:trunc]
 534                                 if trunc < woff+len(wbytes) {
 535                                         expect = expect[:woff+len(wbytes)]
 536                                 }
 537                                 copy(expect[woff:], wbytes)
 538                                 f.Truncate(int64(trunc))
 539                                 pos, err := f.Seek(int64(woff), io.SeekStart)
 540                                 c.Check(pos, check.Equals, int64(woff))
 541                                 c.Check(err, check.IsNil)
 542                                 n, err := f.Write(wbytes)
 543                                 c.Check(n, check.Equals, len(wbytes))
 544                                 c.Check(err, check.IsNil)
 545                                 pos, err = f.Seek(0, io.SeekStart)
 546                                 c.Check(pos, check.Equals, int64(0))
 547                                 c.Check(err, check.IsNil)
 548                                 buf, err := ioutil.ReadAll(f)
 549                                 c.Check(string(buf), check.Equals, string(expect))
 550                                 c.Check(err, check.IsNil)
 551                         }
 552                         s.checkMemSize(c, f)
 553                 }(n)
 554         }
 555         wg.Wait()
 556
 557         root, err := s.fs.Open("/")
 558         c.Assert(err, check.IsNil)
 559         defer root.Close()
 560         fi, err := root.Readdir(-1)
 561         c.Check(err, check.IsNil)
 562         c.Check(len(fi), check.Equals, nfiles)
 563
 564         _, err = s.fs.MarshalManifest(".")
 565         c.Check(err, check.IsNil)
 566         // TODO: check manifest content
 567 }
 568
 569 func (s *CollectionFSSuite) TestRemove(c *check.C) {
 570         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 571         c.Assert(err, check.IsNil)
 572         err = fs.Mkdir("dir0", 0755)
 573         c.Assert(err, check.IsNil)
 574         err = fs.Mkdir("dir1", 0755)
 575         c.Assert(err, check.IsNil)
 576         err = fs.Mkdir("dir1/dir2", 0755)
 577         c.Assert(err, check.IsNil)
 578         err = fs.Mkdir("dir1/dir3", 0755)
 579         c.Assert(err, check.IsNil)
 580
 581         err = fs.Remove("dir0")
 582         c.Check(err, check.IsNil)
 583         err = fs.Remove("dir0")
 584         c.Check(err, check.Equals, os.ErrNotExist)
 585
 586         err = fs.Remove("dir1/dir2/.")
 587         c.Check(err, check.Equals, ErrInvalidArgument)
 588         err = fs.Remove("dir1/dir2/..")
 589         c.Check(err, check.Equals, ErrInvalidArgument)
 590         err = fs.Remove("dir1")
 591         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
 592         err = fs.Remove("dir1/dir2/../../../dir1")
 593         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
 594         err = fs.Remove("dir1/dir3/")
 595         c.Check(err, check.IsNil)
 596         err = fs.RemoveAll("dir1")
 597         c.Check(err, check.IsNil)
 598         err = fs.RemoveAll("dir1")
 599         c.Check(err, check.IsNil)
 600 }
 601
 602 func (s *CollectionFSSuite) TestRename(c *check.C) {
 603         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 604         c.Assert(err, check.IsNil)
 605         const (
 606                 outer = 16
 607                 inner = 16
 608         )
 609         for i := 0; i < outer; i++ {
 610                 err = fs.Mkdir(fmt.Sprintf("dir%d", i), 0755)
 611                 c.Assert(err, check.IsNil)
 612                 for j := 0; j < inner; j++ {
 613                         err = fs.Mkdir(fmt.Sprintf("dir%d/dir%d", i, j), 0755)
 614                         c.Assert(err, check.IsNil)
 615                         for _, fnm := range []string{
 616                                 fmt.Sprintf("dir%d/file%d", i, j),
 617                                 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
 618                         } {
 619                                 f, err := fs.OpenFile(fnm, os.O_CREATE|os.O_WRONLY, 0755)
 620                                 c.Assert(err, check.IsNil)
 621                                 _, err = f.Write([]byte("beep"))
 622                                 c.Assert(err, check.IsNil)
 623                                 f.Close()
 624                         }
 625                 }
 626         }
 627         var wg sync.WaitGroup
 628         for i := 0; i < outer; i++ {
 629                 for j := 0; j < inner; j++ {
 630                         wg.Add(1)
 631                         go func(i, j int) {
 632                                 defer wg.Done()
 633                                 oldname := fmt.Sprintf("dir%d/dir%d/file%d", i, j, j)
 634                                 newname := fmt.Sprintf("dir%d/newfile%d", i, inner-j-1)
 635                                 _, err := fs.Open(newname)
 636                                 c.Check(err, check.Equals, os.ErrNotExist)
 637                                 err = fs.Rename(oldname, newname)
 638                                 c.Check(err, check.IsNil)
 639                                 f, err := fs.Open(newname)
 640                                 c.Check(err, check.IsNil)
 641                                 f.Close()
 642                         }(i, j)
 643
 644                         wg.Add(1)
 645                         go func(i, j int) {
 646                                 defer wg.Done()
 647                                 // oldname does not exist
 648                                 err := fs.Rename(
 649                                         fmt.Sprintf("dir%d/dir%d/missing", i, j),
 650                                         fmt.Sprintf("dir%d/dir%d/file%d", outer-i-1, j, j))
 651                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 652
 653                                 // newname parent dir does not exist
 654                                 err = fs.Rename(
 655                                         fmt.Sprintf("dir%d/dir%d", i, j),
 656                                         fmt.Sprintf("dir%d/missing/irrelevant", outer-i-1))
 657                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 658
 659                                 // oldname parent dir is a file
 660                                 err = fs.Rename(
 661                                         fmt.Sprintf("dir%d/file%d/patherror", i, j),
 662                                         fmt.Sprintf("dir%d/irrelevant", i))
 663                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 664
 665                                 // newname parent dir is a file
 666                                 err = fs.Rename(
 667                                         fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
 668                                         fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
 669                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 670                         }(i, j)
 671                 }
 672         }
 673         wg.Wait()
 674
 675         f, err := fs.OpenFile("dir1/newfile3", 0, 0)
 676         c.Assert(err, check.IsNil)
 677         c.Check(f.Size(), check.Equals, int64(4))
 678         buf, err := ioutil.ReadAll(f)
 679         c.Check(buf, check.DeepEquals, []byte("beep"))
 680         c.Check(err, check.IsNil)
 681         _, err = fs.Open("dir1/dir1/file1")
 682         c.Check(err, check.Equals, os.ErrNotExist)
 683 }
 684
 685 func (s *CollectionFSSuite) TestPersist(c *check.C) {
 686         maxBlockSize = 1024
 687         defer func() { maxBlockSize = 2 << 26 }()
 688
 689         var err error
 690         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 691         c.Assert(err, check.IsNil)
 692         err = s.fs.Mkdir("d:r", 0755)
 693         c.Assert(err, check.IsNil)
 694
 695         expect := map[string][]byte{}
 696
 697         var wg sync.WaitGroup
 698         for _, name := range []string{"random 1", "random:2", "random\\3", "d:r/random4"} {
 699                 buf := make([]byte, 500)
 700                 rand.Read(buf)
 701                 expect[name] = buf
 702
 703                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 704                 c.Assert(err, check.IsNil)
 705                 // Note: we don't close the file until after the test
 706                 // is done. Writes to unclosed files should persist.
 707                 defer f.Close()
 708
 709                 wg.Add(1)
 710                 go func() {
 711                         defer wg.Done()
 712                         for i := 0; i < len(buf); i += 5 {
 713                                 _, err := f.Write(buf[i : i+5])
 714                                 c.Assert(err, check.IsNil)
 715                         }
 716                 }()
 717         }
 718         wg.Wait()
 719
 720         m, err := s.fs.MarshalManifest(".")
 721         c.Check(err, check.IsNil)
 722         c.Logf("%q", m)
 723
 724         root, err := s.fs.Open("/")
 725         c.Assert(err, check.IsNil)
 726         defer root.Close()
 727         fi, err := root.Readdir(-1)
 728         c.Check(err, check.IsNil)
 729         c.Check(len(fi), check.Equals, 4)
 730
 731         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
 732         c.Assert(err, check.IsNil)
 733
 734         root, err = persisted.Open("/")
 735         c.Assert(err, check.IsNil)
 736         defer root.Close()
 737         fi, err = root.Readdir(-1)
 738         c.Check(err, check.IsNil)
 739         c.Check(len(fi), check.Equals, 4)
 740
 741         for name, content := range expect {
 742                 c.Logf("read %q", name)
 743                 f, err := persisted.Open(name)
 744                 c.Assert(err, check.IsNil)
 745                 defer f.Close()
 746                 buf, err := ioutil.ReadAll(f)
 747                 c.Check(err, check.IsNil)
 748                 c.Check(buf, check.DeepEquals, content)
 749         }
 750 }
 751
 752 func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
 753         var err error
 754         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 755         c.Assert(err, check.IsNil)
 756         for _, name := range []string{"dir", "dir/zerodir", "zero", "zero/zero"} {
 757                 err = s.fs.Mkdir(name, 0755)
 758                 c.Assert(err, check.IsNil)
 759         }
 760
 761         expect := map[string][]byte{
 762                 "0":                nil,
 763                 "00":               []byte{},
 764                 "one":              []byte{1},
 765                 "dir/0":            nil,
 766                 "dir/two":          []byte{1, 2},
 767                 "dir/zero":         nil,
 768                 "dir/zerodir/zero": nil,
 769                 "zero/zero/zero":   nil,
 770         }
 771         for name, data := range expect {
 772                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 773                 c.Assert(err, check.IsNil)
 774                 if data != nil {
 775                         _, err := f.Write(data)
 776                         c.Assert(err, check.IsNil)
 777                 }
 778                 f.Close()
 779         }
 780
 781         m, err := s.fs.MarshalManifest(".")
 782         c.Check(err, check.IsNil)
 783         c.Logf("%q", m)
 784
 785         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
 786         c.Assert(err, check.IsNil)
 787
 788         for name, data := range expect {
 789                 f, err := persisted.Open("bogus-" + name)
 790                 c.Check(err, check.NotNil)
 791
 792                 f, err = persisted.Open(name)
 793                 c.Assert(err, check.IsNil)
 794
 795                 if data == nil {
 796                         data = []byte{}
 797                 }
 798                 buf, err := ioutil.ReadAll(f)
 799                 c.Check(err, check.IsNil)
 800                 c.Check(buf, check.DeepEquals, data)
 801         }
 802 }
 803
 804 func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
 805         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 806         c.Assert(err, check.IsNil)
 807
 808         f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
 809         c.Check(f, check.IsNil)
 810         c.Check(err, check.ErrorMatches, `file does not exist`)
 811
 812         f, err = fs.OpenFile("new", os.O_CREATE|os.O_RDONLY, 0)
 813         c.Assert(err, check.IsNil)
 814         defer f.Close()
 815         n, err := f.Write([]byte{1, 2, 3})
 816         c.Check(n, check.Equals, 0)
 817         c.Check(err, check.ErrorMatches, `read-only file`)
 818         n, err = f.Read(make([]byte, 1))
 819         c.Check(n, check.Equals, 0)
 820         c.Check(err, check.Equals, io.EOF)
 821         f, err = fs.OpenFile("new", os.O_RDWR, 0)
 822         c.Assert(err, check.IsNil)
 823         defer f.Close()
 824         _, err = f.Write([]byte{4, 5, 6})
 825         c.Check(err, check.IsNil)
 826         fi, err := f.Stat()
 827         c.Assert(err, check.IsNil)
 828         c.Check(fi.Size(), check.Equals, int64(3))
 829
 830         f, err = fs.OpenFile("new", os.O_TRUNC|os.O_RDWR, 0)
 831         c.Assert(err, check.IsNil)
 832         defer f.Close()
 833         pos, err := f.Seek(0, io.SeekEnd)
 834         c.Check(pos, check.Equals, int64(0))
 835         c.Check(err, check.IsNil)
 836         fi, err = f.Stat()
 837         c.Assert(err, check.IsNil)
 838         c.Check(fi.Size(), check.Equals, int64(0))
 839         fs.Remove("new")
 840
 841         buf := make([]byte, 64)
 842         f, err = fs.OpenFile("append", os.O_EXCL|os.O_CREATE|os.O_RDWR|os.O_APPEND, 0)
 843         c.Assert(err, check.IsNil)
 844         f.Write([]byte{1, 2, 3})
 845         f.Seek(0, io.SeekStart)
 846         n, _ = f.Read(buf[:1])
 847         c.Check(n, check.Equals, 1)
 848         c.Check(buf[:1], check.DeepEquals, []byte{1})
 849         pos, err = f.Seek(0, io.SeekCurrent)
 850         c.Check(pos, check.Equals, int64(1))
 851         f.Write([]byte{4, 5, 6})
 852         pos, err = f.Seek(0, io.SeekCurrent)
 853         c.Check(pos, check.Equals, int64(6))
 854         f.Seek(0, io.SeekStart)
 855         n, err = f.Read(buf)
 856         c.Check(buf[:n], check.DeepEquals, []byte{1, 2, 3, 4, 5, 6})
 857         c.Check(err, check.Equals, io.EOF)
 858         f.Close()
 859
 860         f, err = fs.OpenFile("append", os.O_RDWR|os.O_APPEND, 0)
 861         c.Assert(err, check.IsNil)
 862         pos, err = f.Seek(0, io.SeekCurrent)
 863         c.Check(pos, check.Equals, int64(0))
 864         c.Check(err, check.IsNil)
 865         f.Read(buf[:3])
 866         pos, _ = f.Seek(0, io.SeekCurrent)
 867         c.Check(pos, check.Equals, int64(3))
 868         f.Write([]byte{7, 8, 9})
 869         pos, err = f.Seek(0, io.SeekCurrent)
 870         c.Check(pos, check.Equals, int64(9))
 871         f.Close()
 872
 873         f, err = fs.OpenFile("wronly", os.O_CREATE|os.O_WRONLY, 0)
 874         c.Assert(err, check.IsNil)
 875         n, err = f.Write([]byte{3, 2, 1})
 876         c.Check(n, check.Equals, 3)
 877         c.Check(err, check.IsNil)
 878         pos, _ = f.Seek(0, io.SeekCurrent)
 879         c.Check(pos, check.Equals, int64(3))
 880         pos, _ = f.Seek(0, io.SeekStart)
 881         c.Check(pos, check.Equals, int64(0))
 882         n, err = f.Read(buf)
 883         c.Check(n, check.Equals, 0)
 884         c.Check(err, check.ErrorMatches, `.*O_WRONLY.*`)
 885         f, err = fs.OpenFile("wronly", os.O_RDONLY, 0)
 886         c.Assert(err, check.IsNil)
 887         n, _ = f.Read(buf)
 888         c.Check(buf[:n], check.DeepEquals, []byte{3, 2, 1})
 889
 890         f, err = fs.OpenFile("unsupported", os.O_CREATE|os.O_SYNC, 0)
 891         c.Check(f, check.IsNil)
 892         c.Check(err, check.NotNil)
 893
 894         f, err = fs.OpenFile("append", os.O_RDWR|os.O_WRONLY, 0)
 895         c.Check(f, check.IsNil)
 896         c.Check(err, check.ErrorMatches, `invalid flag.*`)
 897 }
 898
 899 func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
 900         maxBlockSize = 1024
 901         defer func() { maxBlockSize = 2 << 26 }()
 902
 903         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 904         c.Assert(err, check.IsNil)
 905         f, err := fs.OpenFile("50K", os.O_WRONLY|os.O_CREATE, 0)
 906         c.Assert(err, check.IsNil)
 907         defer f.Close()
 908
 909         data := make([]byte, 500)
 910         rand.Read(data)
 911
 912         for i := 0; i < 100; i++ {
 913                 n, err := f.Write(data)
 914                 c.Assert(n, check.Equals, len(data))
 915                 c.Assert(err, check.IsNil)
 916         }
 917
 918         currentMemExtents := func() (memExtents []int) {
 919                 for idx, e := range f.(*filehandle).inode.(*filenode).segments {
 920                         switch e.(type) {
 921                         case *memSegment:
 922                                 memExtents = append(memExtents, idx)
 923                         }
 924                 }
 925                 return
 926         }
 927         c.Check(currentMemExtents(), check.HasLen, 1)
 928
 929         m, err := fs.MarshalManifest(".")
 930         c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
 931         c.Check(err, check.IsNil)
 932         c.Check(currentMemExtents(), check.HasLen, 0)
 933 }
 934
 935 func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
 936         for _, txt := range []string{
 937                 "\n",
 938                 ".\n",
 939                 ". \n",
 940                 ". d41d8cd98f00b204e9800998ecf8427e+0\n",
 941                 ". d41d8cd98f00b204e9800998ecf8427e+0 \n",
 942                 ". 0:0:foo\n",
 943                 ".  0:0:foo\n",
 944                 ". 0:0:foo 0:0:bar\n",
 945                 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo\n",
 946                 ". d41d8cd98f00b204e9800998ecf8427e+0 :0:0:foo\n",
 947                 ". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
 948                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
 949                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
 950                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n",
 951                 "./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
 952         } {
 953                 c.Logf("<-%q", txt)
 954                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
 955                 c.Check(fs, check.IsNil)
 956                 c.Logf("-> %s", err)
 957                 c.Check(err, check.NotNil)
 958         }
 959 }
 960
 961 func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
 962         for _, txt := range []string{
 963                 "",
 964                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
 965                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
 966                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
 967                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
 968         } {
 969                 c.Logf("<-%q", txt)
 970                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
 971                 c.Check(err, check.IsNil)
 972                 c.Check(fs, check.NotNil)
 973         }
 974 }
 975
 976 func (s *CollectionFSSuite) checkMemSize(c *check.C, f File) {
 977         fn := f.(*filehandle).inode.(*filenode)
 978         var memsize int64
 979         for _, seg := range fn.segments {
 980                 if e, ok := seg.(*memSegment); ok {
 981                         memsize += int64(len(e.buf))
 982                 }
 983         }
 984         c.Check(fn.memsize, check.Equals, memsize)
 985 }
 986
 987 type CollectionFSUnitSuite struct{}
 988
 989 var _ = check.Suite(&CollectionFSUnitSuite{})
 990
 991 // expect ~2 seconds to load a manifest with 256K files
 992 func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
 993         const (
 994                 dirCount  = 512
 995                 fileCount = 512
 996         )
 997
 998         mb := bytes.NewBuffer(make([]byte, 0, 40000000))
 999         for i := 0; i < dirCount; i++ {
1000                 fmt.Fprintf(mb, "./dir%d", i)
1001                 for j := 0; j <= fileCount; j++ {
1002                         fmt.Fprintf(mb, " %032x+42+A%040x@%08x", j, j, j)
1003                 }
1004                 for j := 0; j < fileCount; j++ {
1005                         fmt.Fprintf(mb, " %d:%d:dir%d/file%d", j*42+21, 42, j, j)
1006                 }
1007                 mb.Write([]byte{'\n'})
1008         }
1009         coll := Collection{ManifestText: mb.String()}
1010         c.Logf("%s built", time.Now())
1011
1012         var memstats runtime.MemStats
1013         runtime.ReadMemStats(&memstats)
1014         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1015
1016         f, err := coll.FileSystem(nil, nil)
1017         c.Check(err, check.IsNil)
1018         c.Logf("%s loaded", time.Now())
1019
1020         for i := 0; i < dirCount; i++ {
1021                 for j := 0; j < fileCount; j++ {
1022                         f.Stat(fmt.Sprintf("./dir%d/dir%d/file%d", i, j, j))
1023                 }
1024         }
1025         c.Logf("%s Stat() x %d", time.Now(), dirCount*fileCount)
1026
1027         runtime.ReadMemStats(&memstats)
1028         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1029 }
1030
1031 // Gocheck boilerplate
1032 func Test(t *testing.T) {
1033         check.TestingT(t)
1034 }