sdk/go/arvados/fs_collection_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "math/rand"
  15         "net/http"
  16         "os"
  17         "regexp"
  18         "runtime"
  19         "sync"
  20         "testing"
  21         "time"
  22
  23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  24         check "gopkg.in/check.v1"
  25 )
  26
  27 var _ = check.Suite(&CollectionFSSuite{})
  28
  29 type keepClientStub struct {
  30         blocks map[string][]byte
  31         sync.RWMutex
  32 }
  33
  34 var errStub404 = errors.New("404 block not found")
  35
  36 func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error) {
  37         kcs.RLock()
  38         defer kcs.RUnlock()
  39         buf := kcs.blocks[locator[:32]]
  40         if buf == nil {
  41                 return 0, errStub404
  42         }
  43         return copy(p, buf[off:]), nil
  44 }
  45
  46 func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
  47         locator := fmt.Sprintf("%x+%d+A12345@abcde", md5.Sum(p), len(p))
  48         buf := make([]byte, len(p))
  49         copy(buf, p)
  50         kcs.Lock()
  51         defer kcs.Unlock()
  52         kcs.blocks[locator[:32]] = buf
  53         return locator, 1, nil
  54 }
  55
  56 type CollectionFSSuite struct {
  57         client *Client
  58         coll   Collection
  59         fs     CollectionFileSystem
  60         kc     keepClient
  61 }
  62
  63 func (s *CollectionFSSuite) SetUpTest(c *check.C) {
  64         s.client = NewClientFromEnv()
  65         err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
  66         c.Assert(err, check.IsNil)
  67         s.kc = &keepClientStub{
  68                 blocks: map[string][]byte{
  69                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  70                 }}
  71         s.fs, err = s.coll.FileSystem(s.client, s.kc)
  72         c.Assert(err, check.IsNil)
  73 }
  74
  75 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
  76         _, ok := s.fs.(http.FileSystem)
  77         c.Check(ok, check.Equals, true)
  78 }
  79
  80 func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
  81         fs, err := (&Collection{
  82                 ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
  83         }).FileSystem(s.client, s.kc)
  84         c.Assert(err, check.IsNil)
  85
  86         f, err := fs.Open("/foo:foo")
  87         c.Assert(err, check.IsNil)
  88
  89         fis, err := f.Readdir(0)
  90         c.Check(err, check.IsNil)
  91         c.Check(len(fis), check.Equals, 1)
  92         c.Check(fis[0].Name(), check.Equals, "bar:bar")
  93 }
  94
  95 func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
  96         f, err := s.fs.Open("/dir1")
  97         c.Assert(err, check.IsNil)
  98
  99         st, err := f.Stat()
 100         c.Assert(err, check.IsNil)
 101         c.Check(st.Size(), check.Equals, int64(2))
 102         c.Check(st.IsDir(), check.Equals, true)
 103
 104         fis, err := f.Readdir(0)
 105         c.Check(err, check.IsNil)
 106         c.Check(len(fis), check.Equals, 2)
 107         if len(fis) > 0 {
 108                 c.Check(fis[0].Size(), check.Equals, int64(3))
 109         }
 110 }
 111
 112 func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
 113         f, err := s.fs.Open("./dir1")
 114         c.Assert(err, check.IsNil)
 115
 116         fis, err := f.Readdir(1)
 117         c.Check(err, check.IsNil)
 118         c.Check(len(fis), check.Equals, 1)
 119         if len(fis) > 0 {
 120                 c.Check(fis[0].Size(), check.Equals, int64(3))
 121         }
 122
 123         fis, err = f.Readdir(1)
 124         c.Check(err, check.IsNil)
 125         c.Check(len(fis), check.Equals, 1)
 126         if len(fis) > 0 {
 127                 c.Check(fis[0].Size(), check.Equals, int64(3))
 128         }
 129
 130         fis, err = f.Readdir(1)
 131         c.Check(len(fis), check.Equals, 0)
 132         c.Check(err, check.NotNil)
 133         c.Check(err, check.Equals, io.EOF)
 134
 135         f, err = s.fs.Open("dir1")
 136         c.Assert(err, check.IsNil)
 137         fis, err = f.Readdir(1)
 138         c.Check(len(fis), check.Equals, 1)
 139         c.Assert(err, check.IsNil)
 140         fis, err = f.Readdir(2)
 141         c.Check(len(fis), check.Equals, 1)
 142         c.Assert(err, check.IsNil)
 143         fis, err = f.Readdir(2)
 144         c.Check(len(fis), check.Equals, 0)
 145         c.Assert(err, check.Equals, io.EOF)
 146 }
 147
 148 func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
 149         for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
 150                 f, err := s.fs.Open(path)
 151                 c.Assert(err, check.IsNil)
 152
 153                 st, err := f.Stat()
 154                 c.Assert(err, check.IsNil)
 155                 c.Check(st.Size(), check.Equals, int64(1))
 156                 c.Check(st.IsDir(), check.Equals, true)
 157         }
 158         for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
 159                 c.Logf("%q", path)
 160                 f, err := s.fs.Open(path)
 161                 c.Assert(err, check.IsNil)
 162
 163                 st, err := f.Stat()
 164                 c.Assert(err, check.IsNil)
 165                 c.Check(st.Size(), check.Equals, int64(2))
 166                 c.Check(st.IsDir(), check.Equals, true)
 167         }
 168 }
 169
 170 func (s *CollectionFSSuite) TestNotExist(c *check.C) {
 171         for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
 172                 f, err := s.fs.Open(path)
 173                 c.Assert(f, check.IsNil)
 174                 c.Assert(err, check.NotNil)
 175                 c.Assert(os.IsNotExist(err), check.Equals, true)
 176         }
 177 }
 178
 179 func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
 180         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDONLY, 0)
 181         c.Assert(err, check.IsNil)
 182         st, err := f.Stat()
 183         c.Assert(err, check.IsNil)
 184         c.Check(st.Size(), check.Equals, int64(3))
 185         n, err := f.Write([]byte("bar"))
 186         c.Check(n, check.Equals, 0)
 187         c.Check(err, check.Equals, ErrReadOnlyFile)
 188 }
 189
 190 func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
 191         f, err := s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE, 0)
 192         c.Assert(err, check.IsNil)
 193         st, err := f.Stat()
 194         c.Assert(err, check.IsNil)
 195         c.Check(st.Size(), check.Equals, int64(0))
 196
 197         n, err := f.Write([]byte("bar"))
 198         c.Check(n, check.Equals, 3)
 199         c.Check(err, check.IsNil)
 200
 201         c.Check(f.Close(), check.IsNil)
 202
 203         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 204         c.Check(f, check.IsNil)
 205         c.Assert(err, check.NotNil)
 206
 207         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR, 0)
 208         c.Assert(err, check.IsNil)
 209         st, err = f.Stat()
 210         c.Assert(err, check.IsNil)
 211         c.Check(st.Size(), check.Equals, int64(3))
 212
 213         c.Check(f.Close(), check.IsNil)
 214
 215         m, err := s.fs.MarshalManifest(".")
 216         c.Assert(err, check.IsNil)
 217         c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
 218 }
 219
 220 func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
 221         maxBlockSize = 8
 222         defer func() { maxBlockSize = 2 << 26 }()
 223
 224         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 225         c.Assert(err, check.IsNil)
 226         defer f.Close()
 227         st, err := f.Stat()
 228         c.Assert(err, check.IsNil)
 229         c.Check(st.Size(), check.Equals, int64(3))
 230
 231         f2, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 232         c.Assert(err, check.IsNil)
 233         defer f2.Close()
 234
 235         buf := make([]byte, 64)
 236         n, err := f.Read(buf)
 237         c.Check(n, check.Equals, 3)
 238         c.Check(err, check.Equals, io.EOF)
 239         c.Check(string(buf[:3]), check.DeepEquals, "foo")
 240
 241         pos, err := f.Seek(-2, io.SeekCurrent)
 242         c.Check(pos, check.Equals, int64(1))
 243         c.Check(err, check.IsNil)
 244
 245         // Split a storedExtent in two, and insert a memExtent
 246         n, err = f.Write([]byte("*"))
 247         c.Check(n, check.Equals, 1)
 248         c.Check(err, check.IsNil)
 249
 250         pos, err = f.Seek(0, io.SeekCurrent)
 251         c.Check(pos, check.Equals, int64(2))
 252         c.Check(err, check.IsNil)
 253
 254         pos, err = f.Seek(0, io.SeekStart)
 255         c.Check(pos, check.Equals, int64(0))
 256         c.Check(err, check.IsNil)
 257
 258         rbuf, err := ioutil.ReadAll(f)
 259         c.Check(len(rbuf), check.Equals, 3)
 260         c.Check(err, check.IsNil)
 261         c.Check(string(rbuf), check.Equals, "f*o")
 262
 263         // Write multiple blocks in one call
 264         f.Seek(1, io.SeekStart)
 265         n, err = f.Write([]byte("0123456789abcdefg"))
 266         c.Check(n, check.Equals, 17)
 267         c.Check(err, check.IsNil)
 268         pos, err = f.Seek(0, io.SeekCurrent)
 269         c.Check(pos, check.Equals, int64(18))
 270         c.Check(err, check.IsNil)
 271         pos, err = f.Seek(-18, io.SeekCurrent)
 272         c.Check(pos, check.Equals, int64(0))
 273         c.Check(err, check.IsNil)
 274         n, err = io.ReadFull(f, buf)
 275         c.Check(n, check.Equals, 18)
 276         c.Check(err, check.Equals, io.ErrUnexpectedEOF)
 277         c.Check(string(buf[:n]), check.Equals, "f0123456789abcdefg")
 278
 279         buf2, err := ioutil.ReadAll(f2)
 280         c.Check(err, check.IsNil)
 281         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 282
 283         // truncate to current size
 284         err = f.Truncate(18)
 285         c.Check(err, check.IsNil)
 286         f2.Seek(0, io.SeekStart)
 287         buf2, err = ioutil.ReadAll(f2)
 288         c.Check(err, check.IsNil)
 289         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 290
 291         // shrink to zero some data
 292         f.Truncate(15)
 293         f2.Seek(0, io.SeekStart)
 294         buf2, err = ioutil.ReadAll(f2)
 295         c.Check(err, check.IsNil)
 296         c.Check(string(buf2), check.Equals, "f0123456789abcd")
 297
 298         // grow to partial block/extent
 299         f.Truncate(20)
 300         f2.Seek(0, io.SeekStart)
 301         buf2, err = ioutil.ReadAll(f2)
 302         c.Check(err, check.IsNil)
 303         c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
 304
 305         f.Truncate(0)
 306         f2.Seek(0, io.SeekStart)
 307         f2.Write([]byte("12345678abcdefghijkl"))
 308
 309         // grow to block/extent boundary
 310         f.Truncate(64)
 311         f2.Seek(0, io.SeekStart)
 312         buf2, err = ioutil.ReadAll(f2)
 313         c.Check(err, check.IsNil)
 314         c.Check(len(buf2), check.Equals, 64)
 315         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 8)
 316
 317         // shrink to block/extent boundary
 318         err = f.Truncate(32)
 319         c.Check(err, check.IsNil)
 320         f2.Seek(0, io.SeekStart)
 321         buf2, err = ioutil.ReadAll(f2)
 322         c.Check(err, check.IsNil)
 323         c.Check(len(buf2), check.Equals, 32)
 324         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 4)
 325
 326         // shrink to partial block/extent
 327         err = f.Truncate(15)
 328         c.Check(err, check.IsNil)
 329         f2.Seek(0, io.SeekStart)
 330         buf2, err = ioutil.ReadAll(f2)
 331         c.Check(err, check.IsNil)
 332         c.Check(string(buf2), check.Equals, "12345678abcdefg")
 333         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 2)
 334
 335         // Force flush to ensure the block "12345678" gets stored, so
 336         // we know what to expect in the final manifest below.
 337         _, err = s.fs.MarshalManifest(".")
 338         c.Check(err, check.IsNil)
 339
 340         // Truncate to size=3 while f2's ptr is at 15
 341         err = f.Truncate(3)
 342         c.Check(err, check.IsNil)
 343         buf2, err = ioutil.ReadAll(f2)
 344         c.Check(err, check.IsNil)
 345         c.Check(string(buf2), check.Equals, "")
 346         f2.Seek(0, io.SeekStart)
 347         buf2, err = ioutil.ReadAll(f2)
 348         c.Check(err, check.IsNil)
 349         c.Check(string(buf2), check.Equals, "123")
 350         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 1)
 351
 352         m, err := s.fs.MarshalManifest(".")
 353         c.Check(err, check.IsNil)
 354         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 355         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 25d55ad283aa400af464c76d713c07ad+8 3:3:bar 6:3:foo\n")
 356         c.Check(s.fs.Size(), check.Equals, int64(6))
 357 }
 358
 359 func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
 360         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 361         c.Assert(err, check.IsNil)
 362         f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
 363         c.Assert(err, check.IsNil)
 364         defer f.Close()
 365
 366         checkSize := func(size int64) {
 367                 fi, err := f.Stat()
 368                 c.Assert(err, check.IsNil)
 369                 c.Check(fi.Size(), check.Equals, size)
 370
 371                 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
 372                 c.Assert(err, check.IsNil)
 373                 defer f.Close()
 374                 fi, err = f.Stat()
 375                 c.Check(err, check.IsNil)
 376                 c.Check(fi.Size(), check.Equals, size)
 377                 pos, err := f.Seek(0, io.SeekEnd)
 378                 c.Check(err, check.IsNil)
 379                 c.Check(pos, check.Equals, size)
 380         }
 381
 382         f.Seek(2, io.SeekEnd)
 383         checkSize(0)
 384         f.Write([]byte{1})
 385         checkSize(3)
 386
 387         f.Seek(2, io.SeekCurrent)
 388         checkSize(3)
 389         f.Write([]byte{})
 390         checkSize(5)
 391
 392         f.Seek(8, io.SeekStart)
 393         checkSize(5)
 394         n, err := f.Read(make([]byte, 1))
 395         c.Check(n, check.Equals, 0)
 396         c.Check(err, check.Equals, io.EOF)
 397         checkSize(5)
 398         f.Write([]byte{1, 2, 3})
 399         checkSize(11)
 400 }
 401
 402 func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
 403         maxBlockSize = 8
 404         defer func() { maxBlockSize = 2 << 26 }()
 405
 406         var err error
 407         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 408         c.Assert(err, check.IsNil)
 409         for _, name := range []string{"foo", "bar", "baz"} {
 410                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 411                 c.Assert(err, check.IsNil)
 412                 f.Write([]byte(name))
 413                 f.Close()
 414         }
 415
 416         m, err := s.fs.MarshalManifest(".")
 417         c.Check(err, check.IsNil)
 418         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 419         c.Check(m, check.Equals, ". c3c23db5285662ef7172373df0003206+6 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar 3:3:baz 6:3:foo\n")
 420 }
 421
 422 func (s *CollectionFSSuite) TestMkdir(c *check.C) {
 423         err := s.fs.Mkdir("foo/bar", 0755)
 424         c.Check(err, check.Equals, os.ErrNotExist)
 425
 426         f, err := s.fs.OpenFile("foo/bar", os.O_CREATE, 0)
 427         c.Check(err, check.Equals, os.ErrNotExist)
 428
 429         err = s.fs.Mkdir("foo", 0755)
 430         c.Check(err, check.IsNil)
 431
 432         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_WRONLY, 0)
 433         c.Check(err, check.IsNil)
 434         if err == nil {
 435                 defer f.Close()
 436                 f.Write([]byte("foo"))
 437         }
 438
 439         // mkdir fails if a file already exists with that name
 440         err = s.fs.Mkdir("foo/bar", 0755)
 441         c.Check(err, check.NotNil)
 442
 443         err = s.fs.Remove("foo/bar")
 444         c.Check(err, check.IsNil)
 445
 446         // mkdir succeeds after the file is deleted
 447         err = s.fs.Mkdir("foo/bar", 0755)
 448         c.Check(err, check.IsNil)
 449
 450         // creating a file in a nonexistent subdir should still fail
 451         f, err = s.fs.OpenFile("foo/bar/baz/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
 452         c.Check(err, check.Equals, os.ErrNotExist)
 453
 454         f, err = s.fs.OpenFile("foo/bar/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
 455         c.Check(err, check.IsNil)
 456         if err == nil {
 457                 defer f.Close()
 458                 f.Write([]byte("foo"))
 459         }
 460
 461         // creating foo/bar as a regular file should fail
 462         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, 0)
 463         c.Check(err, check.NotNil)
 464
 465         // creating foo/bar as a directory should fail
 466         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, os.ModeDir)
 467         c.Check(err, check.NotNil)
 468         err = s.fs.Mkdir("foo/bar", 0755)
 469         c.Check(err, check.NotNil)
 470
 471         m, err := s.fs.MarshalManifest(".")
 472         c.Check(err, check.IsNil)
 473         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 474         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 3:3:bar 0:3:foo\n./foo/bar acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 475 }
 476
 477 func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
 478         if testing.Short() {
 479                 c.Skip("slow")
 480         }
 481
 482         maxBlockSize = 8
 483         defer func() { maxBlockSize = 2 << 26 }()
 484
 485         var wg sync.WaitGroup
 486         for n := 0; n < 128; n++ {
 487                 wg.Add(1)
 488                 go func() {
 489                         defer wg.Done()
 490                         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 491                         c.Assert(err, check.IsNil)
 492                         defer f.Close()
 493                         for i := 0; i < 6502; i++ {
 494                                 r := rand.Uint32()
 495                                 switch {
 496                                 case r%11 == 0:
 497                                         _, err := s.fs.MarshalManifest(".")
 498                                         c.Check(err, check.IsNil)
 499                                 case r&3 == 0:
 500                                         f.Truncate(int64(rand.Intn(64)))
 501                                 case r&3 == 1:
 502                                         f.Seek(int64(rand.Intn(64)), io.SeekStart)
 503                                 case r&3 == 2:
 504                                         _, err := f.Write([]byte("beep boop"))
 505                                         c.Check(err, check.IsNil)
 506                                 case r&3 == 3:
 507                                         _, err := ioutil.ReadAll(f)
 508                                         c.Check(err, check.IsNil)
 509                                 }
 510                         }
 511                 }()
 512         }
 513         wg.Wait()
 514
 515         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 516         c.Assert(err, check.IsNil)
 517         defer f.Close()
 518         buf, err := ioutil.ReadAll(f)
 519         c.Check(err, check.IsNil)
 520         c.Logf("after lots of random r/w/seek/trunc, buf is %q", buf)
 521 }
 522
 523 func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
 524         maxBlockSize = 40
 525         defer func() { maxBlockSize = 2 << 26 }()
 526
 527         var err error
 528         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 529         c.Assert(err, check.IsNil)
 530
 531         const nfiles = 256
 532         const ngoroutines = 256
 533
 534         var wg sync.WaitGroup
 535         for n := 0; n < nfiles; n++ {
 536                 wg.Add(1)
 537                 go func(n int) {
 538                         defer wg.Done()
 539                         expect := make([]byte, 0, 64)
 540                         wbytes := []byte("there's no simple explanation for anything important that any of us do")
 541                         f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 542                         c.Assert(err, check.IsNil)
 543                         defer f.Close()
 544                         for i := 0; i < ngoroutines; i++ {
 545                                 trunc := rand.Intn(65)
 546                                 woff := rand.Intn(trunc + 1)
 547                                 wbytes = wbytes[:rand.Intn(64-woff+1)]
 548                                 for buf, i := expect[:cap(expect)], len(expect); i < trunc; i++ {
 549                                         buf[i] = 0
 550                                 }
 551                                 expect = expect[:trunc]
 552                                 if trunc < woff+len(wbytes) {
 553                                         expect = expect[:woff+len(wbytes)]
 554                                 }
 555                                 copy(expect[woff:], wbytes)
 556                                 f.Truncate(int64(trunc))
 557                                 pos, err := f.Seek(int64(woff), io.SeekStart)
 558                                 c.Check(pos, check.Equals, int64(woff))
 559                                 c.Check(err, check.IsNil)
 560                                 n, err := f.Write(wbytes)
 561                                 c.Check(n, check.Equals, len(wbytes))
 562                                 c.Check(err, check.IsNil)
 563                                 pos, err = f.Seek(0, io.SeekStart)
 564                                 c.Check(pos, check.Equals, int64(0))
 565                                 c.Check(err, check.IsNil)
 566                                 buf, err := ioutil.ReadAll(f)
 567                                 c.Check(string(buf), check.Equals, string(expect))
 568                                 c.Check(err, check.IsNil)
 569                         }
 570                         s.checkMemSize(c, f)
 571                 }(n)
 572         }
 573         wg.Wait()
 574
 575         root, err := s.fs.Open("/")
 576         c.Assert(err, check.IsNil)
 577         defer root.Close()
 578         fi, err := root.Readdir(-1)
 579         c.Check(err, check.IsNil)
 580         c.Check(len(fi), check.Equals, nfiles)
 581
 582         _, err = s.fs.MarshalManifest(".")
 583         c.Check(err, check.IsNil)
 584         // TODO: check manifest content
 585 }
 586
 587 func (s *CollectionFSSuite) TestRemove(c *check.C) {
 588         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 589         c.Assert(err, check.IsNil)
 590         err = fs.Mkdir("dir0", 0755)
 591         c.Assert(err, check.IsNil)
 592         err = fs.Mkdir("dir1", 0755)
 593         c.Assert(err, check.IsNil)
 594         err = fs.Mkdir("dir1/dir2", 0755)
 595         c.Assert(err, check.IsNil)
 596         err = fs.Mkdir("dir1/dir3", 0755)
 597         c.Assert(err, check.IsNil)
 598
 599         err = fs.Remove("dir0")
 600         c.Check(err, check.IsNil)
 601         err = fs.Remove("dir0")
 602         c.Check(err, check.Equals, os.ErrNotExist)
 603
 604         err = fs.Remove("dir1/dir2/.")
 605         c.Check(err, check.Equals, ErrInvalidArgument)
 606         err = fs.Remove("dir1/dir2/..")
 607         c.Check(err, check.Equals, ErrInvalidArgument)
 608         err = fs.Remove("dir1")
 609         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
 610         err = fs.Remove("dir1/dir2/../../../dir1")
 611         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
 612         err = fs.Remove("dir1/dir3/")
 613         c.Check(err, check.IsNil)
 614         err = fs.RemoveAll("dir1")
 615         c.Check(err, check.IsNil)
 616         err = fs.RemoveAll("dir1")
 617         c.Check(err, check.IsNil)
 618 }
 619
 620 func (s *CollectionFSSuite) TestRenameError(c *check.C) {
 621         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 622         c.Assert(err, check.IsNil)
 623         err = fs.Mkdir("first", 0755)
 624         c.Assert(err, check.IsNil)
 625         err = fs.Mkdir("first/second", 0755)
 626         c.Assert(err, check.IsNil)
 627         f, err := fs.OpenFile("first/second/file", os.O_CREATE|os.O_WRONLY, 0755)
 628         c.Assert(err, check.IsNil)
 629         f.Write([]byte{1, 2, 3, 4, 5})
 630         f.Close()
 631         err = fs.Rename("first", "first/second/third")
 632         c.Check(err, check.Equals, ErrInvalidArgument)
 633         err = fs.Rename("first", "first/third")
 634         c.Check(err, check.Equals, ErrInvalidArgument)
 635         err = fs.Rename("first/second", "second")
 636         c.Check(err, check.IsNil)
 637         f, err = fs.OpenFile("second/file", 0, 0)
 638         c.Assert(err, check.IsNil)
 639         data, err := ioutil.ReadAll(f)
 640         c.Check(err, check.IsNil)
 641         c.Check(data, check.DeepEquals, []byte{1, 2, 3, 4, 5})
 642 }
 643
 644 func (s *CollectionFSSuite) TestRename(c *check.C) {
 645         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 646         c.Assert(err, check.IsNil)
 647         const (
 648                 outer = 16
 649                 inner = 16
 650         )
 651         for i := 0; i < outer; i++ {
 652                 err = fs.Mkdir(fmt.Sprintf("dir%d", i), 0755)
 653                 c.Assert(err, check.IsNil)
 654                 for j := 0; j < inner; j++ {
 655                         err = fs.Mkdir(fmt.Sprintf("dir%d/dir%d", i, j), 0755)
 656                         c.Assert(err, check.IsNil)
 657                         for _, fnm := range []string{
 658                                 fmt.Sprintf("dir%d/file%d", i, j),
 659                                 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
 660                         } {
 661                                 f, err := fs.OpenFile(fnm, os.O_CREATE|os.O_WRONLY, 0755)
 662                                 c.Assert(err, check.IsNil)
 663                                 _, err = f.Write([]byte("beep"))
 664                                 c.Assert(err, check.IsNil)
 665                                 f.Close()
 666                         }
 667                 }
 668         }
 669         var wg sync.WaitGroup
 670         for i := 0; i < outer; i++ {
 671                 for j := 0; j < inner; j++ {
 672                         wg.Add(1)
 673                         go func(i, j int) {
 674                                 defer wg.Done()
 675                                 oldname := fmt.Sprintf("dir%d/dir%d/file%d", i, j, j)
 676                                 newname := fmt.Sprintf("dir%d/newfile%d", i, inner-j-1)
 677                                 _, err := fs.Open(newname)
 678                                 c.Check(err, check.Equals, os.ErrNotExist)
 679                                 err = fs.Rename(oldname, newname)
 680                                 c.Check(err, check.IsNil)
 681                                 f, err := fs.Open(newname)
 682                                 c.Check(err, check.IsNil)
 683                                 f.Close()
 684                         }(i, j)
 685
 686                         wg.Add(1)
 687                         go func(i, j int) {
 688                                 defer wg.Done()
 689                                 // oldname does not exist
 690                                 err := fs.Rename(
 691                                         fmt.Sprintf("dir%d/dir%d/missing", i, j),
 692                                         fmt.Sprintf("dir%d/dir%d/file%d", outer-i-1, j, j))
 693                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 694
 695                                 // newname parent dir does not exist
 696                                 err = fs.Rename(
 697                                         fmt.Sprintf("dir%d/dir%d", i, j),
 698                                         fmt.Sprintf("dir%d/missing/irrelevant", outer-i-1))
 699                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 700
 701                                 // oldname parent dir is a file
 702                                 err = fs.Rename(
 703                                         fmt.Sprintf("dir%d/file%d/patherror", i, j),
 704                                         fmt.Sprintf("dir%d/irrelevant", i))
 705                                 c.Check(err, check.ErrorMatches, `.*not a directory`)
 706
 707                                 // newname parent dir is a file
 708                                 err = fs.Rename(
 709                                         fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
 710                                         fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
 711                                 c.Check(err, check.ErrorMatches, `.*not a directory`)
 712                         }(i, j)
 713                 }
 714         }
 715         wg.Wait()
 716
 717         f, err := fs.OpenFile("dir1/newfile3", 0, 0)
 718         c.Assert(err, check.IsNil)
 719         c.Check(f.Size(), check.Equals, int64(4))
 720         buf, err := ioutil.ReadAll(f)
 721         c.Check(buf, check.DeepEquals, []byte("beep"))
 722         c.Check(err, check.IsNil)
 723         _, err = fs.Open("dir1/dir1/file1")
 724         c.Check(err, check.Equals, os.ErrNotExist)
 725 }
 726
 727 func (s *CollectionFSSuite) TestPersist(c *check.C) {
 728         maxBlockSize = 1024
 729         defer func() { maxBlockSize = 2 << 26 }()
 730
 731         var err error
 732         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 733         c.Assert(err, check.IsNil)
 734         err = s.fs.Mkdir("d:r", 0755)
 735         c.Assert(err, check.IsNil)
 736
 737         expect := map[string][]byte{}
 738
 739         var wg sync.WaitGroup
 740         for _, name := range []string{"random 1", "random:2", "random\\3", "d:r/random4"} {
 741                 buf := make([]byte, 500)
 742                 rand.Read(buf)
 743                 expect[name] = buf
 744
 745                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 746                 c.Assert(err, check.IsNil)
 747                 // Note: we don't close the file until after the test
 748                 // is done. Writes to unclosed files should persist.
 749                 defer f.Close()
 750
 751                 wg.Add(1)
 752                 go func() {
 753                         defer wg.Done()
 754                         for i := 0; i < len(buf); i += 5 {
 755                                 _, err := f.Write(buf[i : i+5])
 756                                 c.Assert(err, check.IsNil)
 757                         }
 758                 }()
 759         }
 760         wg.Wait()
 761
 762         m, err := s.fs.MarshalManifest(".")
 763         c.Check(err, check.IsNil)
 764         c.Logf("%q", m)
 765
 766         root, err := s.fs.Open("/")
 767         c.Assert(err, check.IsNil)
 768         defer root.Close()
 769         fi, err := root.Readdir(-1)
 770         c.Check(err, check.IsNil)
 771         c.Check(len(fi), check.Equals, 4)
 772
 773         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
 774         c.Assert(err, check.IsNil)
 775
 776         root, err = persisted.Open("/")
 777         c.Assert(err, check.IsNil)
 778         defer root.Close()
 779         fi, err = root.Readdir(-1)
 780         c.Check(err, check.IsNil)
 781         c.Check(len(fi), check.Equals, 4)
 782
 783         for name, content := range expect {
 784                 c.Logf("read %q", name)
 785                 f, err := persisted.Open(name)
 786                 c.Assert(err, check.IsNil)
 787                 defer f.Close()
 788                 buf, err := ioutil.ReadAll(f)
 789                 c.Check(err, check.IsNil)
 790                 c.Check(buf, check.DeepEquals, content)
 791         }
 792 }
 793
 794 func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
 795         var err error
 796         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 797         c.Assert(err, check.IsNil)
 798         for _, name := range []string{"dir", "dir/zerodir", "zero", "zero/zero"} {
 799                 err = s.fs.Mkdir(name, 0755)
 800                 c.Assert(err, check.IsNil)
 801         }
 802
 803         expect := map[string][]byte{
 804                 "0":                nil,
 805                 "00":               {},
 806                 "one":              {1},
 807                 "dir/0":            nil,
 808                 "dir/two":          {1, 2},
 809                 "dir/zero":         nil,
 810                 "dir/zerodir/zero": nil,
 811                 "zero/zero/zero":   nil,
 812         }
 813         for name, data := range expect {
 814                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 815                 c.Assert(err, check.IsNil)
 816                 if data != nil {
 817                         _, err := f.Write(data)
 818                         c.Assert(err, check.IsNil)
 819                 }
 820                 f.Close()
 821         }
 822
 823         m, err := s.fs.MarshalManifest(".")
 824         c.Check(err, check.IsNil)
 825         c.Logf("%q", m)
 826
 827         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
 828         c.Assert(err, check.IsNil)
 829
 830         for name, data := range expect {
 831                 _, err = persisted.Open("bogus-" + name)
 832                 c.Check(err, check.NotNil)
 833
 834                 f, err := persisted.Open(name)
 835                 c.Assert(err, check.IsNil)
 836
 837                 if data == nil {
 838                         data = []byte{}
 839                 }
 840                 buf, err := ioutil.ReadAll(f)
 841                 c.Check(err, check.IsNil)
 842                 c.Check(buf, check.DeepEquals, data)
 843         }
 844 }
 845
 846 func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
 847         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 848         c.Assert(err, check.IsNil)
 849
 850         f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
 851         c.Check(f, check.IsNil)
 852         c.Check(err, check.ErrorMatches, `file does not exist`)
 853
 854         f, err = fs.OpenFile("new", os.O_CREATE|os.O_RDONLY, 0)
 855         c.Assert(err, check.IsNil)
 856         defer f.Close()
 857         n, err := f.Write([]byte{1, 2, 3})
 858         c.Check(n, check.Equals, 0)
 859         c.Check(err, check.ErrorMatches, `read-only file`)
 860         n, err = f.Read(make([]byte, 1))
 861         c.Check(n, check.Equals, 0)
 862         c.Check(err, check.Equals, io.EOF)
 863         f, err = fs.OpenFile("new", os.O_RDWR, 0)
 864         c.Assert(err, check.IsNil)
 865         defer f.Close()
 866         _, err = f.Write([]byte{4, 5, 6})
 867         c.Check(err, check.IsNil)
 868         fi, err := f.Stat()
 869         c.Assert(err, check.IsNil)
 870         c.Check(fi.Size(), check.Equals, int64(3))
 871
 872         f, err = fs.OpenFile("new", os.O_TRUNC|os.O_RDWR, 0)
 873         c.Assert(err, check.IsNil)
 874         defer f.Close()
 875         pos, err := f.Seek(0, io.SeekEnd)
 876         c.Check(pos, check.Equals, int64(0))
 877         c.Check(err, check.IsNil)
 878         fi, err = f.Stat()
 879         c.Assert(err, check.IsNil)
 880         c.Check(fi.Size(), check.Equals, int64(0))
 881         fs.Remove("new")
 882
 883         buf := make([]byte, 64)
 884         f, err = fs.OpenFile("append", os.O_EXCL|os.O_CREATE|os.O_RDWR|os.O_APPEND, 0)
 885         c.Assert(err, check.IsNil)
 886         f.Write([]byte{1, 2, 3})
 887         f.Seek(0, io.SeekStart)
 888         n, _ = f.Read(buf[:1])
 889         c.Check(n, check.Equals, 1)
 890         c.Check(buf[:1], check.DeepEquals, []byte{1})
 891         pos, err = f.Seek(0, io.SeekCurrent)
 892         c.Assert(err, check.IsNil)
 893         c.Check(pos, check.Equals, int64(1))
 894         f.Write([]byte{4, 5, 6})
 895         pos, err = f.Seek(0, io.SeekCurrent)
 896         c.Assert(err, check.IsNil)
 897         c.Check(pos, check.Equals, int64(6))
 898         f.Seek(0, io.SeekStart)
 899         n, err = f.Read(buf)
 900         c.Check(buf[:n], check.DeepEquals, []byte{1, 2, 3, 4, 5, 6})
 901         c.Check(err, check.Equals, io.EOF)
 902         f.Close()
 903
 904         f, err = fs.OpenFile("append", os.O_RDWR|os.O_APPEND, 0)
 905         c.Assert(err, check.IsNil)
 906         pos, err = f.Seek(0, io.SeekCurrent)
 907         c.Check(pos, check.Equals, int64(0))
 908         c.Check(err, check.IsNil)
 909         f.Read(buf[:3])
 910         pos, _ = f.Seek(0, io.SeekCurrent)
 911         c.Check(pos, check.Equals, int64(3))
 912         f.Write([]byte{7, 8, 9})
 913         pos, err = f.Seek(0, io.SeekCurrent)
 914         c.Check(err, check.IsNil)
 915         c.Check(pos, check.Equals, int64(9))
 916         f.Close()
 917
 918         f, err = fs.OpenFile("wronly", os.O_CREATE|os.O_WRONLY, 0)
 919         c.Assert(err, check.IsNil)
 920         n, err = f.Write([]byte{3, 2, 1})
 921         c.Check(n, check.Equals, 3)
 922         c.Check(err, check.IsNil)
 923         pos, _ = f.Seek(0, io.SeekCurrent)
 924         c.Check(pos, check.Equals, int64(3))
 925         pos, _ = f.Seek(0, io.SeekStart)
 926         c.Check(pos, check.Equals, int64(0))
 927         n, err = f.Read(buf)
 928         c.Check(n, check.Equals, 0)
 929         c.Check(err, check.ErrorMatches, `.*O_WRONLY.*`)
 930         f, err = fs.OpenFile("wronly", os.O_RDONLY, 0)
 931         c.Assert(err, check.IsNil)
 932         n, _ = f.Read(buf)
 933         c.Check(buf[:n], check.DeepEquals, []byte{3, 2, 1})
 934
 935         f, err = fs.OpenFile("unsupported", os.O_CREATE|os.O_SYNC, 0)
 936         c.Check(f, check.IsNil)
 937         c.Check(err, check.NotNil)
 938
 939         f, err = fs.OpenFile("append", os.O_RDWR|os.O_WRONLY, 0)
 940         c.Check(f, check.IsNil)
 941         c.Check(err, check.ErrorMatches, `invalid flag.*`)
 942 }
 943
 944 func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
 945         maxBlockSize = 1024
 946         defer func() { maxBlockSize = 2 << 26 }()
 947
 948         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 949         c.Assert(err, check.IsNil)
 950         f, err := fs.OpenFile("50K", os.O_WRONLY|os.O_CREATE, 0)
 951         c.Assert(err, check.IsNil)
 952         defer f.Close()
 953
 954         data := make([]byte, 500)
 955         rand.Read(data)
 956
 957         for i := 0; i < 100; i++ {
 958                 n, err := f.Write(data)
 959                 c.Assert(n, check.Equals, len(data))
 960                 c.Assert(err, check.IsNil)
 961         }
 962
 963         currentMemExtents := func() (memExtents []int) {
 964                 for idx, e := range f.(*filehandle).inode.(*filenode).segments {
 965                         switch e.(type) {
 966                         case *memSegment:
 967                                 memExtents = append(memExtents, idx)
 968                         }
 969                 }
 970                 return
 971         }
 972         c.Check(currentMemExtents(), check.HasLen, 1)
 973
 974         m, err := fs.MarshalManifest(".")
 975         c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
 976         c.Check(err, check.IsNil)
 977         c.Check(currentMemExtents(), check.HasLen, 0)
 978 }
 979
 980 func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
 981         for _, txt := range []string{
 982                 "\n",
 983                 ".\n",
 984                 ". \n",
 985                 ". d41d8cd98f00b204e9800998ecf8427e+0\n",
 986                 ". d41d8cd98f00b204e9800998ecf8427e+0 \n",
 987                 ". 0:0:foo\n",
 988                 ".  0:0:foo\n",
 989                 ". 0:0:foo 0:0:bar\n",
 990                 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo\n",
 991                 ". d41d8cd98f00b204e9800998ecf8427e+0 :0:0:foo\n",
 992                 ". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
 993                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
 994                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
 995                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n",
 996                 "./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
 997         } {
 998                 c.Logf("<-%q", txt)
 999                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
1000                 c.Check(fs, check.IsNil)
1001                 c.Logf("-> %s", err)
1002                 c.Check(err, check.NotNil)
1003         }
1004 }
1005
1006 func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
1007         for _, txt := range []string{
1008                 "",
1009                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
1010                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
1011                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
1012                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
1013         } {
1014                 c.Logf("<-%q", txt)
1015                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
1016                 c.Check(err, check.IsNil)
1017                 c.Check(fs, check.NotNil)
1018         }
1019 }
1020
1021 func (s *CollectionFSSuite) checkMemSize(c *check.C, f File) {
1022         fn := f.(*filehandle).inode.(*filenode)
1023         var memsize int64
1024         for _, seg := range fn.segments {
1025                 if e, ok := seg.(*memSegment); ok {
1026                         memsize += int64(len(e.buf))
1027                 }
1028         }
1029         c.Check(fn.memsize, check.Equals, memsize)
1030 }
1031
1032 type CollectionFSUnitSuite struct{}
1033
1034 var _ = check.Suite(&CollectionFSUnitSuite{})
1035
1036 // expect ~2 seconds to load a manifest with 256K files
1037 func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
1038         if testing.Short() {
1039                 c.Skip("slow")
1040         }
1041
1042         const (
1043                 dirCount  = 512
1044                 fileCount = 512
1045         )
1046
1047         mb := bytes.NewBuffer(make([]byte, 0, 40000000))
1048         for i := 0; i < dirCount; i++ {
1049                 fmt.Fprintf(mb, "./dir%d", i)
1050                 for j := 0; j <= fileCount; j++ {
1051                         fmt.Fprintf(mb, " %032x+42+A%040x@%08x", j, j, j)
1052                 }
1053                 for j := 0; j < fileCount; j++ {
1054                         fmt.Fprintf(mb, " %d:%d:dir%d/file%d", j*42+21, 42, j, j)
1055                 }
1056                 mb.Write([]byte{'\n'})
1057         }
1058         coll := Collection{ManifestText: mb.String()}
1059         c.Logf("%s built", time.Now())
1060
1061         var memstats runtime.MemStats
1062         runtime.ReadMemStats(&memstats)
1063         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1064
1065         f, err := coll.FileSystem(nil, nil)
1066         c.Check(err, check.IsNil)
1067         c.Logf("%s loaded", time.Now())
1068         c.Check(f.Size(), check.Equals, int64(42*dirCount*fileCount))
1069
1070         for i := 0; i < dirCount; i++ {
1071                 for j := 0; j < fileCount; j++ {
1072                         f.Stat(fmt.Sprintf("./dir%d/dir%d/file%d", i, j, j))
1073                 }
1074         }
1075         c.Logf("%s Stat() x %d", time.Now(), dirCount*fileCount)
1076
1077         runtime.ReadMemStats(&memstats)
1078         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1079 }
1080
1081 // Gocheck boilerplate
1082 func Test(t *testing.T) {
1083         check.TestingT(t)
1084 }