sdk/go/arvados/collection_fs_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "math/rand"
  15         "net/http"
  16         "os"
  17         "regexp"
  18         "runtime"
  19         "sync"
  20         "testing"
  21         "time"
  22
  23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  24         check "gopkg.in/check.v1"
  25 )
  26
  27 var _ = check.Suite(&CollectionFSSuite{})
  28
  29 type keepClientStub struct {
  30         blocks map[string][]byte
  31         sync.RWMutex
  32 }
  33
  34 var errStub404 = errors.New("404 block not found")
  35
  36 func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error) {
  37         kcs.RLock()
  38         defer kcs.RUnlock()
  39         buf := kcs.blocks[locator[:32]]
  40         if buf == nil {
  41                 return 0, errStub404
  42         }
  43         return copy(p, buf[off:]), nil
  44 }
  45
  46 func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
  47         locator := fmt.Sprintf("%x+%d+A12345@abcde", md5.Sum(p), len(p))
  48         buf := make([]byte, len(p))
  49         copy(buf, p)
  50         kcs.Lock()
  51         defer kcs.Unlock()
  52         kcs.blocks[locator[:32]] = buf
  53         return locator, 1, nil
  54 }
  55
  56 type CollectionFSSuite struct {
  57         client *Client
  58         coll   Collection
  59         fs     CollectionFileSystem
  60         kc     keepClient
  61 }
  62
  63 func (s *CollectionFSSuite) SetUpTest(c *check.C) {
  64         s.client = NewClientFromEnv()
  65         err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
  66         c.Assert(err, check.IsNil)
  67         s.kc = &keepClientStub{
  68                 blocks: map[string][]byte{
  69                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  70                 }}
  71         s.fs, err = s.coll.FileSystem(s.client, s.kc)
  72         c.Assert(err, check.IsNil)
  73 }
  74
  75 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
  76         _, ok := s.fs.(http.FileSystem)
  77         c.Check(ok, check.Equals, true)
  78 }
  79
  80 func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
  81         f, err := s.fs.Open("/dir1")
  82         c.Assert(err, check.IsNil)
  83
  84         st, err := f.Stat()
  85         c.Assert(err, check.IsNil)
  86         c.Check(st.Size(), check.Equals, int64(2))
  87         c.Check(st.IsDir(), check.Equals, true)
  88
  89         fis, err := f.Readdir(0)
  90         c.Check(err, check.IsNil)
  91         c.Check(len(fis), check.Equals, 2)
  92         if len(fis) > 0 {
  93                 c.Check(fis[0].Size(), check.Equals, int64(3))
  94         }
  95 }
  96
  97 func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
  98         f, err := s.fs.Open("./dir1")
  99         c.Assert(err, check.IsNil)
 100
 101         fis, err := f.Readdir(1)
 102         c.Check(err, check.IsNil)
 103         c.Check(len(fis), check.Equals, 1)
 104         if len(fis) > 0 {
 105                 c.Check(fis[0].Size(), check.Equals, int64(3))
 106         }
 107
 108         fis, err = f.Readdir(1)
 109         c.Check(err, check.IsNil)
 110         c.Check(len(fis), check.Equals, 1)
 111         if len(fis) > 0 {
 112                 c.Check(fis[0].Size(), check.Equals, int64(3))
 113         }
 114
 115         fis, err = f.Readdir(1)
 116         c.Check(len(fis), check.Equals, 0)
 117         c.Check(err, check.NotNil)
 118         c.Check(err, check.Equals, io.EOF)
 119
 120         f, err = s.fs.Open("dir1")
 121         c.Assert(err, check.IsNil)
 122         fis, err = f.Readdir(1)
 123         c.Check(len(fis), check.Equals, 1)
 124         c.Assert(err, check.IsNil)
 125         fis, err = f.Readdir(2)
 126         c.Check(len(fis), check.Equals, 1)
 127         c.Assert(err, check.IsNil)
 128         fis, err = f.Readdir(2)
 129         c.Check(len(fis), check.Equals, 0)
 130         c.Assert(err, check.Equals, io.EOF)
 131 }
 132
 133 func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
 134         for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
 135                 f, err := s.fs.Open(path)
 136                 c.Assert(err, check.IsNil)
 137
 138                 st, err := f.Stat()
 139                 c.Assert(err, check.IsNil)
 140                 c.Check(st.Size(), check.Equals, int64(1))
 141                 c.Check(st.IsDir(), check.Equals, true)
 142         }
 143         for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
 144                 c.Logf("%q", path)
 145                 f, err := s.fs.Open(path)
 146                 c.Assert(err, check.IsNil)
 147
 148                 st, err := f.Stat()
 149                 c.Assert(err, check.IsNil)
 150                 c.Check(st.Size(), check.Equals, int64(2))
 151                 c.Check(st.IsDir(), check.Equals, true)
 152         }
 153 }
 154
 155 func (s *CollectionFSSuite) TestNotExist(c *check.C) {
 156         for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
 157                 f, err := s.fs.Open(path)
 158                 c.Assert(f, check.IsNil)
 159                 c.Assert(err, check.NotNil)
 160                 c.Assert(os.IsNotExist(err), check.Equals, true)
 161         }
 162 }
 163
 164 func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
 165         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDONLY, 0)
 166         c.Assert(err, check.IsNil)
 167         st, err := f.Stat()
 168         c.Assert(err, check.IsNil)
 169         c.Check(st.Size(), check.Equals, int64(3))
 170         n, err := f.Write([]byte("bar"))
 171         c.Check(n, check.Equals, 0)
 172         c.Check(err, check.Equals, ErrReadOnlyFile)
 173 }
 174
 175 func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
 176         f, err := s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE, 0)
 177         c.Assert(err, check.IsNil)
 178         st, err := f.Stat()
 179         c.Assert(err, check.IsNil)
 180         c.Check(st.Size(), check.Equals, int64(0))
 181
 182         n, err := f.Write([]byte("bar"))
 183         c.Check(n, check.Equals, 3)
 184         c.Check(err, check.IsNil)
 185
 186         c.Check(f.Close(), check.IsNil)
 187
 188         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 189         c.Check(f, check.IsNil)
 190         c.Assert(err, check.NotNil)
 191
 192         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR, 0)
 193         c.Assert(err, check.IsNil)
 194         st, err = f.Stat()
 195         c.Assert(err, check.IsNil)
 196         c.Check(st.Size(), check.Equals, int64(3))
 197
 198         c.Check(f.Close(), check.IsNil)
 199
 200         m, err := s.fs.MarshalManifest(".")
 201         c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
 202 }
 203
 204 func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
 205         maxBlockSize = 8
 206         defer func() { maxBlockSize = 2 << 26 }()
 207
 208         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 209         c.Assert(err, check.IsNil)
 210         defer f.Close()
 211         st, err := f.Stat()
 212         c.Assert(err, check.IsNil)
 213         c.Check(st.Size(), check.Equals, int64(3))
 214
 215         f2, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 216         c.Assert(err, check.IsNil)
 217         defer f2.Close()
 218
 219         buf := make([]byte, 64)
 220         n, err := f.Read(buf)
 221         c.Check(n, check.Equals, 3)
 222         c.Check(err, check.Equals, io.EOF)
 223         c.Check(string(buf[:3]), check.DeepEquals, "foo")
 224
 225         pos, err := f.Seek(-2, io.SeekCurrent)
 226         c.Check(pos, check.Equals, int64(1))
 227         c.Check(err, check.IsNil)
 228
 229         // Split a storedExtent in two, and insert a memExtent
 230         n, err = f.Write([]byte("*"))
 231         c.Check(n, check.Equals, 1)
 232         c.Check(err, check.IsNil)
 233
 234         pos, err = f.Seek(0, io.SeekCurrent)
 235         c.Check(pos, check.Equals, int64(2))
 236         c.Check(err, check.IsNil)
 237
 238         pos, err = f.Seek(0, io.SeekStart)
 239         c.Check(pos, check.Equals, int64(0))
 240         c.Check(err, check.IsNil)
 241
 242         rbuf, err := ioutil.ReadAll(f)
 243         c.Check(len(rbuf), check.Equals, 3)
 244         c.Check(err, check.IsNil)
 245         c.Check(string(rbuf), check.Equals, "f*o")
 246
 247         // Write multiple blocks in one call
 248         f.Seek(1, io.SeekStart)
 249         n, err = f.Write([]byte("0123456789abcdefg"))
 250         c.Check(n, check.Equals, 17)
 251         c.Check(err, check.IsNil)
 252         pos, err = f.Seek(0, io.SeekCurrent)
 253         c.Check(pos, check.Equals, int64(18))
 254         pos, err = f.Seek(-18, io.SeekCurrent)
 255         c.Check(err, check.IsNil)
 256         n, err = io.ReadFull(f, buf)
 257         c.Check(n, check.Equals, 18)
 258         c.Check(err, check.Equals, io.ErrUnexpectedEOF)
 259         c.Check(string(buf[:n]), check.Equals, "f0123456789abcdefg")
 260
 261         buf2, err := ioutil.ReadAll(f2)
 262         c.Check(err, check.IsNil)
 263         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 264
 265         // truncate to current size
 266         err = f.Truncate(18)
 267         f2.Seek(0, io.SeekStart)
 268         buf2, err = ioutil.ReadAll(f2)
 269         c.Check(err, check.IsNil)
 270         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 271
 272         // shrink to zero some data
 273         f.Truncate(15)
 274         f2.Seek(0, io.SeekStart)
 275         buf2, err = ioutil.ReadAll(f2)
 276         c.Check(err, check.IsNil)
 277         c.Check(string(buf2), check.Equals, "f0123456789abcd")
 278
 279         // grow to partial block/extent
 280         f.Truncate(20)
 281         f2.Seek(0, io.SeekStart)
 282         buf2, err = ioutil.ReadAll(f2)
 283         c.Check(err, check.IsNil)
 284         c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
 285
 286         f.Truncate(0)
 287         f2.Seek(0, io.SeekStart)
 288         f2.Write([]byte("12345678abcdefghijkl"))
 289
 290         // grow to block/extent boundary
 291         f.Truncate(64)
 292         f2.Seek(0, io.SeekStart)
 293         buf2, err = ioutil.ReadAll(f2)
 294         c.Check(err, check.IsNil)
 295         c.Check(len(buf2), check.Equals, 64)
 296         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 8)
 297
 298         // shrink to block/extent boundary
 299         err = f.Truncate(32)
 300         f2.Seek(0, io.SeekStart)
 301         buf2, err = ioutil.ReadAll(f2)
 302         c.Check(err, check.IsNil)
 303         c.Check(len(buf2), check.Equals, 32)
 304         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 4)
 305
 306         // shrink to partial block/extent
 307         err = f.Truncate(15)
 308         f2.Seek(0, io.SeekStart)
 309         buf2, err = ioutil.ReadAll(f2)
 310         c.Check(err, check.IsNil)
 311         c.Check(string(buf2), check.Equals, "12345678abcdefg")
 312         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 2)
 313
 314         // Force flush to ensure the block "12345678" gets stored, so
 315         // we know what to expect in the final manifest below.
 316         _, err = s.fs.MarshalManifest(".")
 317         c.Check(err, check.IsNil)
 318
 319         // Truncate to size=3 while f2's ptr is at 15
 320         err = f.Truncate(3)
 321         c.Check(err, check.IsNil)
 322         buf2, err = ioutil.ReadAll(f2)
 323         c.Check(err, check.IsNil)
 324         c.Check(string(buf2), check.Equals, "")
 325         f2.Seek(0, io.SeekStart)
 326         buf2, err = ioutil.ReadAll(f2)
 327         c.Check(err, check.IsNil)
 328         c.Check(string(buf2), check.Equals, "123")
 329         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 1)
 330
 331         m, err := s.fs.MarshalManifest(".")
 332         c.Check(err, check.IsNil)
 333         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 334         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 25d55ad283aa400af464c76d713c07ad+8 3:3:bar 6:3:foo\n")
 335 }
 336
 337 func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
 338         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 339         c.Assert(err, check.IsNil)
 340         f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
 341         c.Assert(err, check.IsNil)
 342         defer f.Close()
 343
 344         checkSize := func(size int64) {
 345                 fi, err := f.Stat()
 346                 c.Check(fi.Size(), check.Equals, size)
 347
 348                 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
 349                 c.Assert(err, check.IsNil)
 350                 defer f.Close()
 351                 fi, err = f.Stat()
 352                 c.Check(fi.Size(), check.Equals, size)
 353                 pos, err := f.Seek(0, io.SeekEnd)
 354                 c.Check(pos, check.Equals, size)
 355         }
 356
 357         f.Seek(2, io.SeekEnd)
 358         checkSize(0)
 359         f.Write([]byte{1})
 360         checkSize(3)
 361
 362         f.Seek(2, io.SeekCurrent)
 363         checkSize(3)
 364         f.Write([]byte{})
 365         checkSize(5)
 366
 367         f.Seek(8, io.SeekStart)
 368         checkSize(5)
 369         n, err := f.Read(make([]byte, 1))
 370         c.Check(n, check.Equals, 0)
 371         c.Check(err, check.Equals, io.EOF)
 372         checkSize(5)
 373         f.Write([]byte{1, 2, 3})
 374         checkSize(11)
 375 }
 376
 377 func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
 378         maxBlockSize = 8
 379         defer func() { maxBlockSize = 2 << 26 }()
 380
 381         var err error
 382         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 383         c.Assert(err, check.IsNil)
 384         for _, name := range []string{"foo", "bar", "baz"} {
 385                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 386                 c.Assert(err, check.IsNil)
 387                 f.Write([]byte(name))
 388                 f.Close()
 389         }
 390
 391         m, err := s.fs.MarshalManifest(".")
 392         c.Check(err, check.IsNil)
 393         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 394         c.Check(m, check.Equals, ". c3c23db5285662ef7172373df0003206+6 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar 3:3:baz 6:3:foo\n")
 395 }
 396
 397 func (s *CollectionFSSuite) TestMkdir(c *check.C) {
 398         err := s.fs.Mkdir("foo/bar", 0755)
 399         c.Check(err, check.Equals, os.ErrNotExist)
 400
 401         f, err := s.fs.OpenFile("foo/bar", os.O_CREATE, 0)
 402         c.Check(err, check.Equals, os.ErrNotExist)
 403
 404         err = s.fs.Mkdir("foo", 0755)
 405         c.Check(err, check.IsNil)
 406
 407         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_WRONLY, 0)
 408         c.Check(err, check.IsNil)
 409         if err == nil {
 410                 defer f.Close()
 411                 f.Write([]byte("foo"))
 412         }
 413
 414         // mkdir fails if a file already exists with that name
 415         err = s.fs.Mkdir("foo/bar", 0755)
 416         c.Check(err, check.NotNil)
 417
 418         err = s.fs.Remove("foo/bar")
 419         c.Check(err, check.IsNil)
 420
 421         // mkdir succeds after the file is deleted
 422         err = s.fs.Mkdir("foo/bar", 0755)
 423         c.Check(err, check.IsNil)
 424
 425         // creating a file in a nonexistent subdir should still fail
 426         f, err = s.fs.OpenFile("foo/bar/baz/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
 427         c.Check(err, check.Equals, os.ErrNotExist)
 428
 429         f, err = s.fs.OpenFile("foo/bar/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
 430         c.Check(err, check.IsNil)
 431         if err == nil {
 432                 defer f.Close()
 433                 f.Write([]byte("foo"))
 434         }
 435
 436         // creating foo/bar as a regular file should fail
 437         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, 0)
 438         c.Check(err, check.NotNil)
 439
 440         // creating foo/bar as a directory should fail
 441         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, os.ModeDir)
 442         c.Check(err, check.NotNil)
 443         err = s.fs.Mkdir("foo/bar", 0755)
 444         c.Check(err, check.NotNil)
 445
 446         m, err := s.fs.MarshalManifest(".")
 447         c.Check(err, check.IsNil)
 448         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 449         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 3:3:bar 0:3:foo\n./foo/bar acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 450 }
 451
 452 func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
 453         maxBlockSize = 8
 454         defer func() { maxBlockSize = 2 << 26 }()
 455
 456         var wg sync.WaitGroup
 457         for n := 0; n < 128; n++ {
 458                 wg.Add(1)
 459                 go func() {
 460                         defer wg.Done()
 461                         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 462                         c.Assert(err, check.IsNil)
 463                         defer f.Close()
 464                         for i := 0; i < 6502; i++ {
 465                                 switch rand.Int() & 3 {
 466                                 case 0:
 467                                         f.Truncate(int64(rand.Intn(64)))
 468                                 case 1:
 469                                         f.Seek(int64(rand.Intn(64)), io.SeekStart)
 470                                 case 2:
 471                                         _, err := f.Write([]byte("beep boop"))
 472                                         c.Check(err, check.IsNil)
 473                                 case 3:
 474                                         _, err := ioutil.ReadAll(f)
 475                                         c.Check(err, check.IsNil)
 476                                 }
 477                         }
 478                 }()
 479         }
 480         wg.Wait()
 481
 482         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 483         c.Assert(err, check.IsNil)
 484         defer f.Close()
 485         buf, err := ioutil.ReadAll(f)
 486         c.Check(err, check.IsNil)
 487         c.Logf("after lots of random r/w/seek/trunc, buf is %q", buf)
 488 }
 489
 490 func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
 491         maxBlockSize = 40
 492         defer func() { maxBlockSize = 2 << 26 }()
 493
 494         var err error
 495         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 496         c.Assert(err, check.IsNil)
 497
 498         const nfiles = 256
 499         const ngoroutines = 256
 500
 501         var wg sync.WaitGroup
 502         for n := 0; n < nfiles; n++ {
 503                 wg.Add(1)
 504                 go func(n int) {
 505                         defer wg.Done()
 506                         expect := make([]byte, 0, 64)
 507                         wbytes := []byte("there's no simple explanation for anything important that any of us do")
 508                         f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 509                         c.Assert(err, check.IsNil)
 510                         defer f.Close()
 511                         for i := 0; i < ngoroutines; i++ {
 512                                 trunc := rand.Intn(65)
 513                                 woff := rand.Intn(trunc + 1)
 514                                 wbytes = wbytes[:rand.Intn(64-woff+1)]
 515                                 for buf, i := expect[:cap(expect)], len(expect); i < trunc; i++ {
 516                                         buf[i] = 0
 517                                 }
 518                                 expect = expect[:trunc]
 519                                 if trunc < woff+len(wbytes) {
 520                                         expect = expect[:woff+len(wbytes)]
 521                                 }
 522                                 copy(expect[woff:], wbytes)
 523                                 f.Truncate(int64(trunc))
 524                                 pos, err := f.Seek(int64(woff), io.SeekStart)
 525                                 c.Check(pos, check.Equals, int64(woff))
 526                                 c.Check(err, check.IsNil)
 527                                 n, err := f.Write(wbytes)
 528                                 c.Check(n, check.Equals, len(wbytes))
 529                                 c.Check(err, check.IsNil)
 530                                 pos, err = f.Seek(0, io.SeekStart)
 531                                 c.Check(pos, check.Equals, int64(0))
 532                                 c.Check(err, check.IsNil)
 533                                 buf, err := ioutil.ReadAll(f)
 534                                 c.Check(string(buf), check.Equals, string(expect))
 535                                 c.Check(err, check.IsNil)
 536                         }
 537                         s.checkMemSize(c, f)
 538                 }(n)
 539         }
 540         wg.Wait()
 541
 542         root, err := s.fs.Open("/")
 543         c.Assert(err, check.IsNil)
 544         defer root.Close()
 545         fi, err := root.Readdir(-1)
 546         c.Check(err, check.IsNil)
 547         c.Check(len(fi), check.Equals, nfiles)
 548
 549         _, err = s.fs.MarshalManifest(".")
 550         c.Check(err, check.IsNil)
 551         // TODO: check manifest content
 552 }
 553
 554 func (s *CollectionFSSuite) TestRemove(c *check.C) {
 555         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 556         c.Assert(err, check.IsNil)
 557         err = fs.Mkdir("dir0", 0755)
 558         c.Assert(err, check.IsNil)
 559         err = fs.Mkdir("dir1", 0755)
 560         c.Assert(err, check.IsNil)
 561         err = fs.Mkdir("dir1/dir2", 0755)
 562         c.Assert(err, check.IsNil)
 563         err = fs.Mkdir("dir1/dir3", 0755)
 564         c.Assert(err, check.IsNil)
 565
 566         err = fs.Remove("dir0")
 567         c.Check(err, check.IsNil)
 568         err = fs.Remove("dir0")
 569         c.Check(err, check.Equals, os.ErrNotExist)
 570
 571         err = fs.Remove("dir1/dir2/.")
 572         c.Check(err, check.Equals, ErrInvalidArgument)
 573         err = fs.Remove("dir1/dir2/..")
 574         c.Check(err, check.Equals, ErrInvalidArgument)
 575         err = fs.Remove("dir1")
 576         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
 577         err = fs.Remove("dir1/dir2/../../../dir1")
 578         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
 579         err = fs.Remove("dir1/dir3/")
 580         c.Check(err, check.IsNil)
 581         err = fs.RemoveAll("dir1")
 582         c.Check(err, check.IsNil)
 583         err = fs.RemoveAll("dir1")
 584         c.Check(err, check.IsNil)
 585 }
 586
 587 func (s *CollectionFSSuite) TestRename(c *check.C) {
 588         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 589         c.Assert(err, check.IsNil)
 590         const (
 591                 outer = 16
 592                 inner = 16
 593         )
 594         for i := 0; i < outer; i++ {
 595                 err = fs.Mkdir(fmt.Sprintf("dir%d", i), 0755)
 596                 c.Assert(err, check.IsNil)
 597                 for j := 0; j < inner; j++ {
 598                         err = fs.Mkdir(fmt.Sprintf("dir%d/dir%d", i, j), 0755)
 599                         c.Assert(err, check.IsNil)
 600                         for _, fnm := range []string{
 601                                 fmt.Sprintf("dir%d/file%d", i, j),
 602                                 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
 603                         } {
 604                                 f, err := fs.OpenFile(fnm, os.O_CREATE|os.O_WRONLY, 0755)
 605                                 c.Assert(err, check.IsNil)
 606                                 _, err = f.Write([]byte("beep"))
 607                                 c.Assert(err, check.IsNil)
 608                                 f.Close()
 609                         }
 610                 }
 611         }
 612         var wg sync.WaitGroup
 613         for i := 0; i < outer; i++ {
 614                 for j := 0; j < inner; j++ {
 615                         wg.Add(1)
 616                         go func(i, j int) {
 617                                 defer wg.Done()
 618                                 oldname := fmt.Sprintf("dir%d/dir%d/file%d", i, j, j)
 619                                 newname := fmt.Sprintf("dir%d/newfile%d", i, inner-j-1)
 620                                 _, err := fs.Open(newname)
 621                                 c.Check(err, check.Equals, os.ErrNotExist)
 622                                 err = fs.Rename(oldname, newname)
 623                                 c.Check(err, check.IsNil)
 624                                 f, err := fs.Open(newname)
 625                                 c.Check(err, check.IsNil)
 626                                 f.Close()
 627                         }(i, j)
 628
 629                         wg.Add(1)
 630                         go func(i, j int) {
 631                                 defer wg.Done()
 632                                 // oldname does not exist
 633                                 err := fs.Rename(
 634                                         fmt.Sprintf("dir%d/dir%d/missing", i, j),
 635                                         fmt.Sprintf("dir%d/dir%d/file%d", outer-i-1, j, j))
 636                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 637
 638                                 // newname parent dir does not exist
 639                                 err = fs.Rename(
 640                                         fmt.Sprintf("dir%d/dir%d", i, j),
 641                                         fmt.Sprintf("dir%d/missing/irrelevant", outer-i-1))
 642                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 643
 644                                 // oldname parent dir is a file
 645                                 err = fs.Rename(
 646                                         fmt.Sprintf("dir%d/file%d/patherror", i, j),
 647                                         fmt.Sprintf("dir%d/irrelevant", i))
 648                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 649
 650                                 // newname parent dir is a file
 651                                 err = fs.Rename(
 652                                         fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
 653                                         fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
 654                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
 655                         }(i, j)
 656                 }
 657         }
 658         wg.Wait()
 659
 660         f, err := fs.OpenFile("dir1/newfile3", 0, 0)
 661         c.Assert(err, check.IsNil)
 662         c.Check(f.Size(), check.Equals, int64(4))
 663         buf, err := ioutil.ReadAll(f)
 664         c.Check(buf, check.DeepEquals, []byte("beep"))
 665         c.Check(err, check.IsNil)
 666         _, err = fs.Open("dir1/dir1/file1")
 667         c.Check(err, check.Equals, os.ErrNotExist)
 668 }
 669
 670 func (s *CollectionFSSuite) TestPersist(c *check.C) {
 671         maxBlockSize = 1024
 672         defer func() { maxBlockSize = 2 << 26 }()
 673
 674         var err error
 675         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 676         c.Assert(err, check.IsNil)
 677         err = s.fs.Mkdir("d:r", 0755)
 678         c.Assert(err, check.IsNil)
 679
 680         expect := map[string][]byte{}
 681
 682         var wg sync.WaitGroup
 683         for _, name := range []string{"random 1", "random:2", "random\\3", "d:r/random4"} {
 684                 buf := make([]byte, 500)
 685                 rand.Read(buf)
 686                 expect[name] = buf
 687
 688                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 689                 c.Assert(err, check.IsNil)
 690                 // Note: we don't close the file until after the test
 691                 // is done. Writes to unclosed files should persist.
 692                 defer f.Close()
 693
 694                 wg.Add(1)
 695                 go func() {
 696                         defer wg.Done()
 697                         for i := 0; i < len(buf); i += 5 {
 698                                 _, err := f.Write(buf[i : i+5])
 699                                 c.Assert(err, check.IsNil)
 700                         }
 701                 }()
 702         }
 703         wg.Wait()
 704
 705         m, err := s.fs.MarshalManifest(".")
 706         c.Check(err, check.IsNil)
 707         c.Logf("%q", m)
 708
 709         root, err := s.fs.Open("/")
 710         c.Assert(err, check.IsNil)
 711         defer root.Close()
 712         fi, err := root.Readdir(-1)
 713         c.Check(err, check.IsNil)
 714         c.Check(len(fi), check.Equals, 4)
 715
 716         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
 717         c.Assert(err, check.IsNil)
 718
 719         root, err = persisted.Open("/")
 720         c.Assert(err, check.IsNil)
 721         defer root.Close()
 722         fi, err = root.Readdir(-1)
 723         c.Check(err, check.IsNil)
 724         c.Check(len(fi), check.Equals, 4)
 725
 726         for name, content := range expect {
 727                 c.Logf("read %q", name)
 728                 f, err := persisted.Open(name)
 729                 c.Assert(err, check.IsNil)
 730                 defer f.Close()
 731                 buf, err := ioutil.ReadAll(f)
 732                 c.Check(err, check.IsNil)
 733                 c.Check(buf, check.DeepEquals, content)
 734         }
 735 }
 736
 737 func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
 738         var err error
 739         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
 740         c.Assert(err, check.IsNil)
 741         for _, name := range []string{"dir", "dir/zerodir", "zero", "zero/zero"} {
 742                 err = s.fs.Mkdir(name, 0755)
 743                 c.Assert(err, check.IsNil)
 744         }
 745
 746         expect := map[string][]byte{
 747                 "0":                nil,
 748                 "00":               []byte{},
 749                 "one":              []byte{1},
 750                 "dir/0":            nil,
 751                 "dir/two":          []byte{1, 2},
 752                 "dir/zero":         nil,
 753                 "dir/zerodir/zero": nil,
 754                 "zero/zero/zero":   nil,
 755         }
 756         for name, data := range expect {
 757                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 758                 c.Assert(err, check.IsNil)
 759                 if data != nil {
 760                         _, err := f.Write(data)
 761                         c.Assert(err, check.IsNil)
 762                 }
 763                 f.Close()
 764         }
 765
 766         m, err := s.fs.MarshalManifest(".")
 767         c.Check(err, check.IsNil)
 768         c.Logf("%q", m)
 769
 770         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
 771         c.Assert(err, check.IsNil)
 772
 773         for name, data := range expect {
 774                 f, err := persisted.Open("bogus-" + name)
 775                 c.Check(err, check.NotNil)
 776
 777                 f, err = persisted.Open(name)
 778                 c.Assert(err, check.IsNil)
 779
 780                 if data == nil {
 781                         data = []byte{}
 782                 }
 783                 buf, err := ioutil.ReadAll(f)
 784                 c.Check(err, check.IsNil)
 785                 c.Check(buf, check.DeepEquals, data)
 786         }
 787 }
 788
 789 func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
 790         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 791         c.Assert(err, check.IsNil)
 792
 793         f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
 794         c.Check(f, check.IsNil)
 795         c.Check(err, check.ErrorMatches, `file does not exist`)
 796
 797         f, err = fs.OpenFile("new", os.O_CREATE|os.O_RDONLY, 0)
 798         c.Assert(err, check.IsNil)
 799         defer f.Close()
 800         n, err := f.Write([]byte{1, 2, 3})
 801         c.Check(n, check.Equals, 0)
 802         c.Check(err, check.ErrorMatches, `read-only file`)
 803         n, err = f.Read(make([]byte, 1))
 804         c.Check(n, check.Equals, 0)
 805         c.Check(err, check.Equals, io.EOF)
 806         f, err = fs.OpenFile("new", os.O_RDWR, 0)
 807         c.Assert(err, check.IsNil)
 808         defer f.Close()
 809         _, err = f.Write([]byte{4, 5, 6})
 810         c.Check(err, check.IsNil)
 811         fi, err := f.Stat()
 812         c.Assert(err, check.IsNil)
 813         c.Check(fi.Size(), check.Equals, int64(3))
 814
 815         f, err = fs.OpenFile("new", os.O_TRUNC|os.O_RDWR, 0)
 816         c.Assert(err, check.IsNil)
 817         defer f.Close()
 818         pos, err := f.Seek(0, io.SeekEnd)
 819         c.Check(pos, check.Equals, int64(0))
 820         c.Check(err, check.IsNil)
 821         fi, err = f.Stat()
 822         c.Assert(err, check.IsNil)
 823         c.Check(fi.Size(), check.Equals, int64(0))
 824         fs.Remove("new")
 825
 826         buf := make([]byte, 64)
 827         f, err = fs.OpenFile("append", os.O_EXCL|os.O_CREATE|os.O_RDWR|os.O_APPEND, 0)
 828         c.Assert(err, check.IsNil)
 829         f.Write([]byte{1, 2, 3})
 830         f.Seek(0, io.SeekStart)
 831         n, _ = f.Read(buf[:1])
 832         c.Check(n, check.Equals, 1)
 833         c.Check(buf[:1], check.DeepEquals, []byte{1})
 834         pos, err = f.Seek(0, io.SeekCurrent)
 835         c.Check(pos, check.Equals, int64(1))
 836         f.Write([]byte{4, 5, 6})
 837         pos, err = f.Seek(0, io.SeekCurrent)
 838         c.Check(pos, check.Equals, int64(6))
 839         f.Seek(0, io.SeekStart)
 840         n, err = f.Read(buf)
 841         c.Check(buf[:n], check.DeepEquals, []byte{1, 2, 3, 4, 5, 6})
 842         c.Check(err, check.Equals, io.EOF)
 843         f.Close()
 844
 845         f, err = fs.OpenFile("append", os.O_RDWR|os.O_APPEND, 0)
 846         c.Assert(err, check.IsNil)
 847         pos, err = f.Seek(0, io.SeekCurrent)
 848         c.Check(pos, check.Equals, int64(0))
 849         c.Check(err, check.IsNil)
 850         f.Read(buf[:3])
 851         pos, _ = f.Seek(0, io.SeekCurrent)
 852         c.Check(pos, check.Equals, int64(3))
 853         f.Write([]byte{7, 8, 9})
 854         pos, err = f.Seek(0, io.SeekCurrent)
 855         c.Check(pos, check.Equals, int64(9))
 856         f.Close()
 857
 858         f, err = fs.OpenFile("wronly", os.O_CREATE|os.O_WRONLY, 0)
 859         c.Assert(err, check.IsNil)
 860         n, err = f.Write([]byte{3, 2, 1})
 861         c.Check(n, check.Equals, 3)
 862         c.Check(err, check.IsNil)
 863         pos, _ = f.Seek(0, io.SeekCurrent)
 864         c.Check(pos, check.Equals, int64(3))
 865         pos, _ = f.Seek(0, io.SeekStart)
 866         c.Check(pos, check.Equals, int64(0))
 867         n, err = f.Read(buf)
 868         c.Check(n, check.Equals, 0)
 869         c.Check(err, check.ErrorMatches, `.*O_WRONLY.*`)
 870         f, err = fs.OpenFile("wronly", os.O_RDONLY, 0)
 871         c.Assert(err, check.IsNil)
 872         n, _ = f.Read(buf)
 873         c.Check(buf[:n], check.DeepEquals, []byte{3, 2, 1})
 874
 875         f, err = fs.OpenFile("unsupported", os.O_CREATE|os.O_SYNC, 0)
 876         c.Check(f, check.IsNil)
 877         c.Check(err, check.NotNil)
 878
 879         f, err = fs.OpenFile("append", os.O_RDWR|os.O_WRONLY, 0)
 880         c.Check(f, check.IsNil)
 881         c.Check(err, check.ErrorMatches, `invalid flag.*`)
 882 }
 883
 884 func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
 885         maxBlockSize = 1024
 886         defer func() { maxBlockSize = 2 << 26 }()
 887
 888         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
 889         c.Assert(err, check.IsNil)
 890         f, err := fs.OpenFile("50K", os.O_WRONLY|os.O_CREATE, 0)
 891         c.Assert(err, check.IsNil)
 892         defer f.Close()
 893
 894         data := make([]byte, 500)
 895         rand.Read(data)
 896
 897         for i := 0; i < 100; i++ {
 898                 n, err := f.Write(data)
 899                 c.Assert(n, check.Equals, len(data))
 900                 c.Assert(err, check.IsNil)
 901         }
 902
 903         currentMemExtents := func() (memExtents []int) {
 904                 for idx, e := range f.(*filehandle).inode.(*filenode).segments {
 905                         switch e.(type) {
 906                         case *memSegment:
 907                                 memExtents = append(memExtents, idx)
 908                         }
 909                 }
 910                 return
 911         }
 912         c.Check(currentMemExtents(), check.HasLen, 1)
 913
 914         m, err := fs.MarshalManifest(".")
 915         c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
 916         c.Check(err, check.IsNil)
 917         c.Check(currentMemExtents(), check.HasLen, 0)
 918 }
 919
 920 func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
 921         for _, txt := range []string{
 922                 "\n",
 923                 ".\n",
 924                 ". \n",
 925                 ". d41d8cd98f00b204e9800998ecf8427e+0\n",
 926                 ". d41d8cd98f00b204e9800998ecf8427e+0 \n",
 927                 ". 0:0:foo\n",
 928                 ".  0:0:foo\n",
 929                 ". 0:0:foo 0:0:bar\n",
 930                 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo\n",
 931                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo:bar\n",
 932                 ". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
 933                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
 934                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
 935                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n",
 936                 "./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
 937         } {
 938                 c.Logf("<-%q", txt)
 939                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
 940                 c.Check(fs, check.IsNil)
 941                 c.Logf("-> %s", err)
 942                 c.Check(err, check.NotNil)
 943         }
 944 }
 945
 946 func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
 947         for _, txt := range []string{
 948                 "",
 949                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
 950                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
 951                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
 952                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
 953         } {
 954                 c.Logf("<-%q", txt)
 955                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
 956                 c.Check(err, check.IsNil)
 957                 c.Check(fs, check.NotNil)
 958         }
 959 }
 960
 961 func (s *CollectionFSSuite) checkMemSize(c *check.C, f File) {
 962         fn := f.(*filehandle).inode.(*filenode)
 963         var memsize int64
 964         for _, seg := range fn.segments {
 965                 if e, ok := seg.(*memSegment); ok {
 966                         memsize += int64(len(e.buf))
 967                 }
 968         }
 969         c.Check(fn.memsize, check.Equals, memsize)
 970 }
 971
 972 type CollectionFSUnitSuite struct{}
 973
 974 var _ = check.Suite(&CollectionFSUnitSuite{})
 975
 976 // expect ~2 seconds to load a manifest with 256K files
 977 func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
 978         const (
 979                 dirCount  = 512
 980                 fileCount = 512
 981         )
 982
 983         mb := bytes.NewBuffer(make([]byte, 0, 40000000))
 984         for i := 0; i < dirCount; i++ {
 985                 fmt.Fprintf(mb, "./dir%d", i)
 986                 for j := 0; j <= fileCount; j++ {
 987                         fmt.Fprintf(mb, " %032x+42+A%040x@%08x", j, j, j)
 988                 }
 989                 for j := 0; j < fileCount; j++ {
 990                         fmt.Fprintf(mb, " %d:%d:dir%d/file%d", j*42+21, 42, j, j)
 991                 }
 992                 mb.Write([]byte{'\n'})
 993         }
 994         coll := Collection{ManifestText: mb.String()}
 995         c.Logf("%s built", time.Now())
 996
 997         var memstats runtime.MemStats
 998         runtime.ReadMemStats(&memstats)
 999         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1000
1001         f, err := coll.FileSystem(nil, nil)
1002         c.Check(err, check.IsNil)
1003         c.Logf("%s loaded", time.Now())
1004
1005         for i := 0; i < dirCount; i++ {
1006                 for j := 0; j < fileCount; j++ {
1007                         f.Stat(fmt.Sprintf("./dir%d/dir%d/file%d", i, j, j))
1008                 }
1009         }
1010         c.Logf("%s Stat() x %d", time.Now(), dirCount*fileCount)
1011
1012         runtime.ReadMemStats(&memstats)
1013         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1014 }
1015
1016 // Gocheck boilerplate
1017 func Test(t *testing.T) {
1018         check.TestingT(t)
1019 }