1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
27 check "gopkg.in/check.v1"
30 var _ = check.Suite(&CollectionFSSuite{})
32 type keepClientStub struct {
33 blocks map[string][]byte
34 refreshable map[string]bool
35 onWrite func(bufcopy []byte) // called from WriteBlock, before acquiring lock
36 authToken string // client's auth token (used for signing locators)
37 sigkey string // blob signing key
38 sigttl time.Duration // blob signing ttl
42 var errStub404 = errors.New("404 block not found")
44 func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error) {
47 buf := kcs.blocks[locator[:32]]
51 return copy(p, buf[off:]), nil
54 func (kcs *keepClientStub) BlockWrite(_ context.Context, opts BlockWriteOptions) (BlockWriteResponse, error) {
56 panic("oops, stub is not made for this")
58 locator := SignLocator(fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)), kcs.authToken, time.Now().Add(kcs.sigttl), kcs.sigttl, []byte(kcs.sigkey))
59 buf := make([]byte, len(opts.Data))
61 if kcs.onWrite != nil {
64 for _, sc := range opts.StorageClasses {
66 return BlockWriteResponse{}, fmt.Errorf("stub does not write storage class %q", sc)
71 kcs.blocks[locator[:32]] = buf
72 return BlockWriteResponse{Locator: locator, Replicas: 1}, nil
75 var reRemoteSignature = regexp.MustCompile(`\+[AR][^+]*`)
77 func (kcs *keepClientStub) LocalLocator(locator string) (string, error) {
78 if strings.Contains(locator, "+A") {
83 if strings.Contains(locator, "+R") {
84 if len(locator) < 32 {
85 return "", fmt.Errorf("bad locator: %q", locator)
87 if _, ok := kcs.blocks[locator[:32]]; !ok && !kcs.refreshable[locator[:32]] {
88 return "", fmt.Errorf("kcs.refreshable[%q]==false", locator)
91 locator = reRemoteSignature.ReplaceAllLiteralString(locator, "")
92 locator = SignLocator(locator, kcs.authToken, time.Now().Add(kcs.sigttl), kcs.sigttl, []byte(kcs.sigkey))
96 type CollectionFSSuite struct {
99 fs CollectionFileSystem
103 func (s *CollectionFSSuite) SetUpTest(c *check.C) {
104 s.client = NewClientFromEnv()
105 err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+fixtureFooAndBarFilesInDirUUID, nil, nil)
106 c.Assert(err, check.IsNil)
107 s.kc = &keepClientStub{
108 blocks: map[string][]byte{
109 "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
111 sigkey: fixtureBlobSigningKey,
112 sigttl: fixtureBlobSigningTTL,
113 authToken: fixtureActiveToken,
115 s.fs, err = s.coll.FileSystem(s.client, s.kc)
116 c.Assert(err, check.IsNil)
119 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
120 _, ok := s.fs.(http.FileSystem)
121 c.Check(ok, check.Equals, true)
124 func (s *CollectionFSSuite) TestUnattainableStorageClasses(c *check.C) {
125 fs, err := (&Collection{
126 StorageClassesDesired: []string{"unobtainium"},
127 }).FileSystem(s.client, s.kc)
128 c.Assert(err, check.IsNil)
130 f, err := fs.OpenFile("/foo", os.O_CREATE|os.O_WRONLY, 0777)
131 c.Assert(err, check.IsNil)
132 _, err = f.Write([]byte("food"))
133 c.Assert(err, check.IsNil)
135 c.Assert(err, check.IsNil)
136 _, err = fs.MarshalManifest(".")
137 c.Assert(err, check.ErrorMatches, `.*stub does not write storage class \"unobtainium\"`)
140 func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
141 fs, err := (&Collection{
142 ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
143 }).FileSystem(s.client, s.kc)
144 c.Assert(err, check.IsNil)
146 f, err := fs.Open("/foo:foo")
147 c.Assert(err, check.IsNil)
149 fis, err := f.Readdir(0)
150 c.Check(err, check.IsNil)
151 c.Check(len(fis), check.Equals, 1)
152 c.Check(fis[0].Name(), check.Equals, "bar:bar")
155 func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
156 f, err := s.fs.Open("/dir1")
157 c.Assert(err, check.IsNil)
160 c.Assert(err, check.IsNil)
161 c.Check(st.Size(), check.Equals, int64(2))
162 c.Check(st.IsDir(), check.Equals, true)
164 fis, err := f.Readdir(0)
165 c.Check(err, check.IsNil)
166 c.Check(len(fis), check.Equals, 2)
168 c.Check(fis[0].Size(), check.Equals, int64(3))
172 func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
173 f, err := s.fs.Open("./dir1")
174 c.Assert(err, check.IsNil)
176 fis, err := f.Readdir(1)
177 c.Check(err, check.IsNil)
178 c.Check(len(fis), check.Equals, 1)
180 c.Check(fis[0].Size(), check.Equals, int64(3))
183 fis, err = f.Readdir(1)
184 c.Check(err, check.IsNil)
185 c.Check(len(fis), check.Equals, 1)
187 c.Check(fis[0].Size(), check.Equals, int64(3))
190 fis, err = f.Readdir(1)
191 c.Check(len(fis), check.Equals, 0)
192 c.Check(err, check.NotNil)
193 c.Check(err, check.Equals, io.EOF)
195 f, err = s.fs.Open("dir1")
196 c.Assert(err, check.IsNil)
197 fis, err = f.Readdir(1)
198 c.Check(len(fis), check.Equals, 1)
199 c.Assert(err, check.IsNil)
200 fis, err = f.Readdir(2)
201 c.Check(len(fis), check.Equals, 1)
202 c.Assert(err, check.IsNil)
203 fis, err = f.Readdir(2)
204 c.Check(len(fis), check.Equals, 0)
205 c.Assert(err, check.Equals, io.EOF)
208 func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
209 for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
210 f, err := s.fs.Open(path)
211 c.Assert(err, check.IsNil)
214 c.Assert(err, check.IsNil)
215 c.Check(st.Size(), check.Equals, int64(1))
216 c.Check(st.IsDir(), check.Equals, true)
218 for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
220 f, err := s.fs.Open(path)
221 c.Assert(err, check.IsNil)
224 c.Assert(err, check.IsNil)
225 c.Check(st.Size(), check.Equals, int64(2))
226 c.Check(st.IsDir(), check.Equals, true)
230 func (s *CollectionFSSuite) TestNotExist(c *check.C) {
231 for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
232 f, err := s.fs.Open(path)
233 c.Assert(f, check.IsNil)
234 c.Assert(err, check.NotNil)
235 c.Assert(os.IsNotExist(err), check.Equals, true)
239 func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
240 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDONLY, 0)
241 c.Assert(err, check.IsNil)
243 c.Assert(err, check.IsNil)
244 c.Check(st.Size(), check.Equals, int64(3))
245 n, err := f.Write([]byte("bar"))
246 c.Check(n, check.Equals, 0)
247 c.Check(err, check.Equals, ErrReadOnlyFile)
250 func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
251 f, err := s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE, 0)
252 c.Assert(err, check.IsNil)
254 c.Assert(err, check.IsNil)
255 c.Check(st.Size(), check.Equals, int64(0))
257 n, err := f.Write([]byte("bar"))
258 c.Check(n, check.Equals, 3)
259 c.Check(err, check.IsNil)
261 c.Check(f.Close(), check.IsNil)
263 f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
264 c.Check(f, check.IsNil)
265 c.Assert(err, check.NotNil)
267 f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR, 0)
268 c.Assert(err, check.IsNil)
270 c.Assert(err, check.IsNil)
271 c.Check(st.Size(), check.Equals, int64(3))
273 c.Check(f.Close(), check.IsNil)
275 m, err := s.fs.MarshalManifest(".")
276 c.Assert(err, check.IsNil)
277 c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
280 func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
282 defer func() { maxBlockSize = 2 << 26 }()
284 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
285 c.Assert(err, check.IsNil)
288 c.Assert(err, check.IsNil)
289 c.Check(st.Size(), check.Equals, int64(3))
291 f2, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
292 c.Assert(err, check.IsNil)
295 buf := make([]byte, 64)
296 n, err := f.Read(buf)
297 c.Check(n, check.Equals, 3)
298 c.Check(err, check.Equals, io.EOF)
299 c.Check(string(buf[:3]), check.DeepEquals, "foo")
301 pos, err := f.Seek(-2, io.SeekCurrent)
302 c.Check(pos, check.Equals, int64(1))
303 c.Check(err, check.IsNil)
305 // Split a storedExtent in two, and insert a memExtent
306 n, err = f.Write([]byte("*"))
307 c.Check(n, check.Equals, 1)
308 c.Check(err, check.IsNil)
310 pos, err = f.Seek(0, io.SeekCurrent)
311 c.Check(pos, check.Equals, int64(2))
312 c.Check(err, check.IsNil)
314 pos, err = f.Seek(0, io.SeekStart)
315 c.Check(pos, check.Equals, int64(0))
316 c.Check(err, check.IsNil)
318 rbuf, err := ioutil.ReadAll(f)
319 c.Check(len(rbuf), check.Equals, 3)
320 c.Check(err, check.IsNil)
321 c.Check(string(rbuf), check.Equals, "f*o")
323 // Write multiple blocks in one call
324 f.Seek(1, io.SeekStart)
325 n, err = f.Write([]byte("0123456789abcdefg"))
326 c.Check(n, check.Equals, 17)
327 c.Check(err, check.IsNil)
328 pos, err = f.Seek(0, io.SeekCurrent)
329 c.Check(pos, check.Equals, int64(18))
330 c.Check(err, check.IsNil)
331 pos, err = f.Seek(-18, io.SeekCurrent)
332 c.Check(pos, check.Equals, int64(0))
333 c.Check(err, check.IsNil)
334 n, err = io.ReadFull(f, buf)
335 c.Check(n, check.Equals, 18)
336 c.Check(err, check.Equals, io.ErrUnexpectedEOF)
337 c.Check(string(buf[:n]), check.Equals, "f0123456789abcdefg")
339 buf2, err := ioutil.ReadAll(f2)
340 c.Check(err, check.IsNil)
341 c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
343 // truncate to current size
345 c.Check(err, check.IsNil)
346 f2.Seek(0, io.SeekStart)
347 buf2, err = ioutil.ReadAll(f2)
348 c.Check(err, check.IsNil)
349 c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
351 // shrink to zero some data
353 f2.Seek(0, io.SeekStart)
354 buf2, err = ioutil.ReadAll(f2)
355 c.Check(err, check.IsNil)
356 c.Check(string(buf2), check.Equals, "f0123456789abcd")
358 // grow to partial block/extent
360 f2.Seek(0, io.SeekStart)
361 buf2, err = ioutil.ReadAll(f2)
362 c.Check(err, check.IsNil)
363 c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
366 f2.Seek(0, io.SeekStart)
367 f2.Write([]byte("12345678abcdefghijkl"))
369 // grow to block/extent boundary
371 f2.Seek(0, io.SeekStart)
372 buf2, err = ioutil.ReadAll(f2)
373 c.Check(err, check.IsNil)
374 c.Check(len(buf2), check.Equals, 64)
375 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 8)
377 // shrink to block/extent boundary
379 c.Check(err, check.IsNil)
380 f2.Seek(0, io.SeekStart)
381 buf2, err = ioutil.ReadAll(f2)
382 c.Check(err, check.IsNil)
383 c.Check(len(buf2), check.Equals, 32)
384 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 4)
386 // shrink to partial block/extent
388 c.Check(err, check.IsNil)
389 f2.Seek(0, io.SeekStart)
390 buf2, err = ioutil.ReadAll(f2)
391 c.Check(err, check.IsNil)
392 c.Check(string(buf2), check.Equals, "12345678abcdefg")
393 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 2)
395 // Force flush to ensure the block "12345678" gets stored, so
396 // we know what to expect in the final manifest below.
397 _, err = s.fs.MarshalManifest(".")
398 c.Check(err, check.IsNil)
400 // Truncate to size=3 while f2's ptr is at 15
402 c.Check(err, check.IsNil)
403 buf2, err = ioutil.ReadAll(f2)
404 c.Check(err, check.IsNil)
405 c.Check(string(buf2), check.Equals, "")
406 f2.Seek(0, io.SeekStart)
407 buf2, err = ioutil.ReadAll(f2)
408 c.Check(err, check.IsNil)
409 c.Check(string(buf2), check.Equals, "123")
410 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 1)
412 m, err := s.fs.MarshalManifest(".")
413 c.Check(err, check.IsNil)
414 m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
415 c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 25d55ad283aa400af464c76d713c07ad+8 3:3:bar 6:3:foo\n")
416 c.Check(s.fs.Size(), check.Equals, int64(6))
419 func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
420 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
421 c.Assert(err, check.IsNil)
422 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
423 c.Assert(err, check.IsNil)
426 checkSize := func(size int64) {
428 c.Assert(err, check.IsNil)
429 c.Check(fi.Size(), check.Equals, size)
431 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
432 c.Assert(err, check.IsNil)
435 c.Check(err, check.IsNil)
436 c.Check(fi.Size(), check.Equals, size)
437 pos, err := f.Seek(0, io.SeekEnd)
438 c.Check(err, check.IsNil)
439 c.Check(pos, check.Equals, size)
442 f.Seek(2, io.SeekEnd)
447 f.Seek(2, io.SeekCurrent)
452 f.Seek(8, io.SeekStart)
454 n, err := f.Read(make([]byte, 1))
455 c.Check(n, check.Equals, 0)
456 c.Check(err, check.Equals, io.EOF)
458 f.Write([]byte{1, 2, 3})
462 func (s *CollectionFSSuite) TestMarshalCopiesRemoteBlocks(c *check.C) {
465 hash := map[string]string{
466 foo: fmt.Sprintf("%x", md5.Sum([]byte(foo))),
467 bar: fmt.Sprintf("%x", md5.Sum([]byte(bar))),
470 fs, err := (&Collection{
471 ManifestText: ". " + hash[foo] + "+3+Rzaaaa-foo@bab " + hash[bar] + "+3+A12345@ffffff 0:2:fo.txt 2:4:obar.txt\n",
472 }).FileSystem(s.client, s.kc)
473 c.Assert(err, check.IsNil)
474 manifest, err := fs.MarshalManifest(".")
475 c.Check(manifest, check.Equals, "")
476 c.Check(err, check.NotNil)
478 s.kc.refreshable = map[string]bool{hash[bar]: true}
480 for _, sigIn := range []string{"Rzaaaa-foo@bab", "A12345@abcde"} {
481 fs, err = (&Collection{
482 ManifestText: ". " + hash[foo] + "+3+A12345@fffff " + hash[bar] + "+3+" + sigIn + " 0:2:fo.txt 2:4:obar.txt\n",
483 }).FileSystem(s.client, s.kc)
484 c.Assert(err, check.IsNil)
485 manifest, err := fs.MarshalManifest(".")
486 c.Check(err, check.IsNil)
487 // Both blocks should now have +A signatures.
488 c.Check(manifest, check.Matches, `.*\+A.* .*\+A.*\n`)
489 c.Check(manifest, check.Not(check.Matches), `.*\+R.*\n`)
493 func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
495 defer func() { maxBlockSize = 2 << 26 }()
498 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
499 c.Assert(err, check.IsNil)
500 for _, name := range []string{"foo", "bar", "baz"} {
501 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
502 c.Assert(err, check.IsNil)
503 f.Write([]byte(name))
507 m, err := s.fs.MarshalManifest(".")
508 c.Check(err, check.IsNil)
509 m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
510 c.Check(m, check.Equals, ". c3c23db5285662ef7172373df0003206+6 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar 3:3:baz 6:3:foo\n")
513 func (s *CollectionFSSuite) TestMkdir(c *check.C) {
514 err := s.fs.Mkdir("foo/bar", 0755)
515 c.Check(err, check.Equals, os.ErrNotExist)
517 f, err := s.fs.OpenFile("foo/bar", os.O_CREATE, 0)
518 c.Check(err, check.Equals, os.ErrNotExist)
520 err = s.fs.Mkdir("foo", 0755)
521 c.Check(err, check.IsNil)
523 f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_WRONLY, 0)
524 c.Check(err, check.IsNil)
527 f.Write([]byte("foo"))
530 // mkdir fails if a file already exists with that name
531 err = s.fs.Mkdir("foo/bar", 0755)
532 c.Check(err, check.NotNil)
534 err = s.fs.Remove("foo/bar")
535 c.Check(err, check.IsNil)
537 // mkdir succeeds after the file is deleted
538 err = s.fs.Mkdir("foo/bar", 0755)
539 c.Check(err, check.IsNil)
541 // creating a file in a nonexistent subdir should still fail
542 f, err = s.fs.OpenFile("foo/bar/baz/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
543 c.Check(err, check.Equals, os.ErrNotExist)
545 f, err = s.fs.OpenFile("foo/bar/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
546 c.Check(err, check.IsNil)
549 f.Write([]byte("foo"))
552 // creating foo/bar as a regular file should fail
553 f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, 0)
554 c.Check(err, check.NotNil)
556 // creating foo/bar as a directory should fail
557 f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, os.ModeDir)
558 c.Check(err, check.NotNil)
559 err = s.fs.Mkdir("foo/bar", 0755)
560 c.Check(err, check.NotNil)
562 m, err := s.fs.MarshalManifest(".")
563 c.Check(err, check.IsNil)
564 m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
565 c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 3:3:bar 0:3:foo\n./foo/bar acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
568 func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
574 defer func() { maxBlockSize = 1 << 26 }()
576 var wg sync.WaitGroup
577 for n := 0; n < 128; n++ {
581 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
582 c.Assert(err, check.IsNil)
584 for i := 0; i < 1024; i++ {
588 _, err := s.fs.MarshalManifest(".")
589 c.Check(err, check.IsNil)
591 f.Truncate(int64(rand.Intn(64)))
593 f.Seek(int64(rand.Intn(64)), io.SeekStart)
595 _, err := f.Write([]byte("beep boop"))
596 c.Check(err, check.IsNil)
598 _, err := ioutil.ReadAll(f)
599 c.Check(err, check.IsNil)
606 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
607 c.Assert(err, check.IsNil)
609 buf, err := ioutil.ReadAll(f)
610 c.Check(err, check.IsNil)
611 c.Logf("after lots of random r/w/seek/trunc, buf is %q", buf)
614 func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
616 defer func() { maxBlockSize = 2 << 26 }()
619 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
620 c.Assert(err, check.IsNil)
623 const ngoroutines = 256
625 var wg sync.WaitGroup
626 for n := 0; n < ngoroutines; n++ {
630 expect := make([]byte, 0, 64)
631 wbytes := []byte("there's no simple explanation for anything important that any of us do")
632 f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
633 c.Assert(err, check.IsNil)
635 for i := 0; i < nfiles; i++ {
636 trunc := rand.Intn(65)
637 woff := rand.Intn(trunc + 1)
638 wbytes = wbytes[:rand.Intn(64-woff+1)]
639 for buf, i := expect[:cap(expect)], len(expect); i < trunc; i++ {
642 expect = expect[:trunc]
643 if trunc < woff+len(wbytes) {
644 expect = expect[:woff+len(wbytes)]
646 copy(expect[woff:], wbytes)
647 f.Truncate(int64(trunc))
648 pos, err := f.Seek(int64(woff), io.SeekStart)
649 c.Check(pos, check.Equals, int64(woff))
650 c.Check(err, check.IsNil)
651 n, err := f.Write(wbytes)
652 c.Check(n, check.Equals, len(wbytes))
653 c.Check(err, check.IsNil)
654 pos, err = f.Seek(0, io.SeekStart)
655 c.Check(pos, check.Equals, int64(0))
656 c.Check(err, check.IsNil)
657 buf, err := ioutil.ReadAll(f)
658 c.Check(string(buf), check.Equals, string(expect))
659 c.Check(err, check.IsNil)
665 for n := 0; n < ngoroutines; n++ {
666 f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDONLY, 0)
667 c.Assert(err, check.IsNil)
668 f.(*filehandle).inode.(*filenode).waitPrune()
673 root, err := s.fs.Open("/")
674 c.Assert(err, check.IsNil)
676 fi, err := root.Readdir(-1)
677 c.Check(err, check.IsNil)
678 c.Check(len(fi), check.Equals, nfiles)
680 _, err = s.fs.MarshalManifest(".")
681 c.Check(err, check.IsNil)
682 // TODO: check manifest content
685 func (s *CollectionFSSuite) TestRemove(c *check.C) {
686 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
687 c.Assert(err, check.IsNil)
688 err = fs.Mkdir("dir0", 0755)
689 c.Assert(err, check.IsNil)
690 err = fs.Mkdir("dir1", 0755)
691 c.Assert(err, check.IsNil)
692 err = fs.Mkdir("dir1/dir2", 0755)
693 c.Assert(err, check.IsNil)
694 err = fs.Mkdir("dir1/dir3", 0755)
695 c.Assert(err, check.IsNil)
697 err = fs.Remove("dir0")
698 c.Check(err, check.IsNil)
699 err = fs.Remove("dir0")
700 c.Check(err, check.Equals, os.ErrNotExist)
702 err = fs.Remove("dir1/dir2/.")
703 c.Check(err, check.Equals, ErrInvalidArgument)
704 err = fs.Remove("dir1/dir2/..")
705 c.Check(err, check.Equals, ErrInvalidArgument)
706 err = fs.Remove("dir1")
707 c.Check(err, check.Equals, ErrDirectoryNotEmpty)
708 err = fs.Remove("dir1/dir2/../../../dir1")
709 c.Check(err, check.Equals, ErrDirectoryNotEmpty)
710 err = fs.Remove("dir1/dir3/")
711 c.Check(err, check.IsNil)
712 err = fs.RemoveAll("dir1")
713 c.Check(err, check.IsNil)
714 err = fs.RemoveAll("dir1")
715 c.Check(err, check.IsNil)
718 func (s *CollectionFSSuite) TestRenameError(c *check.C) {
719 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
720 c.Assert(err, check.IsNil)
721 err = fs.Mkdir("first", 0755)
722 c.Assert(err, check.IsNil)
723 err = fs.Mkdir("first/second", 0755)
724 c.Assert(err, check.IsNil)
725 f, err := fs.OpenFile("first/second/file", os.O_CREATE|os.O_WRONLY, 0755)
726 c.Assert(err, check.IsNil)
727 f.Write([]byte{1, 2, 3, 4, 5})
729 err = fs.Rename("first", "first/second/third")
730 c.Check(err, check.Equals, ErrInvalidArgument)
731 err = fs.Rename("first", "first/third")
732 c.Check(err, check.Equals, ErrInvalidArgument)
733 err = fs.Rename("first/second", "second")
734 c.Check(err, check.IsNil)
735 f, err = fs.OpenFile("second/file", 0, 0)
736 c.Assert(err, check.IsNil)
737 data, err := ioutil.ReadAll(f)
738 c.Check(err, check.IsNil)
739 c.Check(data, check.DeepEquals, []byte{1, 2, 3, 4, 5})
742 func (s *CollectionFSSuite) TestRenameDirectory(c *check.C) {
743 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
744 c.Assert(err, check.IsNil)
745 err = fs.Mkdir("foo", 0755)
746 c.Assert(err, check.IsNil)
747 err = fs.Mkdir("bar", 0755)
748 c.Assert(err, check.IsNil)
749 err = fs.Rename("bar", "baz")
750 c.Check(err, check.IsNil)
751 err = fs.Rename("foo", "baz")
752 c.Check(err, check.NotNil)
753 err = fs.Rename("foo", "baz/")
754 c.Check(err, check.IsNil)
755 err = fs.Rename("baz/foo", ".")
756 c.Check(err, check.Equals, ErrInvalidArgument)
757 err = fs.Rename("baz/foo/", ".")
758 c.Check(err, check.Equals, ErrInvalidArgument)
761 func (s *CollectionFSSuite) TestRename(c *check.C) {
762 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
763 c.Assert(err, check.IsNil)
768 for i := 0; i < outer; i++ {
769 err = fs.Mkdir(fmt.Sprintf("dir%d", i), 0755)
770 c.Assert(err, check.IsNil)
771 for j := 0; j < inner; j++ {
772 err = fs.Mkdir(fmt.Sprintf("dir%d/dir%d", i, j), 0755)
773 c.Assert(err, check.IsNil)
774 for _, fnm := range []string{
775 fmt.Sprintf("dir%d/file%d", i, j),
776 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
778 f, err := fs.OpenFile(fnm, os.O_CREATE|os.O_WRONLY, 0755)
779 c.Assert(err, check.IsNil)
780 _, err = f.Write([]byte("beep"))
781 c.Assert(err, check.IsNil)
786 var wg sync.WaitGroup
787 for i := 0; i < outer; i++ {
788 for j := 0; j < inner; j++ {
792 oldname := fmt.Sprintf("dir%d/dir%d/file%d", i, j, j)
793 newname := fmt.Sprintf("dir%d/newfile%d", i, inner-j-1)
794 _, err := fs.Open(newname)
795 c.Check(err, check.Equals, os.ErrNotExist)
796 err = fs.Rename(oldname, newname)
797 c.Check(err, check.IsNil)
798 f, err := fs.Open(newname)
799 c.Check(err, check.IsNil)
806 // oldname does not exist
808 fmt.Sprintf("dir%d/dir%d/missing", i, j),
809 fmt.Sprintf("dir%d/dir%d/file%d", outer-i-1, j, j))
810 c.Check(err, check.ErrorMatches, `.*does not exist`)
812 // newname parent dir does not exist
814 fmt.Sprintf("dir%d/dir%d", i, j),
815 fmt.Sprintf("dir%d/missing/irrelevant", outer-i-1))
816 c.Check(err, check.ErrorMatches, `.*does not exist`)
818 // oldname parent dir is a file
820 fmt.Sprintf("dir%d/file%d/patherror", i, j),
821 fmt.Sprintf("dir%d/irrelevant", i))
822 c.Check(err, check.ErrorMatches, `.*not a directory`)
824 // newname parent dir is a file
826 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
827 fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
828 c.Check(err, check.ErrorMatches, `.*not a directory`)
834 f, err := fs.OpenFile("dir1/newfile3", 0, 0)
835 c.Assert(err, check.IsNil)
836 c.Check(f.Size(), check.Equals, int64(4))
837 buf, err := ioutil.ReadAll(f)
838 c.Check(buf, check.DeepEquals, []byte("beep"))
839 c.Check(err, check.IsNil)
840 _, err = fs.Open("dir1/dir1/file1")
841 c.Check(err, check.Equals, os.ErrNotExist)
844 func (s *CollectionFSSuite) TestPersist(c *check.C) {
846 defer func() { maxBlockSize = 2 << 26 }()
849 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
850 c.Assert(err, check.IsNil)
851 err = s.fs.Mkdir("d:r", 0755)
852 c.Assert(err, check.IsNil)
854 expect := map[string][]byte{}
856 var wg sync.WaitGroup
857 for _, name := range []string{"random 1", "random:2", "random\\3", "d:r/random4"} {
858 buf := make([]byte, 500)
862 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
863 c.Assert(err, check.IsNil)
864 // Note: we don't close the file until after the test
865 // is done. Writes to unclosed files should persist.
871 for i := 0; i < len(buf); i += 5 {
872 _, err := f.Write(buf[i : i+5])
873 c.Assert(err, check.IsNil)
879 m, err := s.fs.MarshalManifest(".")
880 c.Check(err, check.IsNil)
883 root, err := s.fs.Open("/")
884 c.Assert(err, check.IsNil)
886 fi, err := root.Readdir(-1)
887 c.Check(err, check.IsNil)
888 c.Check(len(fi), check.Equals, 4)
890 persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
891 c.Assert(err, check.IsNil)
893 root, err = persisted.Open("/")
894 c.Assert(err, check.IsNil)
896 fi, err = root.Readdir(-1)
897 c.Check(err, check.IsNil)
898 c.Check(len(fi), check.Equals, 4)
900 for name, content := range expect {
901 c.Logf("read %q", name)
902 f, err := persisted.Open(name)
903 c.Assert(err, check.IsNil)
905 buf, err := ioutil.ReadAll(f)
906 c.Check(err, check.IsNil)
907 c.Check(buf, check.DeepEquals, content)
911 func (s *CollectionFSSuite) TestPersistEmptyFilesAndDirs(c *check.C) {
913 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
914 c.Assert(err, check.IsNil)
915 for _, name := range []string{"dir", "dir/zerodir", "empty", "not empty", "not empty/empty", "zero", "zero/zero"} {
916 err = s.fs.Mkdir(name, 0755)
917 c.Assert(err, check.IsNil)
920 expect := map[string][]byte{
927 "dir/zerodir/zero": nil,
928 "zero/zero/zero": nil,
930 for name, data := range expect {
931 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
932 c.Assert(err, check.IsNil)
934 _, err := f.Write(data)
935 c.Assert(err, check.IsNil)
940 m, err := s.fs.MarshalManifest(".")
941 c.Check(err, check.IsNil)
944 persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
945 c.Assert(err, check.IsNil)
947 for name, data := range expect {
948 _, err = persisted.Open("bogus-" + name)
949 c.Check(err, check.NotNil)
951 f, err := persisted.Open(name)
952 c.Assert(err, check.IsNil)
957 buf, err := ioutil.ReadAll(f)
958 c.Check(err, check.IsNil)
959 c.Check(buf, check.DeepEquals, data)
962 expectDir := map[string]int{
965 "not empty/empty": 0,
967 for name, expectLen := range expectDir {
968 _, err := persisted.Open(name + "/bogus")
969 c.Check(err, check.NotNil)
971 d, err := persisted.Open(name)
973 c.Check(err, check.IsNil)
974 fi, err := d.Readdir(-1)
975 c.Check(err, check.IsNil)
976 c.Check(fi, check.HasLen, expectLen)
980 func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
981 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
982 c.Assert(err, check.IsNil)
984 f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
985 c.Check(f, check.IsNil)
986 c.Check(err, check.ErrorMatches, `file does not exist`)
988 f, err = fs.OpenFile("new", os.O_CREATE|os.O_RDONLY, 0)
989 c.Assert(err, check.IsNil)
991 n, err := f.Write([]byte{1, 2, 3})
992 c.Check(n, check.Equals, 0)
993 c.Check(err, check.ErrorMatches, `read-only file`)
994 n, err = f.Read(make([]byte, 1))
995 c.Check(n, check.Equals, 0)
996 c.Check(err, check.Equals, io.EOF)
997 f, err = fs.OpenFile("new", os.O_RDWR, 0)
998 c.Assert(err, check.IsNil)
1000 _, err = f.Write([]byte{4, 5, 6})
1001 c.Check(err, check.IsNil)
1003 c.Assert(err, check.IsNil)
1004 c.Check(fi.Size(), check.Equals, int64(3))
1006 f, err = fs.OpenFile("new", os.O_TRUNC|os.O_RDWR, 0)
1007 c.Assert(err, check.IsNil)
1009 pos, err := f.Seek(0, io.SeekEnd)
1010 c.Check(pos, check.Equals, int64(0))
1011 c.Check(err, check.IsNil)
1013 c.Assert(err, check.IsNil)
1014 c.Check(fi.Size(), check.Equals, int64(0))
1017 buf := make([]byte, 64)
1018 f, err = fs.OpenFile("append", os.O_EXCL|os.O_CREATE|os.O_RDWR|os.O_APPEND, 0)
1019 c.Assert(err, check.IsNil)
1020 f.Write([]byte{1, 2, 3})
1021 f.Seek(0, io.SeekStart)
1022 n, _ = f.Read(buf[:1])
1023 c.Check(n, check.Equals, 1)
1024 c.Check(buf[:1], check.DeepEquals, []byte{1})
1025 pos, err = f.Seek(0, io.SeekCurrent)
1026 c.Assert(err, check.IsNil)
1027 c.Check(pos, check.Equals, int64(1))
1028 f.Write([]byte{4, 5, 6})
1029 pos, err = f.Seek(0, io.SeekCurrent)
1030 c.Assert(err, check.IsNil)
1031 c.Check(pos, check.Equals, int64(6))
1032 f.Seek(0, io.SeekStart)
1033 n, err = f.Read(buf)
1034 c.Check(buf[:n], check.DeepEquals, []byte{1, 2, 3, 4, 5, 6})
1035 c.Check(err, check.Equals, io.EOF)
1038 f, err = fs.OpenFile("append", os.O_RDWR|os.O_APPEND, 0)
1039 c.Assert(err, check.IsNil)
1040 pos, err = f.Seek(0, io.SeekCurrent)
1041 c.Check(pos, check.Equals, int64(0))
1042 c.Check(err, check.IsNil)
1044 pos, _ = f.Seek(0, io.SeekCurrent)
1045 c.Check(pos, check.Equals, int64(3))
1046 f.Write([]byte{7, 8, 9})
1047 pos, err = f.Seek(0, io.SeekCurrent)
1048 c.Check(err, check.IsNil)
1049 c.Check(pos, check.Equals, int64(9))
1052 f, err = fs.OpenFile("wronly", os.O_CREATE|os.O_WRONLY, 0)
1053 c.Assert(err, check.IsNil)
1054 n, err = f.Write([]byte{3, 2, 1})
1055 c.Check(n, check.Equals, 3)
1056 c.Check(err, check.IsNil)
1057 pos, _ = f.Seek(0, io.SeekCurrent)
1058 c.Check(pos, check.Equals, int64(3))
1059 pos, _ = f.Seek(0, io.SeekStart)
1060 c.Check(pos, check.Equals, int64(0))
1061 n, err = f.Read(buf)
1062 c.Check(n, check.Equals, 0)
1063 c.Check(err, check.ErrorMatches, `.*O_WRONLY.*`)
1064 f, err = fs.OpenFile("wronly", os.O_RDONLY, 0)
1065 c.Assert(err, check.IsNil)
1067 c.Check(buf[:n], check.DeepEquals, []byte{3, 2, 1})
1069 f, err = fs.OpenFile("unsupported", os.O_CREATE|os.O_SYNC, 0)
1070 c.Check(f, check.IsNil)
1071 c.Check(err, check.NotNil)
1073 f, err = fs.OpenFile("append", os.O_RDWR|os.O_WRONLY, 0)
1074 c.Check(f, check.IsNil)
1075 c.Check(err, check.ErrorMatches, `invalid flag.*`)
1078 func (s *CollectionFSSuite) TestFlushFullBlocksWritingLongFile(c *check.C) {
1079 defer func(cw, mbs int) {
1080 concurrentWriters = cw
1082 }(concurrentWriters, maxBlockSize)
1083 concurrentWriters = 2
1086 proceed := make(chan struct{})
1087 var started, concurrent int32
1089 s.kc.onWrite = func([]byte) {
1090 atomic.AddInt32(&concurrent, 1)
1091 switch atomic.AddInt32(&started, 1) {
1093 // Wait until block 2 starts and finishes, and block 3 starts
1096 c.Check(blk2done, check.Equals, true)
1097 case <-time.After(time.Second):
1098 c.Error("timed out")
1101 time.Sleep(time.Millisecond)
1106 time.Sleep(time.Millisecond)
1108 c.Check(atomic.AddInt32(&concurrent, -1) < int32(concurrentWriters), check.Equals, true)
1111 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
1112 c.Assert(err, check.IsNil)
1113 f, err := fs.OpenFile("50K", os.O_WRONLY|os.O_CREATE, 0)
1114 c.Assert(err, check.IsNil)
1117 data := make([]byte, 500)
1120 for i := 0; i < 100; i++ {
1121 n, err := f.Write(data)
1122 c.Assert(n, check.Equals, len(data))
1123 c.Assert(err, check.IsNil)
1126 currentMemExtents := func() (memExtents []int) {
1127 for idx, e := range f.(*filehandle).inode.(*filenode).segments {
1130 memExtents = append(memExtents, idx)
1135 f.(*filehandle).inode.(*filenode).waitPrune()
1136 c.Check(currentMemExtents(), check.HasLen, 1)
1138 m, err := fs.MarshalManifest(".")
1139 c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
1140 c.Check(err, check.IsNil)
1141 c.Check(currentMemExtents(), check.HasLen, 0)
1144 // Ensure blocks get flushed to disk if a lot of data is written to
1145 // small files/directories without calling sync().
1147 // Write four 512KiB files into each of 256 top-level dirs (total
1148 // 512MiB), calling Flush() every 8 dirs. Ensure memory usage never
1149 // exceeds 24MiB (4 concurrentWriters * 2MiB + 8 unflushed dirs *
1151 func (s *CollectionFSSuite) TestFlushAll(c *check.C) {
1152 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
1153 c.Assert(err, check.IsNil)
1155 s.kc.onWrite = func([]byte) {
1156 // discard flushed data -- otherwise the stub will use
1158 time.Sleep(time.Millisecond)
1161 s.kc.blocks = map[string][]byte{}
1163 for i := 0; i < 256; i++ {
1164 buf := bytes.NewBuffer(make([]byte, 524288))
1165 fmt.Fprintf(buf, "test file in dir%d", i)
1167 dir := fmt.Sprintf("dir%d", i)
1169 for j := 0; j < 2; j++ {
1170 f, err := fs.OpenFile(fmt.Sprintf("%s/file%d", dir, j), os.O_WRONLY|os.O_CREATE, 0)
1171 c.Assert(err, check.IsNil)
1173 _, err = io.Copy(f, buf)
1174 c.Assert(err, check.IsNil)
1181 size := fs.MemorySize()
1182 if !c.Check(size <= 1<<24, check.Equals, true) {
1183 c.Logf("at dir%d fs.MemorySize()=%d", i, size)
1189 // Ensure short blocks at the end of a stream don't get flushed by
1192 // Write 67x 1MiB files to each of 8 dirs, and check that 8 full 64MiB
1193 // blocks have been flushed while 8x 3MiB is still buffered in memory.
1194 func (s *CollectionFSSuite) TestFlushFullBlocksOnly(c *check.C) {
1195 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
1196 c.Assert(err, check.IsNil)
1199 s.kc.onWrite = func(p []byte) {
1200 atomic.AddInt64(&flushed, int64(len(p)))
1204 megabyte := make([]byte, 1<<20)
1205 for i := int64(0); i < nDirs; i++ {
1206 dir := fmt.Sprintf("dir%d", i)
1208 for j := 0; j < 67; j++ {
1209 f, err := fs.OpenFile(fmt.Sprintf("%s/file%d", dir, j), os.O_WRONLY|os.O_CREATE, 0)
1210 c.Assert(err, check.IsNil)
1212 _, err = f.Write(megabyte)
1213 c.Assert(err, check.IsNil)
1216 c.Check(fs.MemorySize(), check.Equals, int64(nDirs*67<<20))
1217 c.Check(flushed, check.Equals, int64(0))
1219 waitForFlush := func(expectUnflushed, expectFlushed int64) {
1220 for deadline := time.Now().Add(5 * time.Second); fs.MemorySize() > expectUnflushed && time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
1222 c.Check(fs.MemorySize(), check.Equals, expectUnflushed)
1223 c.Check(flushed, check.Equals, expectFlushed)
1226 // Nothing flushed yet
1227 waitForFlush((nDirs*67)<<20, 0)
1229 // Flushing a non-empty dir "/" is non-recursive and there are
1230 // no top-level files, so this has no effect
1231 fs.Flush("/", false)
1232 waitForFlush((nDirs*67)<<20, 0)
1234 // Flush the full block in dir0
1235 fs.Flush("dir0", false)
1236 waitForFlush((nDirs*67-64)<<20, 64<<20)
1238 err = fs.Flush("dir-does-not-exist", false)
1239 c.Check(err, check.NotNil)
1241 // Flush full blocks in all dirs
1243 waitForFlush(nDirs*3<<20, nDirs*64<<20)
1245 // Flush non-full blocks, too
1247 waitForFlush(0, nDirs*67<<20)
1250 // Even when writing lots of files/dirs from different goroutines, as
1251 // long as Flush(dir,false) is called after writing each file,
1252 // unflushed data should be limited to one full block per
1253 // concurrentWriter, plus one nearly-full block at the end of each
1255 func (s *CollectionFSSuite) TestMaxUnflushed(c *check.C) {
1257 maxUnflushed := (int64(concurrentWriters) + nDirs) << 26
1259 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
1260 c.Assert(err, check.IsNil)
1262 release := make(chan struct{})
1263 timeout := make(chan struct{})
1264 time.AfterFunc(10*time.Second, func() { close(timeout) })
1265 var putCount, concurrency int64
1267 s.kc.onWrite = func(p []byte) {
1268 defer atomic.AddInt64(&unflushed, -int64(len(p)))
1269 cur := atomic.AddInt64(&concurrency, 1)
1270 defer atomic.AddInt64(&concurrency, -1)
1271 pc := atomic.AddInt64(&putCount, 1)
1272 if pc < int64(concurrentWriters) {
1273 // Block until we reach concurrentWriters, to
1274 // make sure we're really accepting concurrent
1281 } else if pc == int64(concurrentWriters) {
1282 // Unblock the first N-1 PUT reqs.
1285 c.Assert(cur <= int64(concurrentWriters), check.Equals, true)
1286 c.Assert(atomic.LoadInt64(&unflushed) <= maxUnflushed, check.Equals, true)
1289 var owg sync.WaitGroup
1290 megabyte := make([]byte, 1<<20)
1291 for i := int64(0); i < nDirs; i++ {
1292 dir := fmt.Sprintf("dir%d", i)
1297 defer fs.Flush(dir, true)
1298 var iwg sync.WaitGroup
1300 for j := 0; j < 67; j++ {
1304 f, err := fs.OpenFile(fmt.Sprintf("%s/file%d", dir, j), os.O_WRONLY|os.O_CREATE, 0)
1305 c.Assert(err, check.IsNil)
1307 n, err := f.Write(megabyte)
1308 c.Assert(err, check.IsNil)
1309 atomic.AddInt64(&unflushed, int64(n))
1310 fs.Flush(dir, false)
1319 func (s *CollectionFSSuite) TestFlushStress(c *check.C) {
1321 defer func() { done = true }()
1322 time.AfterFunc(10*time.Second, func() {
1324 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1330 s.kc.onWrite = func(p []byte) {
1332 s.kc.blocks = map[string][]byte{}
1334 defer c.Logf("wrote block %d, %d bytes", wrote, len(p))
1336 time.Sleep(20 * time.Millisecond)
1339 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
1340 c.Assert(err, check.IsNil)
1342 data := make([]byte, 1<<20)
1343 for i := 0; i < 3; i++ {
1344 dir := fmt.Sprintf("dir%d", i)
1346 for j := 0; j < 200; j++ {
1348 f, err := fs.OpenFile(fmt.Sprintf("%s/file%d", dir, j), os.O_WRONLY|os.O_CREATE, 0)
1349 c.Assert(err, check.IsNil)
1350 _, err = f.Write(data)
1351 c.Assert(err, check.IsNil)
1353 fs.Flush(dir, false)
1355 _, err := fs.MarshalManifest(".")
1356 c.Check(err, check.IsNil)
1360 func (s *CollectionFSSuite) TestFlushShort(c *check.C) {
1361 s.kc.onWrite = func([]byte) {
1363 s.kc.blocks = map[string][]byte{}
1366 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
1367 c.Assert(err, check.IsNil)
1368 for _, blocksize := range []int{8, 1000000} {
1369 dir := fmt.Sprintf("dir%d", blocksize)
1370 err = fs.Mkdir(dir, 0755)
1371 c.Assert(err, check.IsNil)
1372 data := make([]byte, blocksize)
1373 for i := 0; i < 100; i++ {
1374 f, err := fs.OpenFile(fmt.Sprintf("%s/file%d", dir, i), os.O_WRONLY|os.O_CREATE, 0)
1375 c.Assert(err, check.IsNil)
1376 _, err = f.Write(data)
1377 c.Assert(err, check.IsNil)
1379 fs.Flush(dir, false)
1382 _, err := fs.MarshalManifest(".")
1383 c.Check(err, check.IsNil)
1387 func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
1388 for _, txt := range []string{
1392 ". d41d8cd98f00b204e9800998ecf8427e+0\n",
1393 ". d41d8cd98f00b204e9800998ecf8427e+0 \n",
1396 ". 0:0:foo 0:0:bar\n",
1397 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo\n",
1398 ". d41d8cd98f00b204e9800998ecf8427e+0 :0:0:foo\n",
1399 ". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
1400 ". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
1401 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
1402 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:\\056\n",
1403 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:\\056\\057\\056\n",
1404 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:.\n",
1405 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:..\n",
1406 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:..\n",
1407 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/..\n",
1408 ". d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n",
1409 "./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
1412 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
1413 c.Check(fs, check.IsNil)
1414 c.Logf("-> %s", err)
1415 c.Check(err, check.NotNil)
1419 func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
1420 for _, txt := range []string{
1422 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
1423 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:...\n",
1424 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:. 0:0:. 0:0:\\056 0:0:\\056\n",
1425 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/. 0:0:. 0:0:foo\\057bar\\057\\056\n",
1426 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
1427 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
1430 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
1431 c.Check(err, check.IsNil)
1432 c.Check(fs, check.NotNil)
1436 var bigmanifest = func() string {
1437 var buf bytes.Buffer
1438 for i := 0; i < 2000; i++ {
1439 fmt.Fprintf(&buf, "./dir%d", i)
1440 for i := 0; i < 100; i++ {
1441 fmt.Fprintf(&buf, " d41d8cd98f00b204e9800998ecf8427e+99999")
1443 for i := 0; i < 2000; i++ {
1444 fmt.Fprintf(&buf, " 1200000:300000:file%d", i)
1446 fmt.Fprintf(&buf, "\n")
1451 func (s *CollectionFSSuite) BenchmarkParseManifest(c *check.C) {
1452 DebugLocksPanicMode = false
1453 c.Logf("test manifest is %d bytes", len(bigmanifest))
1454 for i := 0; i < c.N; i++ {
1455 fs, err := (&Collection{ManifestText: bigmanifest}).FileSystem(s.client, s.kc)
1456 c.Check(err, check.IsNil)
1457 c.Check(fs, check.NotNil)
1461 func (s *CollectionFSSuite) checkMemSize(c *check.C, f File) {
1462 fn := f.(*filehandle).inode.(*filenode)
1464 for _, seg := range fn.segments {
1465 if e, ok := seg.(*memSegment); ok {
1466 memsize += int64(len(e.buf))
1469 c.Check(fn.memsize, check.Equals, memsize)
1472 type CollectionFSUnitSuite struct{}
1474 var _ = check.Suite(&CollectionFSUnitSuite{})
1476 // expect ~2 seconds to load a manifest with 256K files
1477 func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
1478 if testing.Short() {
1487 mb := bytes.NewBuffer(make([]byte, 0, 40000000))
1488 for i := 0; i < dirCount; i++ {
1489 fmt.Fprintf(mb, "./dir%d", i)
1490 for j := 0; j <= fileCount; j++ {
1491 fmt.Fprintf(mb, " %032x+42+A%040x@%08x", j, j, j)
1493 for j := 0; j < fileCount; j++ {
1494 fmt.Fprintf(mb, " %d:%d:dir%d/file%d", j*42+21, 42, j, j)
1496 mb.Write([]byte{'\n'})
1498 coll := Collection{ManifestText: mb.String()}
1499 c.Logf("%s built", time.Now())
1501 var memstats runtime.MemStats
1502 runtime.ReadMemStats(&memstats)
1503 c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1505 f, err := coll.FileSystem(nil, nil)
1506 c.Check(err, check.IsNil)
1507 c.Logf("%s loaded", time.Now())
1508 c.Check(f.Size(), check.Equals, int64(42*dirCount*fileCount))
1510 for i := 0; i < dirCount; i++ {
1511 for j := 0; j < fileCount; j++ {
1512 f.Stat(fmt.Sprintf("./dir%d/dir%d/file%d", i, j, j))
1515 c.Logf("%s Stat() x %d", time.Now(), dirCount*fileCount)
1517 runtime.ReadMemStats(&memstats)
1518 c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1521 // Gocheck boilerplate
1522 func Test(t *testing.T) {