1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
23 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
24 check "gopkg.in/check.v1"
27 var _ = check.Suite(&CollectionFSSuite{})
29 type keepClientStub struct {
30 blocks map[string][]byte
34 var errStub404 = errors.New("404 block not found")
36 func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error) {
39 buf := kcs.blocks[locator[:32]]
43 return copy(p, buf[off:]), nil
46 func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
47 locator := fmt.Sprintf("%x+%d+A12345@abcde", md5.Sum(p), len(p))
48 buf := make([]byte, len(p))
52 kcs.blocks[locator[:32]] = buf
53 return locator, 1, nil
56 type CollectionFSSuite struct {
59 fs CollectionFileSystem
63 func (s *CollectionFSSuite) SetUpTest(c *check.C) {
64 s.client = NewClientFromEnv()
65 err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
66 c.Assert(err, check.IsNil)
67 s.kc = &keepClientStub{
68 blocks: map[string][]byte{
69 "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
71 s.fs, err = s.coll.FileSystem(s.client, s.kc)
72 c.Assert(err, check.IsNil)
75 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
76 _, ok := s.fs.(http.FileSystem)
77 c.Check(ok, check.Equals, true)
80 func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
81 fs, err := (&Collection{
82 ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
83 }).FileSystem(s.client, s.kc)
84 c.Assert(err, check.IsNil)
86 f, err := fs.Open("/foo:foo")
87 c.Assert(err, check.IsNil)
89 fis, err := f.Readdir(0)
90 c.Check(err, check.IsNil)
91 c.Check(len(fis), check.Equals, 1)
92 c.Check(fis[0].Name(), check.Equals, "bar:bar")
95 func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
96 f, err := s.fs.Open("/dir1")
97 c.Assert(err, check.IsNil)
100 c.Assert(err, check.IsNil)
101 c.Check(st.Size(), check.Equals, int64(2))
102 c.Check(st.IsDir(), check.Equals, true)
104 fis, err := f.Readdir(0)
105 c.Check(err, check.IsNil)
106 c.Check(len(fis), check.Equals, 2)
108 c.Check(fis[0].Size(), check.Equals, int64(3))
112 func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
113 f, err := s.fs.Open("./dir1")
114 c.Assert(err, check.IsNil)
116 fis, err := f.Readdir(1)
117 c.Check(err, check.IsNil)
118 c.Check(len(fis), check.Equals, 1)
120 c.Check(fis[0].Size(), check.Equals, int64(3))
123 fis, err = f.Readdir(1)
124 c.Check(err, check.IsNil)
125 c.Check(len(fis), check.Equals, 1)
127 c.Check(fis[0].Size(), check.Equals, int64(3))
130 fis, err = f.Readdir(1)
131 c.Check(len(fis), check.Equals, 0)
132 c.Check(err, check.NotNil)
133 c.Check(err, check.Equals, io.EOF)
135 f, err = s.fs.Open("dir1")
136 c.Assert(err, check.IsNil)
137 fis, err = f.Readdir(1)
138 c.Check(len(fis), check.Equals, 1)
139 c.Assert(err, check.IsNil)
140 fis, err = f.Readdir(2)
141 c.Check(len(fis), check.Equals, 1)
142 c.Assert(err, check.IsNil)
143 fis, err = f.Readdir(2)
144 c.Check(len(fis), check.Equals, 0)
145 c.Assert(err, check.Equals, io.EOF)
148 func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
149 for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
150 f, err := s.fs.Open(path)
151 c.Assert(err, check.IsNil)
154 c.Assert(err, check.IsNil)
155 c.Check(st.Size(), check.Equals, int64(1))
156 c.Check(st.IsDir(), check.Equals, true)
158 for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
160 f, err := s.fs.Open(path)
161 c.Assert(err, check.IsNil)
164 c.Assert(err, check.IsNil)
165 c.Check(st.Size(), check.Equals, int64(2))
166 c.Check(st.IsDir(), check.Equals, true)
170 func (s *CollectionFSSuite) TestNotExist(c *check.C) {
171 for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
172 f, err := s.fs.Open(path)
173 c.Assert(f, check.IsNil)
174 c.Assert(err, check.NotNil)
175 c.Assert(os.IsNotExist(err), check.Equals, true)
179 func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
180 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDONLY, 0)
181 c.Assert(err, check.IsNil)
183 c.Assert(err, check.IsNil)
184 c.Check(st.Size(), check.Equals, int64(3))
185 n, err := f.Write([]byte("bar"))
186 c.Check(n, check.Equals, 0)
187 c.Check(err, check.Equals, ErrReadOnlyFile)
190 func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
191 f, err := s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE, 0)
192 c.Assert(err, check.IsNil)
194 c.Assert(err, check.IsNil)
195 c.Check(st.Size(), check.Equals, int64(0))
197 n, err := f.Write([]byte("bar"))
198 c.Check(n, check.Equals, 3)
199 c.Check(err, check.IsNil)
201 c.Check(f.Close(), check.IsNil)
203 f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
204 c.Check(f, check.IsNil)
205 c.Assert(err, check.NotNil)
207 f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR, 0)
208 c.Assert(err, check.IsNil)
210 c.Assert(err, check.IsNil)
211 c.Check(st.Size(), check.Equals, int64(3))
213 c.Check(f.Close(), check.IsNil)
215 m, err := s.fs.MarshalManifest(".")
216 c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
219 func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
221 defer func() { maxBlockSize = 2 << 26 }()
223 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
224 c.Assert(err, check.IsNil)
227 c.Assert(err, check.IsNil)
228 c.Check(st.Size(), check.Equals, int64(3))
230 f2, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
231 c.Assert(err, check.IsNil)
234 buf := make([]byte, 64)
235 n, err := f.Read(buf)
236 c.Check(n, check.Equals, 3)
237 c.Check(err, check.Equals, io.EOF)
238 c.Check(string(buf[:3]), check.DeepEquals, "foo")
240 pos, err := f.Seek(-2, io.SeekCurrent)
241 c.Check(pos, check.Equals, int64(1))
242 c.Check(err, check.IsNil)
244 // Split a storedExtent in two, and insert a memExtent
245 n, err = f.Write([]byte("*"))
246 c.Check(n, check.Equals, 1)
247 c.Check(err, check.IsNil)
249 pos, err = f.Seek(0, io.SeekCurrent)
250 c.Check(pos, check.Equals, int64(2))
251 c.Check(err, check.IsNil)
253 pos, err = f.Seek(0, io.SeekStart)
254 c.Check(pos, check.Equals, int64(0))
255 c.Check(err, check.IsNil)
257 rbuf, err := ioutil.ReadAll(f)
258 c.Check(len(rbuf), check.Equals, 3)
259 c.Check(err, check.IsNil)
260 c.Check(string(rbuf), check.Equals, "f*o")
262 // Write multiple blocks in one call
263 f.Seek(1, io.SeekStart)
264 n, err = f.Write([]byte("0123456789abcdefg"))
265 c.Check(n, check.Equals, 17)
266 c.Check(err, check.IsNil)
267 pos, err = f.Seek(0, io.SeekCurrent)
268 c.Check(pos, check.Equals, int64(18))
269 pos, err = f.Seek(-18, io.SeekCurrent)
270 c.Check(err, check.IsNil)
271 n, err = io.ReadFull(f, buf)
272 c.Check(n, check.Equals, 18)
273 c.Check(err, check.Equals, io.ErrUnexpectedEOF)
274 c.Check(string(buf[:n]), check.Equals, "f0123456789abcdefg")
276 buf2, err := ioutil.ReadAll(f2)
277 c.Check(err, check.IsNil)
278 c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
280 // truncate to current size
282 f2.Seek(0, io.SeekStart)
283 buf2, err = ioutil.ReadAll(f2)
284 c.Check(err, check.IsNil)
285 c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
287 // shrink to zero some data
289 f2.Seek(0, io.SeekStart)
290 buf2, err = ioutil.ReadAll(f2)
291 c.Check(err, check.IsNil)
292 c.Check(string(buf2), check.Equals, "f0123456789abcd")
294 // grow to partial block/extent
296 f2.Seek(0, io.SeekStart)
297 buf2, err = ioutil.ReadAll(f2)
298 c.Check(err, check.IsNil)
299 c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
302 f2.Seek(0, io.SeekStart)
303 f2.Write([]byte("12345678abcdefghijkl"))
305 // grow to block/extent boundary
307 f2.Seek(0, io.SeekStart)
308 buf2, err = ioutil.ReadAll(f2)
309 c.Check(err, check.IsNil)
310 c.Check(len(buf2), check.Equals, 64)
311 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 8)
313 // shrink to block/extent boundary
315 f2.Seek(0, io.SeekStart)
316 buf2, err = ioutil.ReadAll(f2)
317 c.Check(err, check.IsNil)
318 c.Check(len(buf2), check.Equals, 32)
319 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 4)
321 // shrink to partial block/extent
323 f2.Seek(0, io.SeekStart)
324 buf2, err = ioutil.ReadAll(f2)
325 c.Check(err, check.IsNil)
326 c.Check(string(buf2), check.Equals, "12345678abcdefg")
327 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 2)
329 // Force flush to ensure the block "12345678" gets stored, so
330 // we know what to expect in the final manifest below.
331 _, err = s.fs.MarshalManifest(".")
332 c.Check(err, check.IsNil)
334 // Truncate to size=3 while f2's ptr is at 15
336 c.Check(err, check.IsNil)
337 buf2, err = ioutil.ReadAll(f2)
338 c.Check(err, check.IsNil)
339 c.Check(string(buf2), check.Equals, "")
340 f2.Seek(0, io.SeekStart)
341 buf2, err = ioutil.ReadAll(f2)
342 c.Check(err, check.IsNil)
343 c.Check(string(buf2), check.Equals, "123")
344 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 1)
346 m, err := s.fs.MarshalManifest(".")
347 c.Check(err, check.IsNil)
348 m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
349 c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 25d55ad283aa400af464c76d713c07ad+8 3:3:bar 6:3:foo\n")
352 func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
353 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
354 c.Assert(err, check.IsNil)
355 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
356 c.Assert(err, check.IsNil)
359 checkSize := func(size int64) {
361 c.Check(fi.Size(), check.Equals, size)
363 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
364 c.Assert(err, check.IsNil)
367 c.Check(fi.Size(), check.Equals, size)
368 pos, err := f.Seek(0, io.SeekEnd)
369 c.Check(pos, check.Equals, size)
372 f.Seek(2, io.SeekEnd)
377 f.Seek(2, io.SeekCurrent)
382 f.Seek(8, io.SeekStart)
384 n, err := f.Read(make([]byte, 1))
385 c.Check(n, check.Equals, 0)
386 c.Check(err, check.Equals, io.EOF)
388 f.Write([]byte{1, 2, 3})
392 func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
394 defer func() { maxBlockSize = 2 << 26 }()
397 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
398 c.Assert(err, check.IsNil)
399 for _, name := range []string{"foo", "bar", "baz"} {
400 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
401 c.Assert(err, check.IsNil)
402 f.Write([]byte(name))
406 m, err := s.fs.MarshalManifest(".")
407 c.Check(err, check.IsNil)
408 m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
409 c.Check(m, check.Equals, ". c3c23db5285662ef7172373df0003206+6 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar 3:3:baz 6:3:foo\n")
412 func (s *CollectionFSSuite) TestMkdir(c *check.C) {
413 err := s.fs.Mkdir("foo/bar", 0755)
414 c.Check(err, check.Equals, os.ErrNotExist)
416 f, err := s.fs.OpenFile("foo/bar", os.O_CREATE, 0)
417 c.Check(err, check.Equals, os.ErrNotExist)
419 err = s.fs.Mkdir("foo", 0755)
420 c.Check(err, check.IsNil)
422 f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_WRONLY, 0)
423 c.Check(err, check.IsNil)
426 f.Write([]byte("foo"))
429 // mkdir fails if a file already exists with that name
430 err = s.fs.Mkdir("foo/bar", 0755)
431 c.Check(err, check.NotNil)
433 err = s.fs.Remove("foo/bar")
434 c.Check(err, check.IsNil)
436 // mkdir succeds after the file is deleted
437 err = s.fs.Mkdir("foo/bar", 0755)
438 c.Check(err, check.IsNil)
440 // creating a file in a nonexistent subdir should still fail
441 f, err = s.fs.OpenFile("foo/bar/baz/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
442 c.Check(err, check.Equals, os.ErrNotExist)
444 f, err = s.fs.OpenFile("foo/bar/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
445 c.Check(err, check.IsNil)
448 f.Write([]byte("foo"))
451 // creating foo/bar as a regular file should fail
452 f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, 0)
453 c.Check(err, check.NotNil)
455 // creating foo/bar as a directory should fail
456 f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, os.ModeDir)
457 c.Check(err, check.NotNil)
458 err = s.fs.Mkdir("foo/bar", 0755)
459 c.Check(err, check.NotNil)
461 m, err := s.fs.MarshalManifest(".")
462 c.Check(err, check.IsNil)
463 m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
464 c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 3:3:bar 0:3:foo\n./foo/bar acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
467 func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
469 defer func() { maxBlockSize = 2 << 26 }()
471 var wg sync.WaitGroup
472 for n := 0; n < 128; n++ {
476 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
477 c.Assert(err, check.IsNil)
479 for i := 0; i < 6502; i++ {
480 switch rand.Int() & 3 {
482 f.Truncate(int64(rand.Intn(64)))
484 f.Seek(int64(rand.Intn(64)), io.SeekStart)
486 _, err := f.Write([]byte("beep boop"))
487 c.Check(err, check.IsNil)
489 _, err := ioutil.ReadAll(f)
490 c.Check(err, check.IsNil)
497 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
498 c.Assert(err, check.IsNil)
500 buf, err := ioutil.ReadAll(f)
501 c.Check(err, check.IsNil)
502 c.Logf("after lots of random r/w/seek/trunc, buf is %q", buf)
505 func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
507 defer func() { maxBlockSize = 2 << 26 }()
510 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
511 c.Assert(err, check.IsNil)
514 const ngoroutines = 256
516 var wg sync.WaitGroup
517 for n := 0; n < nfiles; n++ {
521 expect := make([]byte, 0, 64)
522 wbytes := []byte("there's no simple explanation for anything important that any of us do")
523 f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
524 c.Assert(err, check.IsNil)
526 for i := 0; i < ngoroutines; i++ {
527 trunc := rand.Intn(65)
528 woff := rand.Intn(trunc + 1)
529 wbytes = wbytes[:rand.Intn(64-woff+1)]
530 for buf, i := expect[:cap(expect)], len(expect); i < trunc; i++ {
533 expect = expect[:trunc]
534 if trunc < woff+len(wbytes) {
535 expect = expect[:woff+len(wbytes)]
537 copy(expect[woff:], wbytes)
538 f.Truncate(int64(trunc))
539 pos, err := f.Seek(int64(woff), io.SeekStart)
540 c.Check(pos, check.Equals, int64(woff))
541 c.Check(err, check.IsNil)
542 n, err := f.Write(wbytes)
543 c.Check(n, check.Equals, len(wbytes))
544 c.Check(err, check.IsNil)
545 pos, err = f.Seek(0, io.SeekStart)
546 c.Check(pos, check.Equals, int64(0))
547 c.Check(err, check.IsNil)
548 buf, err := ioutil.ReadAll(f)
549 c.Check(string(buf), check.Equals, string(expect))
550 c.Check(err, check.IsNil)
557 root, err := s.fs.Open("/")
558 c.Assert(err, check.IsNil)
560 fi, err := root.Readdir(-1)
561 c.Check(err, check.IsNil)
562 c.Check(len(fi), check.Equals, nfiles)
564 _, err = s.fs.MarshalManifest(".")
565 c.Check(err, check.IsNil)
566 // TODO: check manifest content
569 func (s *CollectionFSSuite) TestRemove(c *check.C) {
570 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
571 c.Assert(err, check.IsNil)
572 err = fs.Mkdir("dir0", 0755)
573 c.Assert(err, check.IsNil)
574 err = fs.Mkdir("dir1", 0755)
575 c.Assert(err, check.IsNil)
576 err = fs.Mkdir("dir1/dir2", 0755)
577 c.Assert(err, check.IsNil)
578 err = fs.Mkdir("dir1/dir3", 0755)
579 c.Assert(err, check.IsNil)
581 err = fs.Remove("dir0")
582 c.Check(err, check.IsNil)
583 err = fs.Remove("dir0")
584 c.Check(err, check.Equals, os.ErrNotExist)
586 err = fs.Remove("dir1/dir2/.")
587 c.Check(err, check.Equals, ErrInvalidArgument)
588 err = fs.Remove("dir1/dir2/..")
589 c.Check(err, check.Equals, ErrInvalidArgument)
590 err = fs.Remove("dir1")
591 c.Check(err, check.Equals, ErrDirectoryNotEmpty)
592 err = fs.Remove("dir1/dir2/../../../dir1")
593 c.Check(err, check.Equals, ErrDirectoryNotEmpty)
594 err = fs.Remove("dir1/dir3/")
595 c.Check(err, check.IsNil)
596 err = fs.RemoveAll("dir1")
597 c.Check(err, check.IsNil)
598 err = fs.RemoveAll("dir1")
599 c.Check(err, check.IsNil)
602 func (s *CollectionFSSuite) TestRename(c *check.C) {
603 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
604 c.Assert(err, check.IsNil)
609 for i := 0; i < outer; i++ {
610 err = fs.Mkdir(fmt.Sprintf("dir%d", i), 0755)
611 c.Assert(err, check.IsNil)
612 for j := 0; j < inner; j++ {
613 err = fs.Mkdir(fmt.Sprintf("dir%d/dir%d", i, j), 0755)
614 c.Assert(err, check.IsNil)
615 for _, fnm := range []string{
616 fmt.Sprintf("dir%d/file%d", i, j),
617 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
619 f, err := fs.OpenFile(fnm, os.O_CREATE|os.O_WRONLY, 0755)
620 c.Assert(err, check.IsNil)
621 _, err = f.Write([]byte("beep"))
622 c.Assert(err, check.IsNil)
627 var wg sync.WaitGroup
628 for i := 0; i < outer; i++ {
629 for j := 0; j < inner; j++ {
633 oldname := fmt.Sprintf("dir%d/dir%d/file%d", i, j, j)
634 newname := fmt.Sprintf("dir%d/newfile%d", i, inner-j-1)
635 _, err := fs.Open(newname)
636 c.Check(err, check.Equals, os.ErrNotExist)
637 err = fs.Rename(oldname, newname)
638 c.Check(err, check.IsNil)
639 f, err := fs.Open(newname)
640 c.Check(err, check.IsNil)
647 // oldname does not exist
649 fmt.Sprintf("dir%d/dir%d/missing", i, j),
650 fmt.Sprintf("dir%d/dir%d/file%d", outer-i-1, j, j))
651 c.Check(err, check.ErrorMatches, `.*does not exist`)
653 // newname parent dir does not exist
655 fmt.Sprintf("dir%d/dir%d", i, j),
656 fmt.Sprintf("dir%d/missing/irrelevant", outer-i-1))
657 c.Check(err, check.ErrorMatches, `.*does not exist`)
659 // oldname parent dir is a file
661 fmt.Sprintf("dir%d/file%d/patherror", i, j),
662 fmt.Sprintf("dir%d/irrelevant", i))
663 c.Check(err, check.ErrorMatches, `.*does not exist`)
665 // newname parent dir is a file
667 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
668 fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
669 c.Check(err, check.ErrorMatches, `.*does not exist`)
675 f, err := fs.OpenFile("dir1/newfile3", 0, 0)
676 c.Assert(err, check.IsNil)
677 c.Check(f.Size(), check.Equals, int64(4))
678 buf, err := ioutil.ReadAll(f)
679 c.Check(buf, check.DeepEquals, []byte("beep"))
680 c.Check(err, check.IsNil)
681 _, err = fs.Open("dir1/dir1/file1")
682 c.Check(err, check.Equals, os.ErrNotExist)
685 func (s *CollectionFSSuite) TestPersist(c *check.C) {
687 defer func() { maxBlockSize = 2 << 26 }()
690 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
691 c.Assert(err, check.IsNil)
692 err = s.fs.Mkdir("d:r", 0755)
693 c.Assert(err, check.IsNil)
695 expect := map[string][]byte{}
697 var wg sync.WaitGroup
698 for _, name := range []string{"random 1", "random:2", "random\\3", "d:r/random4"} {
699 buf := make([]byte, 500)
703 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
704 c.Assert(err, check.IsNil)
705 // Note: we don't close the file until after the test
706 // is done. Writes to unclosed files should persist.
712 for i := 0; i < len(buf); i += 5 {
713 _, err := f.Write(buf[i : i+5])
714 c.Assert(err, check.IsNil)
720 m, err := s.fs.MarshalManifest(".")
721 c.Check(err, check.IsNil)
724 root, err := s.fs.Open("/")
725 c.Assert(err, check.IsNil)
727 fi, err := root.Readdir(-1)
728 c.Check(err, check.IsNil)
729 c.Check(len(fi), check.Equals, 4)
731 persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
732 c.Assert(err, check.IsNil)
734 root, err = persisted.Open("/")
735 c.Assert(err, check.IsNil)
737 fi, err = root.Readdir(-1)
738 c.Check(err, check.IsNil)
739 c.Check(len(fi), check.Equals, 4)
741 for name, content := range expect {
742 c.Logf("read %q", name)
743 f, err := persisted.Open(name)
744 c.Assert(err, check.IsNil)
746 buf, err := ioutil.ReadAll(f)
747 c.Check(err, check.IsNil)
748 c.Check(buf, check.DeepEquals, content)
752 func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
754 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
755 c.Assert(err, check.IsNil)
756 for _, name := range []string{"dir", "dir/zerodir", "zero", "zero/zero"} {
757 err = s.fs.Mkdir(name, 0755)
758 c.Assert(err, check.IsNil)
761 expect := map[string][]byte{
766 "dir/two": []byte{1, 2},
768 "dir/zerodir/zero": nil,
769 "zero/zero/zero": nil,
771 for name, data := range expect {
772 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
773 c.Assert(err, check.IsNil)
775 _, err := f.Write(data)
776 c.Assert(err, check.IsNil)
781 m, err := s.fs.MarshalManifest(".")
782 c.Check(err, check.IsNil)
785 persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
786 c.Assert(err, check.IsNil)
788 for name, data := range expect {
789 f, err := persisted.Open("bogus-" + name)
790 c.Check(err, check.NotNil)
792 f, err = persisted.Open(name)
793 c.Assert(err, check.IsNil)
798 buf, err := ioutil.ReadAll(f)
799 c.Check(err, check.IsNil)
800 c.Check(buf, check.DeepEquals, data)
804 func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
805 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
806 c.Assert(err, check.IsNil)
808 f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
809 c.Check(f, check.IsNil)
810 c.Check(err, check.ErrorMatches, `file does not exist`)
812 f, err = fs.OpenFile("new", os.O_CREATE|os.O_RDONLY, 0)
813 c.Assert(err, check.IsNil)
815 n, err := f.Write([]byte{1, 2, 3})
816 c.Check(n, check.Equals, 0)
817 c.Check(err, check.ErrorMatches, `read-only file`)
818 n, err = f.Read(make([]byte, 1))
819 c.Check(n, check.Equals, 0)
820 c.Check(err, check.Equals, io.EOF)
821 f, err = fs.OpenFile("new", os.O_RDWR, 0)
822 c.Assert(err, check.IsNil)
824 _, err = f.Write([]byte{4, 5, 6})
825 c.Check(err, check.IsNil)
827 c.Assert(err, check.IsNil)
828 c.Check(fi.Size(), check.Equals, int64(3))
830 f, err = fs.OpenFile("new", os.O_TRUNC|os.O_RDWR, 0)
831 c.Assert(err, check.IsNil)
833 pos, err := f.Seek(0, io.SeekEnd)
834 c.Check(pos, check.Equals, int64(0))
835 c.Check(err, check.IsNil)
837 c.Assert(err, check.IsNil)
838 c.Check(fi.Size(), check.Equals, int64(0))
841 buf := make([]byte, 64)
842 f, err = fs.OpenFile("append", os.O_EXCL|os.O_CREATE|os.O_RDWR|os.O_APPEND, 0)
843 c.Assert(err, check.IsNil)
844 f.Write([]byte{1, 2, 3})
845 f.Seek(0, io.SeekStart)
846 n, _ = f.Read(buf[:1])
847 c.Check(n, check.Equals, 1)
848 c.Check(buf[:1], check.DeepEquals, []byte{1})
849 pos, err = f.Seek(0, io.SeekCurrent)
850 c.Check(pos, check.Equals, int64(1))
851 f.Write([]byte{4, 5, 6})
852 pos, err = f.Seek(0, io.SeekCurrent)
853 c.Check(pos, check.Equals, int64(6))
854 f.Seek(0, io.SeekStart)
856 c.Check(buf[:n], check.DeepEquals, []byte{1, 2, 3, 4, 5, 6})
857 c.Check(err, check.Equals, io.EOF)
860 f, err = fs.OpenFile("append", os.O_RDWR|os.O_APPEND, 0)
861 c.Assert(err, check.IsNil)
862 pos, err = f.Seek(0, io.SeekCurrent)
863 c.Check(pos, check.Equals, int64(0))
864 c.Check(err, check.IsNil)
866 pos, _ = f.Seek(0, io.SeekCurrent)
867 c.Check(pos, check.Equals, int64(3))
868 f.Write([]byte{7, 8, 9})
869 pos, err = f.Seek(0, io.SeekCurrent)
870 c.Check(pos, check.Equals, int64(9))
873 f, err = fs.OpenFile("wronly", os.O_CREATE|os.O_WRONLY, 0)
874 c.Assert(err, check.IsNil)
875 n, err = f.Write([]byte{3, 2, 1})
876 c.Check(n, check.Equals, 3)
877 c.Check(err, check.IsNil)
878 pos, _ = f.Seek(0, io.SeekCurrent)
879 c.Check(pos, check.Equals, int64(3))
880 pos, _ = f.Seek(0, io.SeekStart)
881 c.Check(pos, check.Equals, int64(0))
883 c.Check(n, check.Equals, 0)
884 c.Check(err, check.ErrorMatches, `.*O_WRONLY.*`)
885 f, err = fs.OpenFile("wronly", os.O_RDONLY, 0)
886 c.Assert(err, check.IsNil)
888 c.Check(buf[:n], check.DeepEquals, []byte{3, 2, 1})
890 f, err = fs.OpenFile("unsupported", os.O_CREATE|os.O_SYNC, 0)
891 c.Check(f, check.IsNil)
892 c.Check(err, check.NotNil)
894 f, err = fs.OpenFile("append", os.O_RDWR|os.O_WRONLY, 0)
895 c.Check(f, check.IsNil)
896 c.Check(err, check.ErrorMatches, `invalid flag.*`)
899 func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
901 defer func() { maxBlockSize = 2 << 26 }()
903 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
904 c.Assert(err, check.IsNil)
905 f, err := fs.OpenFile("50K", os.O_WRONLY|os.O_CREATE, 0)
906 c.Assert(err, check.IsNil)
909 data := make([]byte, 500)
912 for i := 0; i < 100; i++ {
913 n, err := f.Write(data)
914 c.Assert(n, check.Equals, len(data))
915 c.Assert(err, check.IsNil)
918 currentMemExtents := func() (memExtents []int) {
919 for idx, e := range f.(*filehandle).inode.(*filenode).segments {
922 memExtents = append(memExtents, idx)
927 c.Check(currentMemExtents(), check.HasLen, 1)
929 m, err := fs.MarshalManifest(".")
930 c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
931 c.Check(err, check.IsNil)
932 c.Check(currentMemExtents(), check.HasLen, 0)
935 func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
936 for _, txt := range []string{
940 ". d41d8cd98f00b204e9800998ecf8427e+0\n",
941 ". d41d8cd98f00b204e9800998ecf8427e+0 \n",
944 ". 0:0:foo 0:0:bar\n",
945 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo\n",
946 ". d41d8cd98f00b204e9800998ecf8427e+0 :0:0:foo\n",
947 ". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
948 ". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
949 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
950 ". d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n",
951 "./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
954 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
955 c.Check(fs, check.IsNil)
957 c.Check(err, check.NotNil)
961 func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
962 for _, txt := range []string{
964 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
965 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
966 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
967 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
970 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
971 c.Check(err, check.IsNil)
972 c.Check(fs, check.NotNil)
976 func (s *CollectionFSSuite) checkMemSize(c *check.C, f File) {
977 fn := f.(*filehandle).inode.(*filenode)
979 for _, seg := range fn.segments {
980 if e, ok := seg.(*memSegment); ok {
981 memsize += int64(len(e.buf))
984 c.Check(fn.memsize, check.Equals, memsize)
987 type CollectionFSUnitSuite struct{}
989 var _ = check.Suite(&CollectionFSUnitSuite{})
991 // expect ~2 seconds to load a manifest with 256K files
992 func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
998 mb := bytes.NewBuffer(make([]byte, 0, 40000000))
999 for i := 0; i < dirCount; i++ {
1000 fmt.Fprintf(mb, "./dir%d", i)
1001 for j := 0; j <= fileCount; j++ {
1002 fmt.Fprintf(mb, " %032x+42+A%040x@%08x", j, j, j)
1004 for j := 0; j < fileCount; j++ {
1005 fmt.Fprintf(mb, " %d:%d:dir%d/file%d", j*42+21, 42, j, j)
1007 mb.Write([]byte{'\n'})
1009 coll := Collection{ManifestText: mb.String()}
1010 c.Logf("%s built", time.Now())
1012 var memstats runtime.MemStats
1013 runtime.ReadMemStats(&memstats)
1014 c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1016 f, err := coll.FileSystem(nil, nil)
1017 c.Check(err, check.IsNil)
1018 c.Logf("%s loaded", time.Now())
1020 for i := 0; i < dirCount; i++ {
1021 for j := 0; j < fileCount; j++ {
1022 f.Stat(fmt.Sprintf("./dir%d/dir%d/file%d", i, j, j))
1025 c.Logf("%s Stat() x %d", time.Now(), dirCount*fileCount)
1027 runtime.ReadMemStats(&memstats)
1028 c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1031 // Gocheck boilerplate
1032 func Test(t *testing.T) {