1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
25 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
26 check "gopkg.in/check.v1"
29 var _ = check.Suite(&CollectionFSSuite{})
31 type keepClientStub struct {
32 blocks map[string][]byte
33 refreshable map[string]bool
37 var errStub404 = errors.New("404 block not found")
39 func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error) {
42 buf := kcs.blocks[locator[:32]]
46 return copy(p, buf[off:]), nil
49 func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
50 locator := fmt.Sprintf("%x+%d+A12345@abcde", md5.Sum(p), len(p))
51 buf := make([]byte, len(p))
55 kcs.blocks[locator[:32]] = buf
56 return locator, 1, nil
59 var localOrRemoteSignature = regexp.MustCompile(`\+[AR][^+]*`)
61 func (kcs *keepClientStub) LocalLocator(locator string) (string, error) {
64 if strings.Contains(locator, "+R") {
65 if len(locator) < 32 {
66 return "", fmt.Errorf("bad locator: %q", locator)
68 if _, ok := kcs.blocks[locator[:32]]; !ok && !kcs.refreshable[locator[:32]] {
69 return "", fmt.Errorf("kcs.refreshable[%q]==false", locator)
72 fakeSig := fmt.Sprintf("+A%x@%x", sha1.Sum(nil), time.Now().Add(time.Hour*24*14).Unix())
73 return localOrRemoteSignature.ReplaceAllLiteralString(locator, fakeSig), nil
76 type CollectionFSSuite struct {
79 fs CollectionFileSystem
83 func (s *CollectionFSSuite) SetUpTest(c *check.C) {
84 s.client = NewClientFromEnv()
85 err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
86 c.Assert(err, check.IsNil)
87 s.kc = &keepClientStub{
88 blocks: map[string][]byte{
89 "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
91 s.fs, err = s.coll.FileSystem(s.client, s.kc)
92 c.Assert(err, check.IsNil)
95 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
96 _, ok := s.fs.(http.FileSystem)
97 c.Check(ok, check.Equals, true)
100 func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
101 fs, err := (&Collection{
102 ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
103 }).FileSystem(s.client, s.kc)
104 c.Assert(err, check.IsNil)
106 f, err := fs.Open("/foo:foo")
107 c.Assert(err, check.IsNil)
109 fis, err := f.Readdir(0)
110 c.Check(err, check.IsNil)
111 c.Check(len(fis), check.Equals, 1)
112 c.Check(fis[0].Name(), check.Equals, "bar:bar")
115 func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
116 f, err := s.fs.Open("/dir1")
117 c.Assert(err, check.IsNil)
120 c.Assert(err, check.IsNil)
121 c.Check(st.Size(), check.Equals, int64(2))
122 c.Check(st.IsDir(), check.Equals, true)
124 fis, err := f.Readdir(0)
125 c.Check(err, check.IsNil)
126 c.Check(len(fis), check.Equals, 2)
128 c.Check(fis[0].Size(), check.Equals, int64(3))
132 func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
133 f, err := s.fs.Open("./dir1")
134 c.Assert(err, check.IsNil)
136 fis, err := f.Readdir(1)
137 c.Check(err, check.IsNil)
138 c.Check(len(fis), check.Equals, 1)
140 c.Check(fis[0].Size(), check.Equals, int64(3))
143 fis, err = f.Readdir(1)
144 c.Check(err, check.IsNil)
145 c.Check(len(fis), check.Equals, 1)
147 c.Check(fis[0].Size(), check.Equals, int64(3))
150 fis, err = f.Readdir(1)
151 c.Check(len(fis), check.Equals, 0)
152 c.Check(err, check.NotNil)
153 c.Check(err, check.Equals, io.EOF)
155 f, err = s.fs.Open("dir1")
156 c.Assert(err, check.IsNil)
157 fis, err = f.Readdir(1)
158 c.Check(len(fis), check.Equals, 1)
159 c.Assert(err, check.IsNil)
160 fis, err = f.Readdir(2)
161 c.Check(len(fis), check.Equals, 1)
162 c.Assert(err, check.IsNil)
163 fis, err = f.Readdir(2)
164 c.Check(len(fis), check.Equals, 0)
165 c.Assert(err, check.Equals, io.EOF)
168 func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
169 for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
170 f, err := s.fs.Open(path)
171 c.Assert(err, check.IsNil)
174 c.Assert(err, check.IsNil)
175 c.Check(st.Size(), check.Equals, int64(1))
176 c.Check(st.IsDir(), check.Equals, true)
178 for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
180 f, err := s.fs.Open(path)
181 c.Assert(err, check.IsNil)
184 c.Assert(err, check.IsNil)
185 c.Check(st.Size(), check.Equals, int64(2))
186 c.Check(st.IsDir(), check.Equals, true)
190 func (s *CollectionFSSuite) TestNotExist(c *check.C) {
191 for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
192 f, err := s.fs.Open(path)
193 c.Assert(f, check.IsNil)
194 c.Assert(err, check.NotNil)
195 c.Assert(os.IsNotExist(err), check.Equals, true)
199 func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
200 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDONLY, 0)
201 c.Assert(err, check.IsNil)
203 c.Assert(err, check.IsNil)
204 c.Check(st.Size(), check.Equals, int64(3))
205 n, err := f.Write([]byte("bar"))
206 c.Check(n, check.Equals, 0)
207 c.Check(err, check.Equals, ErrReadOnlyFile)
210 func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
211 f, err := s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE, 0)
212 c.Assert(err, check.IsNil)
214 c.Assert(err, check.IsNil)
215 c.Check(st.Size(), check.Equals, int64(0))
217 n, err := f.Write([]byte("bar"))
218 c.Check(n, check.Equals, 3)
219 c.Check(err, check.IsNil)
221 c.Check(f.Close(), check.IsNil)
223 f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
224 c.Check(f, check.IsNil)
225 c.Assert(err, check.NotNil)
227 f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR, 0)
228 c.Assert(err, check.IsNil)
230 c.Assert(err, check.IsNil)
231 c.Check(st.Size(), check.Equals, int64(3))
233 c.Check(f.Close(), check.IsNil)
235 m, err := s.fs.MarshalManifest(".")
236 c.Assert(err, check.IsNil)
237 c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
240 func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
242 defer func() { maxBlockSize = 2 << 26 }()
244 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
245 c.Assert(err, check.IsNil)
248 c.Assert(err, check.IsNil)
249 c.Check(st.Size(), check.Equals, int64(3))
251 f2, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
252 c.Assert(err, check.IsNil)
255 buf := make([]byte, 64)
256 n, err := f.Read(buf)
257 c.Check(n, check.Equals, 3)
258 c.Check(err, check.Equals, io.EOF)
259 c.Check(string(buf[:3]), check.DeepEquals, "foo")
261 pos, err := f.Seek(-2, io.SeekCurrent)
262 c.Check(pos, check.Equals, int64(1))
263 c.Check(err, check.IsNil)
265 // Split a storedExtent in two, and insert a memExtent
266 n, err = f.Write([]byte("*"))
267 c.Check(n, check.Equals, 1)
268 c.Check(err, check.IsNil)
270 pos, err = f.Seek(0, io.SeekCurrent)
271 c.Check(pos, check.Equals, int64(2))
272 c.Check(err, check.IsNil)
274 pos, err = f.Seek(0, io.SeekStart)
275 c.Check(pos, check.Equals, int64(0))
276 c.Check(err, check.IsNil)
278 rbuf, err := ioutil.ReadAll(f)
279 c.Check(len(rbuf), check.Equals, 3)
280 c.Check(err, check.IsNil)
281 c.Check(string(rbuf), check.Equals, "f*o")
283 // Write multiple blocks in one call
284 f.Seek(1, io.SeekStart)
285 n, err = f.Write([]byte("0123456789abcdefg"))
286 c.Check(n, check.Equals, 17)
287 c.Check(err, check.IsNil)
288 pos, err = f.Seek(0, io.SeekCurrent)
289 c.Check(pos, check.Equals, int64(18))
290 c.Check(err, check.IsNil)
291 pos, err = f.Seek(-18, io.SeekCurrent)
292 c.Check(pos, check.Equals, int64(0))
293 c.Check(err, check.IsNil)
294 n, err = io.ReadFull(f, buf)
295 c.Check(n, check.Equals, 18)
296 c.Check(err, check.Equals, io.ErrUnexpectedEOF)
297 c.Check(string(buf[:n]), check.Equals, "f0123456789abcdefg")
299 buf2, err := ioutil.ReadAll(f2)
300 c.Check(err, check.IsNil)
301 c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
303 // truncate to current size
305 c.Check(err, check.IsNil)
306 f2.Seek(0, io.SeekStart)
307 buf2, err = ioutil.ReadAll(f2)
308 c.Check(err, check.IsNil)
309 c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
311 // shrink to zero some data
313 f2.Seek(0, io.SeekStart)
314 buf2, err = ioutil.ReadAll(f2)
315 c.Check(err, check.IsNil)
316 c.Check(string(buf2), check.Equals, "f0123456789abcd")
318 // grow to partial block/extent
320 f2.Seek(0, io.SeekStart)
321 buf2, err = ioutil.ReadAll(f2)
322 c.Check(err, check.IsNil)
323 c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
326 f2.Seek(0, io.SeekStart)
327 f2.Write([]byte("12345678abcdefghijkl"))
329 // grow to block/extent boundary
331 f2.Seek(0, io.SeekStart)
332 buf2, err = ioutil.ReadAll(f2)
333 c.Check(err, check.IsNil)
334 c.Check(len(buf2), check.Equals, 64)
335 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 8)
337 // shrink to block/extent boundary
339 c.Check(err, check.IsNil)
340 f2.Seek(0, io.SeekStart)
341 buf2, err = ioutil.ReadAll(f2)
342 c.Check(err, check.IsNil)
343 c.Check(len(buf2), check.Equals, 32)
344 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 4)
346 // shrink to partial block/extent
348 c.Check(err, check.IsNil)
349 f2.Seek(0, io.SeekStart)
350 buf2, err = ioutil.ReadAll(f2)
351 c.Check(err, check.IsNil)
352 c.Check(string(buf2), check.Equals, "12345678abcdefg")
353 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 2)
355 // Force flush to ensure the block "12345678" gets stored, so
356 // we know what to expect in the final manifest below.
357 _, err = s.fs.MarshalManifest(".")
358 c.Check(err, check.IsNil)
360 // Truncate to size=3 while f2's ptr is at 15
362 c.Check(err, check.IsNil)
363 buf2, err = ioutil.ReadAll(f2)
364 c.Check(err, check.IsNil)
365 c.Check(string(buf2), check.Equals, "")
366 f2.Seek(0, io.SeekStart)
367 buf2, err = ioutil.ReadAll(f2)
368 c.Check(err, check.IsNil)
369 c.Check(string(buf2), check.Equals, "123")
370 c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 1)
372 m, err := s.fs.MarshalManifest(".")
373 c.Check(err, check.IsNil)
374 m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
375 c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 25d55ad283aa400af464c76d713c07ad+8 3:3:bar 6:3:foo\n")
376 c.Check(s.fs.Size(), check.Equals, int64(6))
379 func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
380 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
381 c.Assert(err, check.IsNil)
382 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
383 c.Assert(err, check.IsNil)
386 checkSize := func(size int64) {
388 c.Assert(err, check.IsNil)
389 c.Check(fi.Size(), check.Equals, size)
391 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
392 c.Assert(err, check.IsNil)
395 c.Check(err, check.IsNil)
396 c.Check(fi.Size(), check.Equals, size)
397 pos, err := f.Seek(0, io.SeekEnd)
398 c.Check(err, check.IsNil)
399 c.Check(pos, check.Equals, size)
402 f.Seek(2, io.SeekEnd)
407 f.Seek(2, io.SeekCurrent)
412 f.Seek(8, io.SeekStart)
414 n, err := f.Read(make([]byte, 1))
415 c.Check(n, check.Equals, 0)
416 c.Check(err, check.Equals, io.EOF)
418 f.Write([]byte{1, 2, 3})
422 func (s *CollectionFSSuite) TestMarshalCopiesRemoteBlocks(c *check.C) {
425 hash := map[string]string{
426 foo: fmt.Sprintf("%x", md5.Sum([]byte(foo))),
427 bar: fmt.Sprintf("%x", md5.Sum([]byte(bar))),
430 fs, err := (&Collection{
431 ManifestText: ". " + hash[foo] + "+3+Rzaaaa-foo@bab " + hash[bar] + "+3+A12345@ffffff 0:2:fo.txt 2:4:obar.txt\n",
432 }).FileSystem(s.client, s.kc)
433 c.Assert(err, check.IsNil)
434 manifest, err := fs.MarshalManifest(".")
435 c.Check(manifest, check.Equals, "")
436 c.Check(err, check.NotNil)
438 s.kc.refreshable = map[string]bool{hash[bar]: true}
440 for _, sigIn := range []string{"Rzaaaa-foo@bab", "A12345@abcde"} {
441 fs, err = (&Collection{
442 ManifestText: ". " + hash[foo] + "+3+A12345@fffff " + hash[bar] + "+3+" + sigIn + " 0:2:fo.txt 2:4:obar.txt\n",
443 }).FileSystem(s.client, s.kc)
444 c.Assert(err, check.IsNil)
445 manifest, err := fs.MarshalManifest(".")
446 c.Check(err, check.IsNil)
447 // Both blocks should now have +A signatures.
448 c.Check(manifest, check.Matches, `.*\+A.* .*\+A.*\n`)
449 c.Check(manifest, check.Not(check.Matches), `.*\+R.*\n`)
453 func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
455 defer func() { maxBlockSize = 2 << 26 }()
458 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
459 c.Assert(err, check.IsNil)
460 for _, name := range []string{"foo", "bar", "baz"} {
461 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
462 c.Assert(err, check.IsNil)
463 f.Write([]byte(name))
467 m, err := s.fs.MarshalManifest(".")
468 c.Check(err, check.IsNil)
469 m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
470 c.Check(m, check.Equals, ". c3c23db5285662ef7172373df0003206+6 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar 3:3:baz 6:3:foo\n")
473 func (s *CollectionFSSuite) TestMkdir(c *check.C) {
474 err := s.fs.Mkdir("foo/bar", 0755)
475 c.Check(err, check.Equals, os.ErrNotExist)
477 f, err := s.fs.OpenFile("foo/bar", os.O_CREATE, 0)
478 c.Check(err, check.Equals, os.ErrNotExist)
480 err = s.fs.Mkdir("foo", 0755)
481 c.Check(err, check.IsNil)
483 f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_WRONLY, 0)
484 c.Check(err, check.IsNil)
487 f.Write([]byte("foo"))
490 // mkdir fails if a file already exists with that name
491 err = s.fs.Mkdir("foo/bar", 0755)
492 c.Check(err, check.NotNil)
494 err = s.fs.Remove("foo/bar")
495 c.Check(err, check.IsNil)
497 // mkdir succeeds after the file is deleted
498 err = s.fs.Mkdir("foo/bar", 0755)
499 c.Check(err, check.IsNil)
501 // creating a file in a nonexistent subdir should still fail
502 f, err = s.fs.OpenFile("foo/bar/baz/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
503 c.Check(err, check.Equals, os.ErrNotExist)
505 f, err = s.fs.OpenFile("foo/bar/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
506 c.Check(err, check.IsNil)
509 f.Write([]byte("foo"))
512 // creating foo/bar as a regular file should fail
513 f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, 0)
514 c.Check(err, check.NotNil)
516 // creating foo/bar as a directory should fail
517 f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, os.ModeDir)
518 c.Check(err, check.NotNil)
519 err = s.fs.Mkdir("foo/bar", 0755)
520 c.Check(err, check.NotNil)
522 m, err := s.fs.MarshalManifest(".")
523 c.Check(err, check.IsNil)
524 m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
525 c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 3:3:bar 0:3:foo\n./foo/bar acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
528 func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
534 defer func() { maxBlockSize = 2 << 26 }()
536 var wg sync.WaitGroup
537 for n := 0; n < 128; n++ {
541 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
542 c.Assert(err, check.IsNil)
544 for i := 0; i < 1024; i++ {
548 _, err := s.fs.MarshalManifest(".")
549 c.Check(err, check.IsNil)
551 f.Truncate(int64(rand.Intn(64)))
553 f.Seek(int64(rand.Intn(64)), io.SeekStart)
555 _, err := f.Write([]byte("beep boop"))
556 c.Check(err, check.IsNil)
558 _, err := ioutil.ReadAll(f)
559 c.Check(err, check.IsNil)
566 f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
567 c.Assert(err, check.IsNil)
569 buf, err := ioutil.ReadAll(f)
570 c.Check(err, check.IsNil)
571 c.Logf("after lots of random r/w/seek/trunc, buf is %q", buf)
574 func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
576 defer func() { maxBlockSize = 2 << 26 }()
579 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
580 c.Assert(err, check.IsNil)
583 const ngoroutines = 256
585 var wg sync.WaitGroup
586 for n := 0; n < nfiles; n++ {
590 expect := make([]byte, 0, 64)
591 wbytes := []byte("there's no simple explanation for anything important that any of us do")
592 f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
593 c.Assert(err, check.IsNil)
595 for i := 0; i < ngoroutines; i++ {
596 trunc := rand.Intn(65)
597 woff := rand.Intn(trunc + 1)
598 wbytes = wbytes[:rand.Intn(64-woff+1)]
599 for buf, i := expect[:cap(expect)], len(expect); i < trunc; i++ {
602 expect = expect[:trunc]
603 if trunc < woff+len(wbytes) {
604 expect = expect[:woff+len(wbytes)]
606 copy(expect[woff:], wbytes)
607 f.Truncate(int64(trunc))
608 pos, err := f.Seek(int64(woff), io.SeekStart)
609 c.Check(pos, check.Equals, int64(woff))
610 c.Check(err, check.IsNil)
611 n, err := f.Write(wbytes)
612 c.Check(n, check.Equals, len(wbytes))
613 c.Check(err, check.IsNil)
614 pos, err = f.Seek(0, io.SeekStart)
615 c.Check(pos, check.Equals, int64(0))
616 c.Check(err, check.IsNil)
617 buf, err := ioutil.ReadAll(f)
618 c.Check(string(buf), check.Equals, string(expect))
619 c.Check(err, check.IsNil)
626 root, err := s.fs.Open("/")
627 c.Assert(err, check.IsNil)
629 fi, err := root.Readdir(-1)
630 c.Check(err, check.IsNil)
631 c.Check(len(fi), check.Equals, nfiles)
633 _, err = s.fs.MarshalManifest(".")
634 c.Check(err, check.IsNil)
635 // TODO: check manifest content
638 func (s *CollectionFSSuite) TestRemove(c *check.C) {
639 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
640 c.Assert(err, check.IsNil)
641 err = fs.Mkdir("dir0", 0755)
642 c.Assert(err, check.IsNil)
643 err = fs.Mkdir("dir1", 0755)
644 c.Assert(err, check.IsNil)
645 err = fs.Mkdir("dir1/dir2", 0755)
646 c.Assert(err, check.IsNil)
647 err = fs.Mkdir("dir1/dir3", 0755)
648 c.Assert(err, check.IsNil)
650 err = fs.Remove("dir0")
651 c.Check(err, check.IsNil)
652 err = fs.Remove("dir0")
653 c.Check(err, check.Equals, os.ErrNotExist)
655 err = fs.Remove("dir1/dir2/.")
656 c.Check(err, check.Equals, ErrInvalidArgument)
657 err = fs.Remove("dir1/dir2/..")
658 c.Check(err, check.Equals, ErrInvalidArgument)
659 err = fs.Remove("dir1")
660 c.Check(err, check.Equals, ErrDirectoryNotEmpty)
661 err = fs.Remove("dir1/dir2/../../../dir1")
662 c.Check(err, check.Equals, ErrDirectoryNotEmpty)
663 err = fs.Remove("dir1/dir3/")
664 c.Check(err, check.IsNil)
665 err = fs.RemoveAll("dir1")
666 c.Check(err, check.IsNil)
667 err = fs.RemoveAll("dir1")
668 c.Check(err, check.IsNil)
671 func (s *CollectionFSSuite) TestRenameError(c *check.C) {
672 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
673 c.Assert(err, check.IsNil)
674 err = fs.Mkdir("first", 0755)
675 c.Assert(err, check.IsNil)
676 err = fs.Mkdir("first/second", 0755)
677 c.Assert(err, check.IsNil)
678 f, err := fs.OpenFile("first/second/file", os.O_CREATE|os.O_WRONLY, 0755)
679 c.Assert(err, check.IsNil)
680 f.Write([]byte{1, 2, 3, 4, 5})
682 err = fs.Rename("first", "first/second/third")
683 c.Check(err, check.Equals, ErrInvalidArgument)
684 err = fs.Rename("first", "first/third")
685 c.Check(err, check.Equals, ErrInvalidArgument)
686 err = fs.Rename("first/second", "second")
687 c.Check(err, check.IsNil)
688 f, err = fs.OpenFile("second/file", 0, 0)
689 c.Assert(err, check.IsNil)
690 data, err := ioutil.ReadAll(f)
691 c.Check(err, check.IsNil)
692 c.Check(data, check.DeepEquals, []byte{1, 2, 3, 4, 5})
695 func (s *CollectionFSSuite) TestRename(c *check.C) {
696 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
697 c.Assert(err, check.IsNil)
702 for i := 0; i < outer; i++ {
703 err = fs.Mkdir(fmt.Sprintf("dir%d", i), 0755)
704 c.Assert(err, check.IsNil)
705 for j := 0; j < inner; j++ {
706 err = fs.Mkdir(fmt.Sprintf("dir%d/dir%d", i, j), 0755)
707 c.Assert(err, check.IsNil)
708 for _, fnm := range []string{
709 fmt.Sprintf("dir%d/file%d", i, j),
710 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
712 f, err := fs.OpenFile(fnm, os.O_CREATE|os.O_WRONLY, 0755)
713 c.Assert(err, check.IsNil)
714 _, err = f.Write([]byte("beep"))
715 c.Assert(err, check.IsNil)
720 var wg sync.WaitGroup
721 for i := 0; i < outer; i++ {
722 for j := 0; j < inner; j++ {
726 oldname := fmt.Sprintf("dir%d/dir%d/file%d", i, j, j)
727 newname := fmt.Sprintf("dir%d/newfile%d", i, inner-j-1)
728 _, err := fs.Open(newname)
729 c.Check(err, check.Equals, os.ErrNotExist)
730 err = fs.Rename(oldname, newname)
731 c.Check(err, check.IsNil)
732 f, err := fs.Open(newname)
733 c.Check(err, check.IsNil)
740 // oldname does not exist
742 fmt.Sprintf("dir%d/dir%d/missing", i, j),
743 fmt.Sprintf("dir%d/dir%d/file%d", outer-i-1, j, j))
744 c.Check(err, check.ErrorMatches, `.*does not exist`)
746 // newname parent dir does not exist
748 fmt.Sprintf("dir%d/dir%d", i, j),
749 fmt.Sprintf("dir%d/missing/irrelevant", outer-i-1))
750 c.Check(err, check.ErrorMatches, `.*does not exist`)
752 // oldname parent dir is a file
754 fmt.Sprintf("dir%d/file%d/patherror", i, j),
755 fmt.Sprintf("dir%d/irrelevant", i))
756 c.Check(err, check.ErrorMatches, `.*not a directory`)
758 // newname parent dir is a file
760 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
761 fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
762 c.Check(err, check.ErrorMatches, `.*not a directory`)
768 f, err := fs.OpenFile("dir1/newfile3", 0, 0)
769 c.Assert(err, check.IsNil)
770 c.Check(f.Size(), check.Equals, int64(4))
771 buf, err := ioutil.ReadAll(f)
772 c.Check(buf, check.DeepEquals, []byte("beep"))
773 c.Check(err, check.IsNil)
774 _, err = fs.Open("dir1/dir1/file1")
775 c.Check(err, check.Equals, os.ErrNotExist)
778 func (s *CollectionFSSuite) TestPersist(c *check.C) {
780 defer func() { maxBlockSize = 2 << 26 }()
783 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
784 c.Assert(err, check.IsNil)
785 err = s.fs.Mkdir("d:r", 0755)
786 c.Assert(err, check.IsNil)
788 expect := map[string][]byte{}
790 var wg sync.WaitGroup
791 for _, name := range []string{"random 1", "random:2", "random\\3", "d:r/random4"} {
792 buf := make([]byte, 500)
796 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
797 c.Assert(err, check.IsNil)
798 // Note: we don't close the file until after the test
799 // is done. Writes to unclosed files should persist.
805 for i := 0; i < len(buf); i += 5 {
806 _, err := f.Write(buf[i : i+5])
807 c.Assert(err, check.IsNil)
813 m, err := s.fs.MarshalManifest(".")
814 c.Check(err, check.IsNil)
817 root, err := s.fs.Open("/")
818 c.Assert(err, check.IsNil)
820 fi, err := root.Readdir(-1)
821 c.Check(err, check.IsNil)
822 c.Check(len(fi), check.Equals, 4)
824 persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
825 c.Assert(err, check.IsNil)
827 root, err = persisted.Open("/")
828 c.Assert(err, check.IsNil)
830 fi, err = root.Readdir(-1)
831 c.Check(err, check.IsNil)
832 c.Check(len(fi), check.Equals, 4)
834 for name, content := range expect {
835 c.Logf("read %q", name)
836 f, err := persisted.Open(name)
837 c.Assert(err, check.IsNil)
839 buf, err := ioutil.ReadAll(f)
840 c.Check(err, check.IsNil)
841 c.Check(buf, check.DeepEquals, content)
845 func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
847 s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
848 c.Assert(err, check.IsNil)
849 for _, name := range []string{"dir", "dir/zerodir", "zero", "zero/zero"} {
850 err = s.fs.Mkdir(name, 0755)
851 c.Assert(err, check.IsNil)
854 expect := map[string][]byte{
861 "dir/zerodir/zero": nil,
862 "zero/zero/zero": nil,
864 for name, data := range expect {
865 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
866 c.Assert(err, check.IsNil)
868 _, err := f.Write(data)
869 c.Assert(err, check.IsNil)
874 m, err := s.fs.MarshalManifest(".")
875 c.Check(err, check.IsNil)
878 persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
879 c.Assert(err, check.IsNil)
881 for name, data := range expect {
882 _, err = persisted.Open("bogus-" + name)
883 c.Check(err, check.NotNil)
885 f, err := persisted.Open(name)
886 c.Assert(err, check.IsNil)
891 buf, err := ioutil.ReadAll(f)
892 c.Check(err, check.IsNil)
893 c.Check(buf, check.DeepEquals, data)
897 func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
898 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
899 c.Assert(err, check.IsNil)
901 f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
902 c.Check(f, check.IsNil)
903 c.Check(err, check.ErrorMatches, `file does not exist`)
905 f, err = fs.OpenFile("new", os.O_CREATE|os.O_RDONLY, 0)
906 c.Assert(err, check.IsNil)
908 n, err := f.Write([]byte{1, 2, 3})
909 c.Check(n, check.Equals, 0)
910 c.Check(err, check.ErrorMatches, `read-only file`)
911 n, err = f.Read(make([]byte, 1))
912 c.Check(n, check.Equals, 0)
913 c.Check(err, check.Equals, io.EOF)
914 f, err = fs.OpenFile("new", os.O_RDWR, 0)
915 c.Assert(err, check.IsNil)
917 _, err = f.Write([]byte{4, 5, 6})
918 c.Check(err, check.IsNil)
920 c.Assert(err, check.IsNil)
921 c.Check(fi.Size(), check.Equals, int64(3))
923 f, err = fs.OpenFile("new", os.O_TRUNC|os.O_RDWR, 0)
924 c.Assert(err, check.IsNil)
926 pos, err := f.Seek(0, io.SeekEnd)
927 c.Check(pos, check.Equals, int64(0))
928 c.Check(err, check.IsNil)
930 c.Assert(err, check.IsNil)
931 c.Check(fi.Size(), check.Equals, int64(0))
934 buf := make([]byte, 64)
935 f, err = fs.OpenFile("append", os.O_EXCL|os.O_CREATE|os.O_RDWR|os.O_APPEND, 0)
936 c.Assert(err, check.IsNil)
937 f.Write([]byte{1, 2, 3})
938 f.Seek(0, io.SeekStart)
939 n, _ = f.Read(buf[:1])
940 c.Check(n, check.Equals, 1)
941 c.Check(buf[:1], check.DeepEquals, []byte{1})
942 pos, err = f.Seek(0, io.SeekCurrent)
943 c.Assert(err, check.IsNil)
944 c.Check(pos, check.Equals, int64(1))
945 f.Write([]byte{4, 5, 6})
946 pos, err = f.Seek(0, io.SeekCurrent)
947 c.Assert(err, check.IsNil)
948 c.Check(pos, check.Equals, int64(6))
949 f.Seek(0, io.SeekStart)
951 c.Check(buf[:n], check.DeepEquals, []byte{1, 2, 3, 4, 5, 6})
952 c.Check(err, check.Equals, io.EOF)
955 f, err = fs.OpenFile("append", os.O_RDWR|os.O_APPEND, 0)
956 c.Assert(err, check.IsNil)
957 pos, err = f.Seek(0, io.SeekCurrent)
958 c.Check(pos, check.Equals, int64(0))
959 c.Check(err, check.IsNil)
961 pos, _ = f.Seek(0, io.SeekCurrent)
962 c.Check(pos, check.Equals, int64(3))
963 f.Write([]byte{7, 8, 9})
964 pos, err = f.Seek(0, io.SeekCurrent)
965 c.Check(err, check.IsNil)
966 c.Check(pos, check.Equals, int64(9))
969 f, err = fs.OpenFile("wronly", os.O_CREATE|os.O_WRONLY, 0)
970 c.Assert(err, check.IsNil)
971 n, err = f.Write([]byte{3, 2, 1})
972 c.Check(n, check.Equals, 3)
973 c.Check(err, check.IsNil)
974 pos, _ = f.Seek(0, io.SeekCurrent)
975 c.Check(pos, check.Equals, int64(3))
976 pos, _ = f.Seek(0, io.SeekStart)
977 c.Check(pos, check.Equals, int64(0))
979 c.Check(n, check.Equals, 0)
980 c.Check(err, check.ErrorMatches, `.*O_WRONLY.*`)
981 f, err = fs.OpenFile("wronly", os.O_RDONLY, 0)
982 c.Assert(err, check.IsNil)
984 c.Check(buf[:n], check.DeepEquals, []byte{3, 2, 1})
986 f, err = fs.OpenFile("unsupported", os.O_CREATE|os.O_SYNC, 0)
987 c.Check(f, check.IsNil)
988 c.Check(err, check.NotNil)
990 f, err = fs.OpenFile("append", os.O_RDWR|os.O_WRONLY, 0)
991 c.Check(f, check.IsNil)
992 c.Check(err, check.ErrorMatches, `invalid flag.*`)
995 func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
997 defer func() { maxBlockSize = 2 << 26 }()
999 fs, err := (&Collection{}).FileSystem(s.client, s.kc)
1000 c.Assert(err, check.IsNil)
1001 f, err := fs.OpenFile("50K", os.O_WRONLY|os.O_CREATE, 0)
1002 c.Assert(err, check.IsNil)
1005 data := make([]byte, 500)
1008 for i := 0; i < 100; i++ {
1009 n, err := f.Write(data)
1010 c.Assert(n, check.Equals, len(data))
1011 c.Assert(err, check.IsNil)
1014 currentMemExtents := func() (memExtents []int) {
1015 for idx, e := range f.(*filehandle).inode.(*filenode).segments {
1018 memExtents = append(memExtents, idx)
1023 c.Check(currentMemExtents(), check.HasLen, 1)
1025 m, err := fs.MarshalManifest(".")
1026 c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
1027 c.Check(err, check.IsNil)
1028 c.Check(currentMemExtents(), check.HasLen, 0)
1031 func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
1032 for _, txt := range []string{
1036 ". d41d8cd98f00b204e9800998ecf8427e+0\n",
1037 ". d41d8cd98f00b204e9800998ecf8427e+0 \n",
1040 ". 0:0:foo 0:0:bar\n",
1041 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo\n",
1042 ". d41d8cd98f00b204e9800998ecf8427e+0 :0:0:foo\n",
1043 ". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
1044 ". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
1045 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
1046 ". d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n",
1047 "./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
1050 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
1051 c.Check(fs, check.IsNil)
1052 c.Logf("-> %s", err)
1053 c.Check(err, check.NotNil)
1057 func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
1058 for _, txt := range []string{
1060 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
1061 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
1062 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
1063 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
1066 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
1067 c.Check(err, check.IsNil)
1068 c.Check(fs, check.NotNil)
1072 func (s *CollectionFSSuite) checkMemSize(c *check.C, f File) {
1073 fn := f.(*filehandle).inode.(*filenode)
1075 for _, seg := range fn.segments {
1076 if e, ok := seg.(*memSegment); ok {
1077 memsize += int64(len(e.buf))
1080 c.Check(fn.memsize, check.Equals, memsize)
1083 type CollectionFSUnitSuite struct{}
1085 var _ = check.Suite(&CollectionFSUnitSuite{})
1087 // expect ~2 seconds to load a manifest with 256K files
1088 func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
1089 if testing.Short() {
1098 mb := bytes.NewBuffer(make([]byte, 0, 40000000))
1099 for i := 0; i < dirCount; i++ {
1100 fmt.Fprintf(mb, "./dir%d", i)
1101 for j := 0; j <= fileCount; j++ {
1102 fmt.Fprintf(mb, " %032x+42+A%040x@%08x", j, j, j)
1104 for j := 0; j < fileCount; j++ {
1105 fmt.Fprintf(mb, " %d:%d:dir%d/file%d", j*42+21, 42, j, j)
1107 mb.Write([]byte{'\n'})
1109 coll := Collection{ManifestText: mb.String()}
1110 c.Logf("%s built", time.Now())
1112 var memstats runtime.MemStats
1113 runtime.ReadMemStats(&memstats)
1114 c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1116 f, err := coll.FileSystem(nil, nil)
1117 c.Check(err, check.IsNil)
1118 c.Logf("%s loaded", time.Now())
1119 c.Check(f.Size(), check.Equals, int64(42*dirCount*fileCount))
1121 for i := 0; i < dirCount; i++ {
1122 for j := 0; j < fileCount; j++ {
1123 f.Stat(fmt.Sprintf("./dir%d/dir%d/file%d", i, j, j))
1126 c.Logf("%s Stat() x %d", time.Now(), dirCount*fileCount)
1128 runtime.ReadMemStats(&memstats)
1129 c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1132 // Gocheck boilerplate
1133 func Test(t *testing.T) {