Merge branch '14345-webdav-lock-and-empty-dir'
[arvados.git] / sdk / go / arvados / fs_collection_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "bytes"
9         "crypto/md5"
10         "crypto/sha1"
11         "errors"
12         "fmt"
13         "io"
14         "io/ioutil"
15         "math/rand"
16         "net/http"
17         "os"
18         "regexp"
19         "runtime"
20         "strings"
21         "sync"
22         "testing"
23         "time"
24
25         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
26         check "gopkg.in/check.v1"
27 )
28
29 var _ = check.Suite(&CollectionFSSuite{})
30
31 type keepClientStub struct {
32         blocks      map[string][]byte
33         refreshable map[string]bool
34         sync.RWMutex
35 }
36
37 var errStub404 = errors.New("404 block not found")
38
39 func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error) {
40         kcs.RLock()
41         defer kcs.RUnlock()
42         buf := kcs.blocks[locator[:32]]
43         if buf == nil {
44                 return 0, errStub404
45         }
46         return copy(p, buf[off:]), nil
47 }
48
49 func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
50         locator := fmt.Sprintf("%x+%d+A12345@abcde", md5.Sum(p), len(p))
51         buf := make([]byte, len(p))
52         copy(buf, p)
53         kcs.Lock()
54         defer kcs.Unlock()
55         kcs.blocks[locator[:32]] = buf
56         return locator, 1, nil
57 }
58
59 var localOrRemoteSignature = regexp.MustCompile(`\+[AR][^+]*`)
60
61 func (kcs *keepClientStub) LocalLocator(locator string) (string, error) {
62         kcs.Lock()
63         defer kcs.Unlock()
64         if strings.Contains(locator, "+R") {
65                 if len(locator) < 32 {
66                         return "", fmt.Errorf("bad locator: %q", locator)
67                 }
68                 if _, ok := kcs.blocks[locator[:32]]; !ok && !kcs.refreshable[locator[:32]] {
69                         return "", fmt.Errorf("kcs.refreshable[%q]==false", locator)
70                 }
71         }
72         fakeSig := fmt.Sprintf("+A%x@%x", sha1.Sum(nil), time.Now().Add(time.Hour*24*14).Unix())
73         return localOrRemoteSignature.ReplaceAllLiteralString(locator, fakeSig), nil
74 }
75
76 type CollectionFSSuite struct {
77         client *Client
78         coll   Collection
79         fs     CollectionFileSystem
80         kc     *keepClientStub
81 }
82
83 func (s *CollectionFSSuite) SetUpTest(c *check.C) {
84         s.client = NewClientFromEnv()
85         err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
86         c.Assert(err, check.IsNil)
87         s.kc = &keepClientStub{
88                 blocks: map[string][]byte{
89                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
90                 }}
91         s.fs, err = s.coll.FileSystem(s.client, s.kc)
92         c.Assert(err, check.IsNil)
93 }
94
95 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
96         _, ok := s.fs.(http.FileSystem)
97         c.Check(ok, check.Equals, true)
98 }
99
100 func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
101         fs, err := (&Collection{
102                 ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
103         }).FileSystem(s.client, s.kc)
104         c.Assert(err, check.IsNil)
105
106         f, err := fs.Open("/foo:foo")
107         c.Assert(err, check.IsNil)
108
109         fis, err := f.Readdir(0)
110         c.Check(err, check.IsNil)
111         c.Check(len(fis), check.Equals, 1)
112         c.Check(fis[0].Name(), check.Equals, "bar:bar")
113 }
114
115 func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
116         f, err := s.fs.Open("/dir1")
117         c.Assert(err, check.IsNil)
118
119         st, err := f.Stat()
120         c.Assert(err, check.IsNil)
121         c.Check(st.Size(), check.Equals, int64(2))
122         c.Check(st.IsDir(), check.Equals, true)
123
124         fis, err := f.Readdir(0)
125         c.Check(err, check.IsNil)
126         c.Check(len(fis), check.Equals, 2)
127         if len(fis) > 0 {
128                 c.Check(fis[0].Size(), check.Equals, int64(3))
129         }
130 }
131
132 func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
133         f, err := s.fs.Open("./dir1")
134         c.Assert(err, check.IsNil)
135
136         fis, err := f.Readdir(1)
137         c.Check(err, check.IsNil)
138         c.Check(len(fis), check.Equals, 1)
139         if len(fis) > 0 {
140                 c.Check(fis[0].Size(), check.Equals, int64(3))
141         }
142
143         fis, err = f.Readdir(1)
144         c.Check(err, check.IsNil)
145         c.Check(len(fis), check.Equals, 1)
146         if len(fis) > 0 {
147                 c.Check(fis[0].Size(), check.Equals, int64(3))
148         }
149
150         fis, err = f.Readdir(1)
151         c.Check(len(fis), check.Equals, 0)
152         c.Check(err, check.NotNil)
153         c.Check(err, check.Equals, io.EOF)
154
155         f, err = s.fs.Open("dir1")
156         c.Assert(err, check.IsNil)
157         fis, err = f.Readdir(1)
158         c.Check(len(fis), check.Equals, 1)
159         c.Assert(err, check.IsNil)
160         fis, err = f.Readdir(2)
161         c.Check(len(fis), check.Equals, 1)
162         c.Assert(err, check.IsNil)
163         fis, err = f.Readdir(2)
164         c.Check(len(fis), check.Equals, 0)
165         c.Assert(err, check.Equals, io.EOF)
166 }
167
168 func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
169         for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
170                 f, err := s.fs.Open(path)
171                 c.Assert(err, check.IsNil)
172
173                 st, err := f.Stat()
174                 c.Assert(err, check.IsNil)
175                 c.Check(st.Size(), check.Equals, int64(1))
176                 c.Check(st.IsDir(), check.Equals, true)
177         }
178         for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
179                 c.Logf("%q", path)
180                 f, err := s.fs.Open(path)
181                 c.Assert(err, check.IsNil)
182
183                 st, err := f.Stat()
184                 c.Assert(err, check.IsNil)
185                 c.Check(st.Size(), check.Equals, int64(2))
186                 c.Check(st.IsDir(), check.Equals, true)
187         }
188 }
189
190 func (s *CollectionFSSuite) TestNotExist(c *check.C) {
191         for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
192                 f, err := s.fs.Open(path)
193                 c.Assert(f, check.IsNil)
194                 c.Assert(err, check.NotNil)
195                 c.Assert(os.IsNotExist(err), check.Equals, true)
196         }
197 }
198
199 func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
200         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDONLY, 0)
201         c.Assert(err, check.IsNil)
202         st, err := f.Stat()
203         c.Assert(err, check.IsNil)
204         c.Check(st.Size(), check.Equals, int64(3))
205         n, err := f.Write([]byte("bar"))
206         c.Check(n, check.Equals, 0)
207         c.Check(err, check.Equals, ErrReadOnlyFile)
208 }
209
210 func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
211         f, err := s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE, 0)
212         c.Assert(err, check.IsNil)
213         st, err := f.Stat()
214         c.Assert(err, check.IsNil)
215         c.Check(st.Size(), check.Equals, int64(0))
216
217         n, err := f.Write([]byte("bar"))
218         c.Check(n, check.Equals, 3)
219         c.Check(err, check.IsNil)
220
221         c.Check(f.Close(), check.IsNil)
222
223         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
224         c.Check(f, check.IsNil)
225         c.Assert(err, check.NotNil)
226
227         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR, 0)
228         c.Assert(err, check.IsNil)
229         st, err = f.Stat()
230         c.Assert(err, check.IsNil)
231         c.Check(st.Size(), check.Equals, int64(3))
232
233         c.Check(f.Close(), check.IsNil)
234
235         m, err := s.fs.MarshalManifest(".")
236         c.Assert(err, check.IsNil)
237         c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
238 }
239
240 func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
241         maxBlockSize = 8
242         defer func() { maxBlockSize = 2 << 26 }()
243
244         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
245         c.Assert(err, check.IsNil)
246         defer f.Close()
247         st, err := f.Stat()
248         c.Assert(err, check.IsNil)
249         c.Check(st.Size(), check.Equals, int64(3))
250
251         f2, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
252         c.Assert(err, check.IsNil)
253         defer f2.Close()
254
255         buf := make([]byte, 64)
256         n, err := f.Read(buf)
257         c.Check(n, check.Equals, 3)
258         c.Check(err, check.Equals, io.EOF)
259         c.Check(string(buf[:3]), check.DeepEquals, "foo")
260
261         pos, err := f.Seek(-2, io.SeekCurrent)
262         c.Check(pos, check.Equals, int64(1))
263         c.Check(err, check.IsNil)
264
265         // Split a storedExtent in two, and insert a memExtent
266         n, err = f.Write([]byte("*"))
267         c.Check(n, check.Equals, 1)
268         c.Check(err, check.IsNil)
269
270         pos, err = f.Seek(0, io.SeekCurrent)
271         c.Check(pos, check.Equals, int64(2))
272         c.Check(err, check.IsNil)
273
274         pos, err = f.Seek(0, io.SeekStart)
275         c.Check(pos, check.Equals, int64(0))
276         c.Check(err, check.IsNil)
277
278         rbuf, err := ioutil.ReadAll(f)
279         c.Check(len(rbuf), check.Equals, 3)
280         c.Check(err, check.IsNil)
281         c.Check(string(rbuf), check.Equals, "f*o")
282
283         // Write multiple blocks in one call
284         f.Seek(1, io.SeekStart)
285         n, err = f.Write([]byte("0123456789abcdefg"))
286         c.Check(n, check.Equals, 17)
287         c.Check(err, check.IsNil)
288         pos, err = f.Seek(0, io.SeekCurrent)
289         c.Check(pos, check.Equals, int64(18))
290         c.Check(err, check.IsNil)
291         pos, err = f.Seek(-18, io.SeekCurrent)
292         c.Check(pos, check.Equals, int64(0))
293         c.Check(err, check.IsNil)
294         n, err = io.ReadFull(f, buf)
295         c.Check(n, check.Equals, 18)
296         c.Check(err, check.Equals, io.ErrUnexpectedEOF)
297         c.Check(string(buf[:n]), check.Equals, "f0123456789abcdefg")
298
299         buf2, err := ioutil.ReadAll(f2)
300         c.Check(err, check.IsNil)
301         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
302
303         // truncate to current size
304         err = f.Truncate(18)
305         c.Check(err, check.IsNil)
306         f2.Seek(0, io.SeekStart)
307         buf2, err = ioutil.ReadAll(f2)
308         c.Check(err, check.IsNil)
309         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
310
311         // shrink to zero some data
312         f.Truncate(15)
313         f2.Seek(0, io.SeekStart)
314         buf2, err = ioutil.ReadAll(f2)
315         c.Check(err, check.IsNil)
316         c.Check(string(buf2), check.Equals, "f0123456789abcd")
317
318         // grow to partial block/extent
319         f.Truncate(20)
320         f2.Seek(0, io.SeekStart)
321         buf2, err = ioutil.ReadAll(f2)
322         c.Check(err, check.IsNil)
323         c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
324
325         f.Truncate(0)
326         f2.Seek(0, io.SeekStart)
327         f2.Write([]byte("12345678abcdefghijkl"))
328
329         // grow to block/extent boundary
330         f.Truncate(64)
331         f2.Seek(0, io.SeekStart)
332         buf2, err = ioutil.ReadAll(f2)
333         c.Check(err, check.IsNil)
334         c.Check(len(buf2), check.Equals, 64)
335         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 8)
336
337         // shrink to block/extent boundary
338         err = f.Truncate(32)
339         c.Check(err, check.IsNil)
340         f2.Seek(0, io.SeekStart)
341         buf2, err = ioutil.ReadAll(f2)
342         c.Check(err, check.IsNil)
343         c.Check(len(buf2), check.Equals, 32)
344         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 4)
345
346         // shrink to partial block/extent
347         err = f.Truncate(15)
348         c.Check(err, check.IsNil)
349         f2.Seek(0, io.SeekStart)
350         buf2, err = ioutil.ReadAll(f2)
351         c.Check(err, check.IsNil)
352         c.Check(string(buf2), check.Equals, "12345678abcdefg")
353         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 2)
354
355         // Force flush to ensure the block "12345678" gets stored, so
356         // we know what to expect in the final manifest below.
357         _, err = s.fs.MarshalManifest(".")
358         c.Check(err, check.IsNil)
359
360         // Truncate to size=3 while f2's ptr is at 15
361         err = f.Truncate(3)
362         c.Check(err, check.IsNil)
363         buf2, err = ioutil.ReadAll(f2)
364         c.Check(err, check.IsNil)
365         c.Check(string(buf2), check.Equals, "")
366         f2.Seek(0, io.SeekStart)
367         buf2, err = ioutil.ReadAll(f2)
368         c.Check(err, check.IsNil)
369         c.Check(string(buf2), check.Equals, "123")
370         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 1)
371
372         m, err := s.fs.MarshalManifest(".")
373         c.Check(err, check.IsNil)
374         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
375         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 25d55ad283aa400af464c76d713c07ad+8 3:3:bar 6:3:foo\n")
376         c.Check(s.fs.Size(), check.Equals, int64(6))
377 }
378
379 func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
380         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
381         c.Assert(err, check.IsNil)
382         f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
383         c.Assert(err, check.IsNil)
384         defer f.Close()
385
386         checkSize := func(size int64) {
387                 fi, err := f.Stat()
388                 c.Assert(err, check.IsNil)
389                 c.Check(fi.Size(), check.Equals, size)
390
391                 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
392                 c.Assert(err, check.IsNil)
393                 defer f.Close()
394                 fi, err = f.Stat()
395                 c.Check(err, check.IsNil)
396                 c.Check(fi.Size(), check.Equals, size)
397                 pos, err := f.Seek(0, io.SeekEnd)
398                 c.Check(err, check.IsNil)
399                 c.Check(pos, check.Equals, size)
400         }
401
402         f.Seek(2, io.SeekEnd)
403         checkSize(0)
404         f.Write([]byte{1})
405         checkSize(3)
406
407         f.Seek(2, io.SeekCurrent)
408         checkSize(3)
409         f.Write([]byte{})
410         checkSize(5)
411
412         f.Seek(8, io.SeekStart)
413         checkSize(5)
414         n, err := f.Read(make([]byte, 1))
415         c.Check(n, check.Equals, 0)
416         c.Check(err, check.Equals, io.EOF)
417         checkSize(5)
418         f.Write([]byte{1, 2, 3})
419         checkSize(11)
420 }
421
422 func (s *CollectionFSSuite) TestMarshalCopiesRemoteBlocks(c *check.C) {
423         foo := "foo"
424         bar := "bar"
425         hash := map[string]string{
426                 foo: fmt.Sprintf("%x", md5.Sum([]byte(foo))),
427                 bar: fmt.Sprintf("%x", md5.Sum([]byte(bar))),
428         }
429
430         fs, err := (&Collection{
431                 ManifestText: ". " + hash[foo] + "+3+Rzaaaa-foo@bab " + hash[bar] + "+3+A12345@ffffff 0:2:fo.txt 2:4:obar.txt\n",
432         }).FileSystem(s.client, s.kc)
433         c.Assert(err, check.IsNil)
434         manifest, err := fs.MarshalManifest(".")
435         c.Check(manifest, check.Equals, "")
436         c.Check(err, check.NotNil)
437
438         s.kc.refreshable = map[string]bool{hash[bar]: true}
439
440         for _, sigIn := range []string{"Rzaaaa-foo@bab", "A12345@abcde"} {
441                 fs, err = (&Collection{
442                         ManifestText: ". " + hash[foo] + "+3+A12345@fffff " + hash[bar] + "+3+" + sigIn + " 0:2:fo.txt 2:4:obar.txt\n",
443                 }).FileSystem(s.client, s.kc)
444                 c.Assert(err, check.IsNil)
445                 manifest, err := fs.MarshalManifest(".")
446                 c.Check(err, check.IsNil)
447                 // Both blocks should now have +A signatures.
448                 c.Check(manifest, check.Matches, `.*\+A.* .*\+A.*\n`)
449                 c.Check(manifest, check.Not(check.Matches), `.*\+R.*\n`)
450         }
451 }
452
453 func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
454         maxBlockSize = 8
455         defer func() { maxBlockSize = 2 << 26 }()
456
457         var err error
458         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
459         c.Assert(err, check.IsNil)
460         for _, name := range []string{"foo", "bar", "baz"} {
461                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
462                 c.Assert(err, check.IsNil)
463                 f.Write([]byte(name))
464                 f.Close()
465         }
466
467         m, err := s.fs.MarshalManifest(".")
468         c.Check(err, check.IsNil)
469         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
470         c.Check(m, check.Equals, ". c3c23db5285662ef7172373df0003206+6 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar 3:3:baz 6:3:foo\n")
471 }
472
473 func (s *CollectionFSSuite) TestMkdir(c *check.C) {
474         err := s.fs.Mkdir("foo/bar", 0755)
475         c.Check(err, check.Equals, os.ErrNotExist)
476
477         f, err := s.fs.OpenFile("foo/bar", os.O_CREATE, 0)
478         c.Check(err, check.Equals, os.ErrNotExist)
479
480         err = s.fs.Mkdir("foo", 0755)
481         c.Check(err, check.IsNil)
482
483         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_WRONLY, 0)
484         c.Check(err, check.IsNil)
485         if err == nil {
486                 defer f.Close()
487                 f.Write([]byte("foo"))
488         }
489
490         // mkdir fails if a file already exists with that name
491         err = s.fs.Mkdir("foo/bar", 0755)
492         c.Check(err, check.NotNil)
493
494         err = s.fs.Remove("foo/bar")
495         c.Check(err, check.IsNil)
496
497         // mkdir succeeds after the file is deleted
498         err = s.fs.Mkdir("foo/bar", 0755)
499         c.Check(err, check.IsNil)
500
501         // creating a file in a nonexistent subdir should still fail
502         f, err = s.fs.OpenFile("foo/bar/baz/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
503         c.Check(err, check.Equals, os.ErrNotExist)
504
505         f, err = s.fs.OpenFile("foo/bar/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
506         c.Check(err, check.IsNil)
507         if err == nil {
508                 defer f.Close()
509                 f.Write([]byte("foo"))
510         }
511
512         // creating foo/bar as a regular file should fail
513         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, 0)
514         c.Check(err, check.NotNil)
515
516         // creating foo/bar as a directory should fail
517         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, os.ModeDir)
518         c.Check(err, check.NotNil)
519         err = s.fs.Mkdir("foo/bar", 0755)
520         c.Check(err, check.NotNil)
521
522         m, err := s.fs.MarshalManifest(".")
523         c.Check(err, check.IsNil)
524         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
525         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 3:3:bar 0:3:foo\n./foo/bar acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
526 }
527
528 func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
529         if testing.Short() {
530                 c.Skip("slow")
531         }
532
533         maxBlockSize = 8
534         defer func() { maxBlockSize = 2 << 26 }()
535
536         var wg sync.WaitGroup
537         for n := 0; n < 128; n++ {
538                 wg.Add(1)
539                 go func() {
540                         defer wg.Done()
541                         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
542                         c.Assert(err, check.IsNil)
543                         defer f.Close()
544                         for i := 0; i < 1024; i++ {
545                                 r := rand.Uint32()
546                                 switch {
547                                 case r%11 == 0:
548                                         _, err := s.fs.MarshalManifest(".")
549                                         c.Check(err, check.IsNil)
550                                 case r&3 == 0:
551                                         f.Truncate(int64(rand.Intn(64)))
552                                 case r&3 == 1:
553                                         f.Seek(int64(rand.Intn(64)), io.SeekStart)
554                                 case r&3 == 2:
555                                         _, err := f.Write([]byte("beep boop"))
556                                         c.Check(err, check.IsNil)
557                                 case r&3 == 3:
558                                         _, err := ioutil.ReadAll(f)
559                                         c.Check(err, check.IsNil)
560                                 }
561                         }
562                 }()
563         }
564         wg.Wait()
565
566         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
567         c.Assert(err, check.IsNil)
568         defer f.Close()
569         buf, err := ioutil.ReadAll(f)
570         c.Check(err, check.IsNil)
571         c.Logf("after lots of random r/w/seek/trunc, buf is %q", buf)
572 }
573
574 func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
575         maxBlockSize = 40
576         defer func() { maxBlockSize = 2 << 26 }()
577
578         var err error
579         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
580         c.Assert(err, check.IsNil)
581
582         const nfiles = 256
583         const ngoroutines = 256
584
585         var wg sync.WaitGroup
586         for n := 0; n < nfiles; n++ {
587                 wg.Add(1)
588                 go func(n int) {
589                         defer wg.Done()
590                         expect := make([]byte, 0, 64)
591                         wbytes := []byte("there's no simple explanation for anything important that any of us do")
592                         f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
593                         c.Assert(err, check.IsNil)
594                         defer f.Close()
595                         for i := 0; i < ngoroutines; i++ {
596                                 trunc := rand.Intn(65)
597                                 woff := rand.Intn(trunc + 1)
598                                 wbytes = wbytes[:rand.Intn(64-woff+1)]
599                                 for buf, i := expect[:cap(expect)], len(expect); i < trunc; i++ {
600                                         buf[i] = 0
601                                 }
602                                 expect = expect[:trunc]
603                                 if trunc < woff+len(wbytes) {
604                                         expect = expect[:woff+len(wbytes)]
605                                 }
606                                 copy(expect[woff:], wbytes)
607                                 f.Truncate(int64(trunc))
608                                 pos, err := f.Seek(int64(woff), io.SeekStart)
609                                 c.Check(pos, check.Equals, int64(woff))
610                                 c.Check(err, check.IsNil)
611                                 n, err := f.Write(wbytes)
612                                 c.Check(n, check.Equals, len(wbytes))
613                                 c.Check(err, check.IsNil)
614                                 pos, err = f.Seek(0, io.SeekStart)
615                                 c.Check(pos, check.Equals, int64(0))
616                                 c.Check(err, check.IsNil)
617                                 buf, err := ioutil.ReadAll(f)
618                                 c.Check(string(buf), check.Equals, string(expect))
619                                 c.Check(err, check.IsNil)
620                         }
621                         s.checkMemSize(c, f)
622                 }(n)
623         }
624         wg.Wait()
625
626         root, err := s.fs.Open("/")
627         c.Assert(err, check.IsNil)
628         defer root.Close()
629         fi, err := root.Readdir(-1)
630         c.Check(err, check.IsNil)
631         c.Check(len(fi), check.Equals, nfiles)
632
633         _, err = s.fs.MarshalManifest(".")
634         c.Check(err, check.IsNil)
635         // TODO: check manifest content
636 }
637
638 func (s *CollectionFSSuite) TestRemove(c *check.C) {
639         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
640         c.Assert(err, check.IsNil)
641         err = fs.Mkdir("dir0", 0755)
642         c.Assert(err, check.IsNil)
643         err = fs.Mkdir("dir1", 0755)
644         c.Assert(err, check.IsNil)
645         err = fs.Mkdir("dir1/dir2", 0755)
646         c.Assert(err, check.IsNil)
647         err = fs.Mkdir("dir1/dir3", 0755)
648         c.Assert(err, check.IsNil)
649
650         err = fs.Remove("dir0")
651         c.Check(err, check.IsNil)
652         err = fs.Remove("dir0")
653         c.Check(err, check.Equals, os.ErrNotExist)
654
655         err = fs.Remove("dir1/dir2/.")
656         c.Check(err, check.Equals, ErrInvalidArgument)
657         err = fs.Remove("dir1/dir2/..")
658         c.Check(err, check.Equals, ErrInvalidArgument)
659         err = fs.Remove("dir1")
660         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
661         err = fs.Remove("dir1/dir2/../../../dir1")
662         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
663         err = fs.Remove("dir1/dir3/")
664         c.Check(err, check.IsNil)
665         err = fs.RemoveAll("dir1")
666         c.Check(err, check.IsNil)
667         err = fs.RemoveAll("dir1")
668         c.Check(err, check.IsNil)
669 }
670
671 func (s *CollectionFSSuite) TestRenameError(c *check.C) {
672         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
673         c.Assert(err, check.IsNil)
674         err = fs.Mkdir("first", 0755)
675         c.Assert(err, check.IsNil)
676         err = fs.Mkdir("first/second", 0755)
677         c.Assert(err, check.IsNil)
678         f, err := fs.OpenFile("first/second/file", os.O_CREATE|os.O_WRONLY, 0755)
679         c.Assert(err, check.IsNil)
680         f.Write([]byte{1, 2, 3, 4, 5})
681         f.Close()
682         err = fs.Rename("first", "first/second/third")
683         c.Check(err, check.Equals, ErrInvalidArgument)
684         err = fs.Rename("first", "first/third")
685         c.Check(err, check.Equals, ErrInvalidArgument)
686         err = fs.Rename("first/second", "second")
687         c.Check(err, check.IsNil)
688         f, err = fs.OpenFile("second/file", 0, 0)
689         c.Assert(err, check.IsNil)
690         data, err := ioutil.ReadAll(f)
691         c.Check(err, check.IsNil)
692         c.Check(data, check.DeepEquals, []byte{1, 2, 3, 4, 5})
693 }
694
695 func (s *CollectionFSSuite) TestRenameDirectory(c *check.C) {
696         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
697         c.Assert(err, check.IsNil)
698         err = fs.Mkdir("foo", 0755)
699         c.Assert(err, check.IsNil)
700         err = fs.Mkdir("bar", 0755)
701         c.Assert(err, check.IsNil)
702         err = fs.Rename("bar", "baz")
703         c.Check(err, check.IsNil)
704         err = fs.Rename("foo", "baz")
705         c.Check(err, check.NotNil)
706         err = fs.Rename("foo", "baz/")
707         c.Check(err, check.IsNil)
708         err = fs.Rename("baz/foo", ".")
709         c.Check(err, check.Equals, ErrInvalidArgument)
710         err = fs.Rename("baz/foo/", ".")
711         c.Check(err, check.Equals, ErrInvalidArgument)
712 }
713
714 func (s *CollectionFSSuite) TestRename(c *check.C) {
715         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
716         c.Assert(err, check.IsNil)
717         const (
718                 outer = 16
719                 inner = 16
720         )
721         for i := 0; i < outer; i++ {
722                 err = fs.Mkdir(fmt.Sprintf("dir%d", i), 0755)
723                 c.Assert(err, check.IsNil)
724                 for j := 0; j < inner; j++ {
725                         err = fs.Mkdir(fmt.Sprintf("dir%d/dir%d", i, j), 0755)
726                         c.Assert(err, check.IsNil)
727                         for _, fnm := range []string{
728                                 fmt.Sprintf("dir%d/file%d", i, j),
729                                 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
730                         } {
731                                 f, err := fs.OpenFile(fnm, os.O_CREATE|os.O_WRONLY, 0755)
732                                 c.Assert(err, check.IsNil)
733                                 _, err = f.Write([]byte("beep"))
734                                 c.Assert(err, check.IsNil)
735                                 f.Close()
736                         }
737                 }
738         }
739         var wg sync.WaitGroup
740         for i := 0; i < outer; i++ {
741                 for j := 0; j < inner; j++ {
742                         wg.Add(1)
743                         go func(i, j int) {
744                                 defer wg.Done()
745                                 oldname := fmt.Sprintf("dir%d/dir%d/file%d", i, j, j)
746                                 newname := fmt.Sprintf("dir%d/newfile%d", i, inner-j-1)
747                                 _, err := fs.Open(newname)
748                                 c.Check(err, check.Equals, os.ErrNotExist)
749                                 err = fs.Rename(oldname, newname)
750                                 c.Check(err, check.IsNil)
751                                 f, err := fs.Open(newname)
752                                 c.Check(err, check.IsNil)
753                                 f.Close()
754                         }(i, j)
755
756                         wg.Add(1)
757                         go func(i, j int) {
758                                 defer wg.Done()
759                                 // oldname does not exist
760                                 err := fs.Rename(
761                                         fmt.Sprintf("dir%d/dir%d/missing", i, j),
762                                         fmt.Sprintf("dir%d/dir%d/file%d", outer-i-1, j, j))
763                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
764
765                                 // newname parent dir does not exist
766                                 err = fs.Rename(
767                                         fmt.Sprintf("dir%d/dir%d", i, j),
768                                         fmt.Sprintf("dir%d/missing/irrelevant", outer-i-1))
769                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
770
771                                 // oldname parent dir is a file
772                                 err = fs.Rename(
773                                         fmt.Sprintf("dir%d/file%d/patherror", i, j),
774                                         fmt.Sprintf("dir%d/irrelevant", i))
775                                 c.Check(err, check.ErrorMatches, `.*not a directory`)
776
777                                 // newname parent dir is a file
778                                 err = fs.Rename(
779                                         fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
780                                         fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
781                                 c.Check(err, check.ErrorMatches, `.*not a directory`)
782                         }(i, j)
783                 }
784         }
785         wg.Wait()
786
787         f, err := fs.OpenFile("dir1/newfile3", 0, 0)
788         c.Assert(err, check.IsNil)
789         c.Check(f.Size(), check.Equals, int64(4))
790         buf, err := ioutil.ReadAll(f)
791         c.Check(buf, check.DeepEquals, []byte("beep"))
792         c.Check(err, check.IsNil)
793         _, err = fs.Open("dir1/dir1/file1")
794         c.Check(err, check.Equals, os.ErrNotExist)
795 }
796
797 func (s *CollectionFSSuite) TestPersist(c *check.C) {
798         maxBlockSize = 1024
799         defer func() { maxBlockSize = 2 << 26 }()
800
801         var err error
802         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
803         c.Assert(err, check.IsNil)
804         err = s.fs.Mkdir("d:r", 0755)
805         c.Assert(err, check.IsNil)
806
807         expect := map[string][]byte{}
808
809         var wg sync.WaitGroup
810         for _, name := range []string{"random 1", "random:2", "random\\3", "d:r/random4"} {
811                 buf := make([]byte, 500)
812                 rand.Read(buf)
813                 expect[name] = buf
814
815                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
816                 c.Assert(err, check.IsNil)
817                 // Note: we don't close the file until after the test
818                 // is done. Writes to unclosed files should persist.
819                 defer f.Close()
820
821                 wg.Add(1)
822                 go func() {
823                         defer wg.Done()
824                         for i := 0; i < len(buf); i += 5 {
825                                 _, err := f.Write(buf[i : i+5])
826                                 c.Assert(err, check.IsNil)
827                         }
828                 }()
829         }
830         wg.Wait()
831
832         m, err := s.fs.MarshalManifest(".")
833         c.Check(err, check.IsNil)
834         c.Logf("%q", m)
835
836         root, err := s.fs.Open("/")
837         c.Assert(err, check.IsNil)
838         defer root.Close()
839         fi, err := root.Readdir(-1)
840         c.Check(err, check.IsNil)
841         c.Check(len(fi), check.Equals, 4)
842
843         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
844         c.Assert(err, check.IsNil)
845
846         root, err = persisted.Open("/")
847         c.Assert(err, check.IsNil)
848         defer root.Close()
849         fi, err = root.Readdir(-1)
850         c.Check(err, check.IsNil)
851         c.Check(len(fi), check.Equals, 4)
852
853         for name, content := range expect {
854                 c.Logf("read %q", name)
855                 f, err := persisted.Open(name)
856                 c.Assert(err, check.IsNil)
857                 defer f.Close()
858                 buf, err := ioutil.ReadAll(f)
859                 c.Check(err, check.IsNil)
860                 c.Check(buf, check.DeepEquals, content)
861         }
862 }
863
864 func (s *CollectionFSSuite) TestPersistEmptyFilesAndDirs(c *check.C) {
865         var err error
866         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
867         c.Assert(err, check.IsNil)
868         for _, name := range []string{"dir", "dir/zerodir", "empty", "not empty", "not empty/empty", "zero", "zero/zero"} {
869                 err = s.fs.Mkdir(name, 0755)
870                 c.Assert(err, check.IsNil)
871         }
872
873         expect := map[string][]byte{
874                 "0":                nil,
875                 "00":               {},
876                 "one":              {1},
877                 "dir/0":            nil,
878                 "dir/two":          {1, 2},
879                 "dir/zero":         nil,
880                 "dir/zerodir/zero": nil,
881                 "zero/zero/zero":   nil,
882         }
883         for name, data := range expect {
884                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
885                 c.Assert(err, check.IsNil)
886                 if data != nil {
887                         _, err := f.Write(data)
888                         c.Assert(err, check.IsNil)
889                 }
890                 f.Close()
891         }
892
893         m, err := s.fs.MarshalManifest(".")
894         c.Check(err, check.IsNil)
895         c.Logf("%q", m)
896
897         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
898         c.Assert(err, check.IsNil)
899
900         for name, data := range expect {
901                 _, err = persisted.Open("bogus-" + name)
902                 c.Check(err, check.NotNil)
903
904                 f, err := persisted.Open(name)
905                 c.Assert(err, check.IsNil)
906
907                 if data == nil {
908                         data = []byte{}
909                 }
910                 buf, err := ioutil.ReadAll(f)
911                 c.Check(err, check.IsNil)
912                 c.Check(buf, check.DeepEquals, data)
913         }
914
915         expectDir := map[string]int{
916                 "empty":           0,
917                 "not empty":       1,
918                 "not empty/empty": 0,
919         }
920         for name, expectLen := range expectDir {
921                 _, err := persisted.Open(name + "/bogus")
922                 c.Check(err, check.NotNil)
923
924                 d, err := persisted.Open(name)
925                 defer d.Close()
926                 c.Check(err, check.IsNil)
927                 fi, err := d.Readdir(-1)
928                 c.Check(err, check.IsNil)
929                 c.Check(fi, check.HasLen, expectLen)
930         }
931 }
932
933 func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
934         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
935         c.Assert(err, check.IsNil)
936
937         f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
938         c.Check(f, check.IsNil)
939         c.Check(err, check.ErrorMatches, `file does not exist`)
940
941         f, err = fs.OpenFile("new", os.O_CREATE|os.O_RDONLY, 0)
942         c.Assert(err, check.IsNil)
943         defer f.Close()
944         n, err := f.Write([]byte{1, 2, 3})
945         c.Check(n, check.Equals, 0)
946         c.Check(err, check.ErrorMatches, `read-only file`)
947         n, err = f.Read(make([]byte, 1))
948         c.Check(n, check.Equals, 0)
949         c.Check(err, check.Equals, io.EOF)
950         f, err = fs.OpenFile("new", os.O_RDWR, 0)
951         c.Assert(err, check.IsNil)
952         defer f.Close()
953         _, err = f.Write([]byte{4, 5, 6})
954         c.Check(err, check.IsNil)
955         fi, err := f.Stat()
956         c.Assert(err, check.IsNil)
957         c.Check(fi.Size(), check.Equals, int64(3))
958
959         f, err = fs.OpenFile("new", os.O_TRUNC|os.O_RDWR, 0)
960         c.Assert(err, check.IsNil)
961         defer f.Close()
962         pos, err := f.Seek(0, io.SeekEnd)
963         c.Check(pos, check.Equals, int64(0))
964         c.Check(err, check.IsNil)
965         fi, err = f.Stat()
966         c.Assert(err, check.IsNil)
967         c.Check(fi.Size(), check.Equals, int64(0))
968         fs.Remove("new")
969
970         buf := make([]byte, 64)
971         f, err = fs.OpenFile("append", os.O_EXCL|os.O_CREATE|os.O_RDWR|os.O_APPEND, 0)
972         c.Assert(err, check.IsNil)
973         f.Write([]byte{1, 2, 3})
974         f.Seek(0, io.SeekStart)
975         n, _ = f.Read(buf[:1])
976         c.Check(n, check.Equals, 1)
977         c.Check(buf[:1], check.DeepEquals, []byte{1})
978         pos, err = f.Seek(0, io.SeekCurrent)
979         c.Assert(err, check.IsNil)
980         c.Check(pos, check.Equals, int64(1))
981         f.Write([]byte{4, 5, 6})
982         pos, err = f.Seek(0, io.SeekCurrent)
983         c.Assert(err, check.IsNil)
984         c.Check(pos, check.Equals, int64(6))
985         f.Seek(0, io.SeekStart)
986         n, err = f.Read(buf)
987         c.Check(buf[:n], check.DeepEquals, []byte{1, 2, 3, 4, 5, 6})
988         c.Check(err, check.Equals, io.EOF)
989         f.Close()
990
991         f, err = fs.OpenFile("append", os.O_RDWR|os.O_APPEND, 0)
992         c.Assert(err, check.IsNil)
993         pos, err = f.Seek(0, io.SeekCurrent)
994         c.Check(pos, check.Equals, int64(0))
995         c.Check(err, check.IsNil)
996         f.Read(buf[:3])
997         pos, _ = f.Seek(0, io.SeekCurrent)
998         c.Check(pos, check.Equals, int64(3))
999         f.Write([]byte{7, 8, 9})
1000         pos, err = f.Seek(0, io.SeekCurrent)
1001         c.Check(err, check.IsNil)
1002         c.Check(pos, check.Equals, int64(9))
1003         f.Close()
1004
1005         f, err = fs.OpenFile("wronly", os.O_CREATE|os.O_WRONLY, 0)
1006         c.Assert(err, check.IsNil)
1007         n, err = f.Write([]byte{3, 2, 1})
1008         c.Check(n, check.Equals, 3)
1009         c.Check(err, check.IsNil)
1010         pos, _ = f.Seek(0, io.SeekCurrent)
1011         c.Check(pos, check.Equals, int64(3))
1012         pos, _ = f.Seek(0, io.SeekStart)
1013         c.Check(pos, check.Equals, int64(0))
1014         n, err = f.Read(buf)
1015         c.Check(n, check.Equals, 0)
1016         c.Check(err, check.ErrorMatches, `.*O_WRONLY.*`)
1017         f, err = fs.OpenFile("wronly", os.O_RDONLY, 0)
1018         c.Assert(err, check.IsNil)
1019         n, _ = f.Read(buf)
1020         c.Check(buf[:n], check.DeepEquals, []byte{3, 2, 1})
1021
1022         f, err = fs.OpenFile("unsupported", os.O_CREATE|os.O_SYNC, 0)
1023         c.Check(f, check.IsNil)
1024         c.Check(err, check.NotNil)
1025
1026         f, err = fs.OpenFile("append", os.O_RDWR|os.O_WRONLY, 0)
1027         c.Check(f, check.IsNil)
1028         c.Check(err, check.ErrorMatches, `invalid flag.*`)
1029 }
1030
1031 func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
1032         maxBlockSize = 1024
1033         defer func() { maxBlockSize = 2 << 26 }()
1034
1035         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
1036         c.Assert(err, check.IsNil)
1037         f, err := fs.OpenFile("50K", os.O_WRONLY|os.O_CREATE, 0)
1038         c.Assert(err, check.IsNil)
1039         defer f.Close()
1040
1041         data := make([]byte, 500)
1042         rand.Read(data)
1043
1044         for i := 0; i < 100; i++ {
1045                 n, err := f.Write(data)
1046                 c.Assert(n, check.Equals, len(data))
1047                 c.Assert(err, check.IsNil)
1048         }
1049
1050         currentMemExtents := func() (memExtents []int) {
1051                 for idx, e := range f.(*filehandle).inode.(*filenode).segments {
1052                         switch e.(type) {
1053                         case *memSegment:
1054                                 memExtents = append(memExtents, idx)
1055                         }
1056                 }
1057                 return
1058         }
1059         c.Check(currentMemExtents(), check.HasLen, 1)
1060
1061         m, err := fs.MarshalManifest(".")
1062         c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
1063         c.Check(err, check.IsNil)
1064         c.Check(currentMemExtents(), check.HasLen, 0)
1065 }
1066
1067 func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
1068         for _, txt := range []string{
1069                 "\n",
1070                 ".\n",
1071                 ". \n",
1072                 ". d41d8cd98f00b204e9800998ecf8427e+0\n",
1073                 ". d41d8cd98f00b204e9800998ecf8427e+0 \n",
1074                 ". 0:0:foo\n",
1075                 ".  0:0:foo\n",
1076                 ". 0:0:foo 0:0:bar\n",
1077                 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo\n",
1078                 ". d41d8cd98f00b204e9800998ecf8427e+0 :0:0:foo\n",
1079                 ". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
1080                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
1081                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
1082                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:\\056\n",
1083                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:\\056\\057\\056\n",
1084                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:.\n",
1085                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:..\n",
1086                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:..\n",
1087                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/..\n",
1088                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n",
1089                 "./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
1090         } {
1091                 c.Logf("<-%q", txt)
1092                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
1093                 c.Check(fs, check.IsNil)
1094                 c.Logf("-> %s", err)
1095                 c.Check(err, check.NotNil)
1096         }
1097 }
1098
1099 func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
1100         for _, txt := range []string{
1101                 "",
1102                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
1103                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:...\n",
1104                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:. 0:0:. 0:0:\\056 0:0:\\056\n",
1105                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/. 0:0:. 0:0:foo\\057bar\\057\\056\n",
1106                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
1107                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
1108         } {
1109                 c.Logf("<-%q", txt)
1110                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
1111                 c.Check(err, check.IsNil)
1112                 c.Check(fs, check.NotNil)
1113         }
1114 }
1115
1116 func (s *CollectionFSSuite) checkMemSize(c *check.C, f File) {
1117         fn := f.(*filehandle).inode.(*filenode)
1118         var memsize int64
1119         for _, seg := range fn.segments {
1120                 if e, ok := seg.(*memSegment); ok {
1121                         memsize += int64(len(e.buf))
1122                 }
1123         }
1124         c.Check(fn.memsize, check.Equals, memsize)
1125 }
1126
1127 type CollectionFSUnitSuite struct{}
1128
1129 var _ = check.Suite(&CollectionFSUnitSuite{})
1130
1131 // expect ~2 seconds to load a manifest with 256K files
1132 func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
1133         if testing.Short() {
1134                 c.Skip("slow")
1135         }
1136
1137         const (
1138                 dirCount  = 512
1139                 fileCount = 512
1140         )
1141
1142         mb := bytes.NewBuffer(make([]byte, 0, 40000000))
1143         for i := 0; i < dirCount; i++ {
1144                 fmt.Fprintf(mb, "./dir%d", i)
1145                 for j := 0; j <= fileCount; j++ {
1146                         fmt.Fprintf(mb, " %032x+42+A%040x@%08x", j, j, j)
1147                 }
1148                 for j := 0; j < fileCount; j++ {
1149                         fmt.Fprintf(mb, " %d:%d:dir%d/file%d", j*42+21, 42, j, j)
1150                 }
1151                 mb.Write([]byte{'\n'})
1152         }
1153         coll := Collection{ManifestText: mb.String()}
1154         c.Logf("%s built", time.Now())
1155
1156         var memstats runtime.MemStats
1157         runtime.ReadMemStats(&memstats)
1158         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1159
1160         f, err := coll.FileSystem(nil, nil)
1161         c.Check(err, check.IsNil)
1162         c.Logf("%s loaded", time.Now())
1163         c.Check(f.Size(), check.Equals, int64(42*dirCount*fileCount))
1164
1165         for i := 0; i < dirCount; i++ {
1166                 for j := 0; j < fileCount; j++ {
1167                         f.Stat(fmt.Sprintf("./dir%d/dir%d/file%d", i, j, j))
1168                 }
1169         }
1170         c.Logf("%s Stat() x %d", time.Now(), dirCount*fileCount)
1171
1172         runtime.ReadMemStats(&memstats)
1173         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1174 }
1175
1176 // Gocheck boilerplate
1177 func Test(t *testing.T) {
1178         check.TestingT(t)
1179 }