Merge branch '12913-secondary-mounts' refs #12913
[arvados.git] / sdk / go / arvados / collection_fs_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "bytes"
9         "crypto/md5"
10         "errors"
11         "fmt"
12         "io"
13         "io/ioutil"
14         "math/rand"
15         "net/http"
16         "os"
17         "regexp"
18         "runtime"
19         "sync"
20         "testing"
21         "time"
22
23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
24         check "gopkg.in/check.v1"
25 )
26
27 var _ = check.Suite(&CollectionFSSuite{})
28
29 type keepClientStub struct {
30         blocks map[string][]byte
31         sync.RWMutex
32 }
33
34 var errStub404 = errors.New("404 block not found")
35
36 func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error) {
37         kcs.RLock()
38         defer kcs.RUnlock()
39         buf := kcs.blocks[locator[:32]]
40         if buf == nil {
41                 return 0, errStub404
42         }
43         return copy(p, buf[off:]), nil
44 }
45
46 func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
47         locator := fmt.Sprintf("%x+%d+A12345@abcde", md5.Sum(p), len(p))
48         buf := make([]byte, len(p))
49         copy(buf, p)
50         kcs.Lock()
51         defer kcs.Unlock()
52         kcs.blocks[locator[:32]] = buf
53         return locator, 1, nil
54 }
55
56 type CollectionFSSuite struct {
57         client *Client
58         coll   Collection
59         fs     CollectionFileSystem
60         kc     keepClient
61 }
62
63 func (s *CollectionFSSuite) SetUpTest(c *check.C) {
64         s.client = NewClientFromEnv()
65         err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
66         c.Assert(err, check.IsNil)
67         s.kc = &keepClientStub{
68                 blocks: map[string][]byte{
69                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
70                 }}
71         s.fs, err = s.coll.FileSystem(s.client, s.kc)
72         c.Assert(err, check.IsNil)
73 }
74
75 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
76         _, ok := s.fs.(http.FileSystem)
77         c.Check(ok, check.Equals, true)
78 }
79
80 func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
81         fs, err := (&Collection{
82                 ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
83         }).FileSystem(s.client, s.kc)
84         c.Assert(err, check.IsNil)
85
86         f, err := fs.Open("/foo:foo")
87         c.Assert(err, check.IsNil)
88
89         fis, err := f.Readdir(0)
90         c.Check(err, check.IsNil)
91         c.Check(len(fis), check.Equals, 1)
92         c.Check(fis[0].Name(), check.Equals, "bar:bar")
93 }
94
95 func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
96         f, err := s.fs.Open("/dir1")
97         c.Assert(err, check.IsNil)
98
99         st, err := f.Stat()
100         c.Assert(err, check.IsNil)
101         c.Check(st.Size(), check.Equals, int64(2))
102         c.Check(st.IsDir(), check.Equals, true)
103
104         fis, err := f.Readdir(0)
105         c.Check(err, check.IsNil)
106         c.Check(len(fis), check.Equals, 2)
107         if len(fis) > 0 {
108                 c.Check(fis[0].Size(), check.Equals, int64(3))
109         }
110 }
111
112 func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
113         f, err := s.fs.Open("./dir1")
114         c.Assert(err, check.IsNil)
115
116         fis, err := f.Readdir(1)
117         c.Check(err, check.IsNil)
118         c.Check(len(fis), check.Equals, 1)
119         if len(fis) > 0 {
120                 c.Check(fis[0].Size(), check.Equals, int64(3))
121         }
122
123         fis, err = f.Readdir(1)
124         c.Check(err, check.IsNil)
125         c.Check(len(fis), check.Equals, 1)
126         if len(fis) > 0 {
127                 c.Check(fis[0].Size(), check.Equals, int64(3))
128         }
129
130         fis, err = f.Readdir(1)
131         c.Check(len(fis), check.Equals, 0)
132         c.Check(err, check.NotNil)
133         c.Check(err, check.Equals, io.EOF)
134
135         f, err = s.fs.Open("dir1")
136         c.Assert(err, check.IsNil)
137         fis, err = f.Readdir(1)
138         c.Check(len(fis), check.Equals, 1)
139         c.Assert(err, check.IsNil)
140         fis, err = f.Readdir(2)
141         c.Check(len(fis), check.Equals, 1)
142         c.Assert(err, check.IsNil)
143         fis, err = f.Readdir(2)
144         c.Check(len(fis), check.Equals, 0)
145         c.Assert(err, check.Equals, io.EOF)
146 }
147
148 func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
149         for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
150                 f, err := s.fs.Open(path)
151                 c.Assert(err, check.IsNil)
152
153                 st, err := f.Stat()
154                 c.Assert(err, check.IsNil)
155                 c.Check(st.Size(), check.Equals, int64(1))
156                 c.Check(st.IsDir(), check.Equals, true)
157         }
158         for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
159                 c.Logf("%q", path)
160                 f, err := s.fs.Open(path)
161                 c.Assert(err, check.IsNil)
162
163                 st, err := f.Stat()
164                 c.Assert(err, check.IsNil)
165                 c.Check(st.Size(), check.Equals, int64(2))
166                 c.Check(st.IsDir(), check.Equals, true)
167         }
168 }
169
170 func (s *CollectionFSSuite) TestNotExist(c *check.C) {
171         for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
172                 f, err := s.fs.Open(path)
173                 c.Assert(f, check.IsNil)
174                 c.Assert(err, check.NotNil)
175                 c.Assert(os.IsNotExist(err), check.Equals, true)
176         }
177 }
178
179 func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
180         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDONLY, 0)
181         c.Assert(err, check.IsNil)
182         st, err := f.Stat()
183         c.Assert(err, check.IsNil)
184         c.Check(st.Size(), check.Equals, int64(3))
185         n, err := f.Write([]byte("bar"))
186         c.Check(n, check.Equals, 0)
187         c.Check(err, check.Equals, ErrReadOnlyFile)
188 }
189
190 func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
191         f, err := s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE, 0)
192         c.Assert(err, check.IsNil)
193         st, err := f.Stat()
194         c.Assert(err, check.IsNil)
195         c.Check(st.Size(), check.Equals, int64(0))
196
197         n, err := f.Write([]byte("bar"))
198         c.Check(n, check.Equals, 3)
199         c.Check(err, check.IsNil)
200
201         c.Check(f.Close(), check.IsNil)
202
203         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
204         c.Check(f, check.IsNil)
205         c.Assert(err, check.NotNil)
206
207         f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR, 0)
208         c.Assert(err, check.IsNil)
209         st, err = f.Stat()
210         c.Assert(err, check.IsNil)
211         c.Check(st.Size(), check.Equals, int64(3))
212
213         c.Check(f.Close(), check.IsNil)
214
215         m, err := s.fs.MarshalManifest(".")
216         c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
217 }
218
219 func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
220         maxBlockSize = 8
221         defer func() { maxBlockSize = 2 << 26 }()
222
223         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
224         c.Assert(err, check.IsNil)
225         defer f.Close()
226         st, err := f.Stat()
227         c.Assert(err, check.IsNil)
228         c.Check(st.Size(), check.Equals, int64(3))
229
230         f2, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
231         c.Assert(err, check.IsNil)
232         defer f2.Close()
233
234         buf := make([]byte, 64)
235         n, err := f.Read(buf)
236         c.Check(n, check.Equals, 3)
237         c.Check(err, check.Equals, io.EOF)
238         c.Check(string(buf[:3]), check.DeepEquals, "foo")
239
240         pos, err := f.Seek(-2, io.SeekCurrent)
241         c.Check(pos, check.Equals, int64(1))
242         c.Check(err, check.IsNil)
243
244         // Split a storedExtent in two, and insert a memExtent
245         n, err = f.Write([]byte("*"))
246         c.Check(n, check.Equals, 1)
247         c.Check(err, check.IsNil)
248
249         pos, err = f.Seek(0, io.SeekCurrent)
250         c.Check(pos, check.Equals, int64(2))
251         c.Check(err, check.IsNil)
252
253         pos, err = f.Seek(0, io.SeekStart)
254         c.Check(pos, check.Equals, int64(0))
255         c.Check(err, check.IsNil)
256
257         rbuf, err := ioutil.ReadAll(f)
258         c.Check(len(rbuf), check.Equals, 3)
259         c.Check(err, check.IsNil)
260         c.Check(string(rbuf), check.Equals, "f*o")
261
262         // Write multiple blocks in one call
263         f.Seek(1, io.SeekStart)
264         n, err = f.Write([]byte("0123456789abcdefg"))
265         c.Check(n, check.Equals, 17)
266         c.Check(err, check.IsNil)
267         pos, err = f.Seek(0, io.SeekCurrent)
268         c.Check(pos, check.Equals, int64(18))
269         pos, err = f.Seek(-18, io.SeekCurrent)
270         c.Check(err, check.IsNil)
271         n, err = io.ReadFull(f, buf)
272         c.Check(n, check.Equals, 18)
273         c.Check(err, check.Equals, io.ErrUnexpectedEOF)
274         c.Check(string(buf[:n]), check.Equals, "f0123456789abcdefg")
275
276         buf2, err := ioutil.ReadAll(f2)
277         c.Check(err, check.IsNil)
278         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
279
280         // truncate to current size
281         err = f.Truncate(18)
282         f2.Seek(0, io.SeekStart)
283         buf2, err = ioutil.ReadAll(f2)
284         c.Check(err, check.IsNil)
285         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
286
287         // shrink to zero some data
288         f.Truncate(15)
289         f2.Seek(0, io.SeekStart)
290         buf2, err = ioutil.ReadAll(f2)
291         c.Check(err, check.IsNil)
292         c.Check(string(buf2), check.Equals, "f0123456789abcd")
293
294         // grow to partial block/extent
295         f.Truncate(20)
296         f2.Seek(0, io.SeekStart)
297         buf2, err = ioutil.ReadAll(f2)
298         c.Check(err, check.IsNil)
299         c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
300
301         f.Truncate(0)
302         f2.Seek(0, io.SeekStart)
303         f2.Write([]byte("12345678abcdefghijkl"))
304
305         // grow to block/extent boundary
306         f.Truncate(64)
307         f2.Seek(0, io.SeekStart)
308         buf2, err = ioutil.ReadAll(f2)
309         c.Check(err, check.IsNil)
310         c.Check(len(buf2), check.Equals, 64)
311         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 8)
312
313         // shrink to block/extent boundary
314         err = f.Truncate(32)
315         f2.Seek(0, io.SeekStart)
316         buf2, err = ioutil.ReadAll(f2)
317         c.Check(err, check.IsNil)
318         c.Check(len(buf2), check.Equals, 32)
319         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 4)
320
321         // shrink to partial block/extent
322         err = f.Truncate(15)
323         f2.Seek(0, io.SeekStart)
324         buf2, err = ioutil.ReadAll(f2)
325         c.Check(err, check.IsNil)
326         c.Check(string(buf2), check.Equals, "12345678abcdefg")
327         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 2)
328
329         // Force flush to ensure the block "12345678" gets stored, so
330         // we know what to expect in the final manifest below.
331         _, err = s.fs.MarshalManifest(".")
332         c.Check(err, check.IsNil)
333
334         // Truncate to size=3 while f2's ptr is at 15
335         err = f.Truncate(3)
336         c.Check(err, check.IsNil)
337         buf2, err = ioutil.ReadAll(f2)
338         c.Check(err, check.IsNil)
339         c.Check(string(buf2), check.Equals, "")
340         f2.Seek(0, io.SeekStart)
341         buf2, err = ioutil.ReadAll(f2)
342         c.Check(err, check.IsNil)
343         c.Check(string(buf2), check.Equals, "123")
344         c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 1)
345
346         m, err := s.fs.MarshalManifest(".")
347         c.Check(err, check.IsNil)
348         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
349         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 25d55ad283aa400af464c76d713c07ad+8 3:3:bar 6:3:foo\n")
350 }
351
352 func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
353         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
354         c.Assert(err, check.IsNil)
355         f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
356         c.Assert(err, check.IsNil)
357         defer f.Close()
358
359         checkSize := func(size int64) {
360                 fi, err := f.Stat()
361                 c.Check(fi.Size(), check.Equals, size)
362
363                 f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
364                 c.Assert(err, check.IsNil)
365                 defer f.Close()
366                 fi, err = f.Stat()
367                 c.Check(fi.Size(), check.Equals, size)
368                 pos, err := f.Seek(0, io.SeekEnd)
369                 c.Check(pos, check.Equals, size)
370         }
371
372         f.Seek(2, io.SeekEnd)
373         checkSize(0)
374         f.Write([]byte{1})
375         checkSize(3)
376
377         f.Seek(2, io.SeekCurrent)
378         checkSize(3)
379         f.Write([]byte{})
380         checkSize(5)
381
382         f.Seek(8, io.SeekStart)
383         checkSize(5)
384         n, err := f.Read(make([]byte, 1))
385         c.Check(n, check.Equals, 0)
386         c.Check(err, check.Equals, io.EOF)
387         checkSize(5)
388         f.Write([]byte{1, 2, 3})
389         checkSize(11)
390 }
391
392 func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
393         maxBlockSize = 8
394         defer func() { maxBlockSize = 2 << 26 }()
395
396         var err error
397         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
398         c.Assert(err, check.IsNil)
399         for _, name := range []string{"foo", "bar", "baz"} {
400                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
401                 c.Assert(err, check.IsNil)
402                 f.Write([]byte(name))
403                 f.Close()
404         }
405
406         m, err := s.fs.MarshalManifest(".")
407         c.Check(err, check.IsNil)
408         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
409         c.Check(m, check.Equals, ". c3c23db5285662ef7172373df0003206+6 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar 3:3:baz 6:3:foo\n")
410 }
411
412 func (s *CollectionFSSuite) TestMkdir(c *check.C) {
413         err := s.fs.Mkdir("foo/bar", 0755)
414         c.Check(err, check.Equals, os.ErrNotExist)
415
416         f, err := s.fs.OpenFile("foo/bar", os.O_CREATE, 0)
417         c.Check(err, check.Equals, os.ErrNotExist)
418
419         err = s.fs.Mkdir("foo", 0755)
420         c.Check(err, check.IsNil)
421
422         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_WRONLY, 0)
423         c.Check(err, check.IsNil)
424         if err == nil {
425                 defer f.Close()
426                 f.Write([]byte("foo"))
427         }
428
429         // mkdir fails if a file already exists with that name
430         err = s.fs.Mkdir("foo/bar", 0755)
431         c.Check(err, check.NotNil)
432
433         err = s.fs.Remove("foo/bar")
434         c.Check(err, check.IsNil)
435
436         // mkdir succeds after the file is deleted
437         err = s.fs.Mkdir("foo/bar", 0755)
438         c.Check(err, check.IsNil)
439
440         // creating a file in a nonexistent subdir should still fail
441         f, err = s.fs.OpenFile("foo/bar/baz/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
442         c.Check(err, check.Equals, os.ErrNotExist)
443
444         f, err = s.fs.OpenFile("foo/bar/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
445         c.Check(err, check.IsNil)
446         if err == nil {
447                 defer f.Close()
448                 f.Write([]byte("foo"))
449         }
450
451         // creating foo/bar as a regular file should fail
452         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, 0)
453         c.Check(err, check.NotNil)
454
455         // creating foo/bar as a directory should fail
456         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, os.ModeDir)
457         c.Check(err, check.NotNil)
458         err = s.fs.Mkdir("foo/bar", 0755)
459         c.Check(err, check.NotNil)
460
461         m, err := s.fs.MarshalManifest(".")
462         c.Check(err, check.IsNil)
463         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
464         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 3:3:bar 0:3:foo\n./foo/bar acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
465 }
466
467 func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
468         maxBlockSize = 8
469         defer func() { maxBlockSize = 2 << 26 }()
470
471         var wg sync.WaitGroup
472         for n := 0; n < 128; n++ {
473                 wg.Add(1)
474                 go func() {
475                         defer wg.Done()
476                         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
477                         c.Assert(err, check.IsNil)
478                         defer f.Close()
479                         for i := 0; i < 6502; i++ {
480                                 switch rand.Int() & 3 {
481                                 case 0:
482                                         f.Truncate(int64(rand.Intn(64)))
483                                 case 1:
484                                         f.Seek(int64(rand.Intn(64)), io.SeekStart)
485                                 case 2:
486                                         _, err := f.Write([]byte("beep boop"))
487                                         c.Check(err, check.IsNil)
488                                 case 3:
489                                         _, err := ioutil.ReadAll(f)
490                                         c.Check(err, check.IsNil)
491                                 }
492                         }
493                 }()
494         }
495         wg.Wait()
496
497         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
498         c.Assert(err, check.IsNil)
499         defer f.Close()
500         buf, err := ioutil.ReadAll(f)
501         c.Check(err, check.IsNil)
502         c.Logf("after lots of random r/w/seek/trunc, buf is %q", buf)
503 }
504
505 func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
506         maxBlockSize = 40
507         defer func() { maxBlockSize = 2 << 26 }()
508
509         var err error
510         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
511         c.Assert(err, check.IsNil)
512
513         const nfiles = 256
514         const ngoroutines = 256
515
516         var wg sync.WaitGroup
517         for n := 0; n < nfiles; n++ {
518                 wg.Add(1)
519                 go func(n int) {
520                         defer wg.Done()
521                         expect := make([]byte, 0, 64)
522                         wbytes := []byte("there's no simple explanation for anything important that any of us do")
523                         f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
524                         c.Assert(err, check.IsNil)
525                         defer f.Close()
526                         for i := 0; i < ngoroutines; i++ {
527                                 trunc := rand.Intn(65)
528                                 woff := rand.Intn(trunc + 1)
529                                 wbytes = wbytes[:rand.Intn(64-woff+1)]
530                                 for buf, i := expect[:cap(expect)], len(expect); i < trunc; i++ {
531                                         buf[i] = 0
532                                 }
533                                 expect = expect[:trunc]
534                                 if trunc < woff+len(wbytes) {
535                                         expect = expect[:woff+len(wbytes)]
536                                 }
537                                 copy(expect[woff:], wbytes)
538                                 f.Truncate(int64(trunc))
539                                 pos, err := f.Seek(int64(woff), io.SeekStart)
540                                 c.Check(pos, check.Equals, int64(woff))
541                                 c.Check(err, check.IsNil)
542                                 n, err := f.Write(wbytes)
543                                 c.Check(n, check.Equals, len(wbytes))
544                                 c.Check(err, check.IsNil)
545                                 pos, err = f.Seek(0, io.SeekStart)
546                                 c.Check(pos, check.Equals, int64(0))
547                                 c.Check(err, check.IsNil)
548                                 buf, err := ioutil.ReadAll(f)
549                                 c.Check(string(buf), check.Equals, string(expect))
550                                 c.Check(err, check.IsNil)
551                         }
552                         s.checkMemSize(c, f)
553                 }(n)
554         }
555         wg.Wait()
556
557         root, err := s.fs.Open("/")
558         c.Assert(err, check.IsNil)
559         defer root.Close()
560         fi, err := root.Readdir(-1)
561         c.Check(err, check.IsNil)
562         c.Check(len(fi), check.Equals, nfiles)
563
564         _, err = s.fs.MarshalManifest(".")
565         c.Check(err, check.IsNil)
566         // TODO: check manifest content
567 }
568
569 func (s *CollectionFSSuite) TestRemove(c *check.C) {
570         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
571         c.Assert(err, check.IsNil)
572         err = fs.Mkdir("dir0", 0755)
573         c.Assert(err, check.IsNil)
574         err = fs.Mkdir("dir1", 0755)
575         c.Assert(err, check.IsNil)
576         err = fs.Mkdir("dir1/dir2", 0755)
577         c.Assert(err, check.IsNil)
578         err = fs.Mkdir("dir1/dir3", 0755)
579         c.Assert(err, check.IsNil)
580
581         err = fs.Remove("dir0")
582         c.Check(err, check.IsNil)
583         err = fs.Remove("dir0")
584         c.Check(err, check.Equals, os.ErrNotExist)
585
586         err = fs.Remove("dir1/dir2/.")
587         c.Check(err, check.Equals, ErrInvalidArgument)
588         err = fs.Remove("dir1/dir2/..")
589         c.Check(err, check.Equals, ErrInvalidArgument)
590         err = fs.Remove("dir1")
591         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
592         err = fs.Remove("dir1/dir2/../../../dir1")
593         c.Check(err, check.Equals, ErrDirectoryNotEmpty)
594         err = fs.Remove("dir1/dir3/")
595         c.Check(err, check.IsNil)
596         err = fs.RemoveAll("dir1")
597         c.Check(err, check.IsNil)
598         err = fs.RemoveAll("dir1")
599         c.Check(err, check.IsNil)
600 }
601
602 func (s *CollectionFSSuite) TestRenameError(c *check.C) {
603         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
604         c.Assert(err, check.IsNil)
605         err = fs.Mkdir("first", 0755)
606         c.Assert(err, check.IsNil)
607         err = fs.Mkdir("first/second", 0755)
608         c.Assert(err, check.IsNil)
609         f, err := fs.OpenFile("first/second/file", os.O_CREATE|os.O_WRONLY, 0755)
610         c.Assert(err, check.IsNil)
611         f.Write([]byte{1, 2, 3, 4, 5})
612         f.Close()
613         err = fs.Rename("first", "first/second/third")
614         c.Check(err, check.Equals, ErrInvalidArgument)
615         err = fs.Rename("first", "first/third")
616         c.Check(err, check.Equals, ErrInvalidArgument)
617         err = fs.Rename("first/second", "second")
618         c.Check(err, check.IsNil)
619         f, err = fs.OpenFile("second/file", 0, 0)
620         c.Assert(err, check.IsNil)
621         data, err := ioutil.ReadAll(f)
622         c.Check(err, check.IsNil)
623         c.Check(data, check.DeepEquals, []byte{1, 2, 3, 4, 5})
624 }
625
626 func (s *CollectionFSSuite) TestRename(c *check.C) {
627         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
628         c.Assert(err, check.IsNil)
629         const (
630                 outer = 16
631                 inner = 16
632         )
633         for i := 0; i < outer; i++ {
634                 err = fs.Mkdir(fmt.Sprintf("dir%d", i), 0755)
635                 c.Assert(err, check.IsNil)
636                 for j := 0; j < inner; j++ {
637                         err = fs.Mkdir(fmt.Sprintf("dir%d/dir%d", i, j), 0755)
638                         c.Assert(err, check.IsNil)
639                         for _, fnm := range []string{
640                                 fmt.Sprintf("dir%d/file%d", i, j),
641                                 fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
642                         } {
643                                 f, err := fs.OpenFile(fnm, os.O_CREATE|os.O_WRONLY, 0755)
644                                 c.Assert(err, check.IsNil)
645                                 _, err = f.Write([]byte("beep"))
646                                 c.Assert(err, check.IsNil)
647                                 f.Close()
648                         }
649                 }
650         }
651         var wg sync.WaitGroup
652         for i := 0; i < outer; i++ {
653                 for j := 0; j < inner; j++ {
654                         wg.Add(1)
655                         go func(i, j int) {
656                                 defer wg.Done()
657                                 oldname := fmt.Sprintf("dir%d/dir%d/file%d", i, j, j)
658                                 newname := fmt.Sprintf("dir%d/newfile%d", i, inner-j-1)
659                                 _, err := fs.Open(newname)
660                                 c.Check(err, check.Equals, os.ErrNotExist)
661                                 err = fs.Rename(oldname, newname)
662                                 c.Check(err, check.IsNil)
663                                 f, err := fs.Open(newname)
664                                 c.Check(err, check.IsNil)
665                                 f.Close()
666                         }(i, j)
667
668                         wg.Add(1)
669                         go func(i, j int) {
670                                 defer wg.Done()
671                                 // oldname does not exist
672                                 err := fs.Rename(
673                                         fmt.Sprintf("dir%d/dir%d/missing", i, j),
674                                         fmt.Sprintf("dir%d/dir%d/file%d", outer-i-1, j, j))
675                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
676
677                                 // newname parent dir does not exist
678                                 err = fs.Rename(
679                                         fmt.Sprintf("dir%d/dir%d", i, j),
680                                         fmt.Sprintf("dir%d/missing/irrelevant", outer-i-1))
681                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
682
683                                 // oldname parent dir is a file
684                                 err = fs.Rename(
685                                         fmt.Sprintf("dir%d/file%d/patherror", i, j),
686                                         fmt.Sprintf("dir%d/irrelevant", i))
687                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
688
689                                 // newname parent dir is a file
690                                 err = fs.Rename(
691                                         fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
692                                         fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
693                                 c.Check(err, check.ErrorMatches, `.*does not exist`)
694                         }(i, j)
695                 }
696         }
697         wg.Wait()
698
699         f, err := fs.OpenFile("dir1/newfile3", 0, 0)
700         c.Assert(err, check.IsNil)
701         c.Check(f.Size(), check.Equals, int64(4))
702         buf, err := ioutil.ReadAll(f)
703         c.Check(buf, check.DeepEquals, []byte("beep"))
704         c.Check(err, check.IsNil)
705         _, err = fs.Open("dir1/dir1/file1")
706         c.Check(err, check.Equals, os.ErrNotExist)
707 }
708
709 func (s *CollectionFSSuite) TestPersist(c *check.C) {
710         maxBlockSize = 1024
711         defer func() { maxBlockSize = 2 << 26 }()
712
713         var err error
714         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
715         c.Assert(err, check.IsNil)
716         err = s.fs.Mkdir("d:r", 0755)
717         c.Assert(err, check.IsNil)
718
719         expect := map[string][]byte{}
720
721         var wg sync.WaitGroup
722         for _, name := range []string{"random 1", "random:2", "random\\3", "d:r/random4"} {
723                 buf := make([]byte, 500)
724                 rand.Read(buf)
725                 expect[name] = buf
726
727                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
728                 c.Assert(err, check.IsNil)
729                 // Note: we don't close the file until after the test
730                 // is done. Writes to unclosed files should persist.
731                 defer f.Close()
732
733                 wg.Add(1)
734                 go func() {
735                         defer wg.Done()
736                         for i := 0; i < len(buf); i += 5 {
737                                 _, err := f.Write(buf[i : i+5])
738                                 c.Assert(err, check.IsNil)
739                         }
740                 }()
741         }
742         wg.Wait()
743
744         m, err := s.fs.MarshalManifest(".")
745         c.Check(err, check.IsNil)
746         c.Logf("%q", m)
747
748         root, err := s.fs.Open("/")
749         c.Assert(err, check.IsNil)
750         defer root.Close()
751         fi, err := root.Readdir(-1)
752         c.Check(err, check.IsNil)
753         c.Check(len(fi), check.Equals, 4)
754
755         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
756         c.Assert(err, check.IsNil)
757
758         root, err = persisted.Open("/")
759         c.Assert(err, check.IsNil)
760         defer root.Close()
761         fi, err = root.Readdir(-1)
762         c.Check(err, check.IsNil)
763         c.Check(len(fi), check.Equals, 4)
764
765         for name, content := range expect {
766                 c.Logf("read %q", name)
767                 f, err := persisted.Open(name)
768                 c.Assert(err, check.IsNil)
769                 defer f.Close()
770                 buf, err := ioutil.ReadAll(f)
771                 c.Check(err, check.IsNil)
772                 c.Check(buf, check.DeepEquals, content)
773         }
774 }
775
776 func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
777         var err error
778         s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
779         c.Assert(err, check.IsNil)
780         for _, name := range []string{"dir", "dir/zerodir", "zero", "zero/zero"} {
781                 err = s.fs.Mkdir(name, 0755)
782                 c.Assert(err, check.IsNil)
783         }
784
785         expect := map[string][]byte{
786                 "0":                nil,
787                 "00":               []byte{},
788                 "one":              []byte{1},
789                 "dir/0":            nil,
790                 "dir/two":          []byte{1, 2},
791                 "dir/zero":         nil,
792                 "dir/zerodir/zero": nil,
793                 "zero/zero/zero":   nil,
794         }
795         for name, data := range expect {
796                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
797                 c.Assert(err, check.IsNil)
798                 if data != nil {
799                         _, err := f.Write(data)
800                         c.Assert(err, check.IsNil)
801                 }
802                 f.Close()
803         }
804
805         m, err := s.fs.MarshalManifest(".")
806         c.Check(err, check.IsNil)
807         c.Logf("%q", m)
808
809         persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
810         c.Assert(err, check.IsNil)
811
812         for name, data := range expect {
813                 f, err := persisted.Open("bogus-" + name)
814                 c.Check(err, check.NotNil)
815
816                 f, err = persisted.Open(name)
817                 c.Assert(err, check.IsNil)
818
819                 if data == nil {
820                         data = []byte{}
821                 }
822                 buf, err := ioutil.ReadAll(f)
823                 c.Check(err, check.IsNil)
824                 c.Check(buf, check.DeepEquals, data)
825         }
826 }
827
828 func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
829         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
830         c.Assert(err, check.IsNil)
831
832         f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
833         c.Check(f, check.IsNil)
834         c.Check(err, check.ErrorMatches, `file does not exist`)
835
836         f, err = fs.OpenFile("new", os.O_CREATE|os.O_RDONLY, 0)
837         c.Assert(err, check.IsNil)
838         defer f.Close()
839         n, err := f.Write([]byte{1, 2, 3})
840         c.Check(n, check.Equals, 0)
841         c.Check(err, check.ErrorMatches, `read-only file`)
842         n, err = f.Read(make([]byte, 1))
843         c.Check(n, check.Equals, 0)
844         c.Check(err, check.Equals, io.EOF)
845         f, err = fs.OpenFile("new", os.O_RDWR, 0)
846         c.Assert(err, check.IsNil)
847         defer f.Close()
848         _, err = f.Write([]byte{4, 5, 6})
849         c.Check(err, check.IsNil)
850         fi, err := f.Stat()
851         c.Assert(err, check.IsNil)
852         c.Check(fi.Size(), check.Equals, int64(3))
853
854         f, err = fs.OpenFile("new", os.O_TRUNC|os.O_RDWR, 0)
855         c.Assert(err, check.IsNil)
856         defer f.Close()
857         pos, err := f.Seek(0, io.SeekEnd)
858         c.Check(pos, check.Equals, int64(0))
859         c.Check(err, check.IsNil)
860         fi, err = f.Stat()
861         c.Assert(err, check.IsNil)
862         c.Check(fi.Size(), check.Equals, int64(0))
863         fs.Remove("new")
864
865         buf := make([]byte, 64)
866         f, err = fs.OpenFile("append", os.O_EXCL|os.O_CREATE|os.O_RDWR|os.O_APPEND, 0)
867         c.Assert(err, check.IsNil)
868         f.Write([]byte{1, 2, 3})
869         f.Seek(0, io.SeekStart)
870         n, _ = f.Read(buf[:1])
871         c.Check(n, check.Equals, 1)
872         c.Check(buf[:1], check.DeepEquals, []byte{1})
873         pos, err = f.Seek(0, io.SeekCurrent)
874         c.Check(pos, check.Equals, int64(1))
875         f.Write([]byte{4, 5, 6})
876         pos, err = f.Seek(0, io.SeekCurrent)
877         c.Check(pos, check.Equals, int64(6))
878         f.Seek(0, io.SeekStart)
879         n, err = f.Read(buf)
880         c.Check(buf[:n], check.DeepEquals, []byte{1, 2, 3, 4, 5, 6})
881         c.Check(err, check.Equals, io.EOF)
882         f.Close()
883
884         f, err = fs.OpenFile("append", os.O_RDWR|os.O_APPEND, 0)
885         c.Assert(err, check.IsNil)
886         pos, err = f.Seek(0, io.SeekCurrent)
887         c.Check(pos, check.Equals, int64(0))
888         c.Check(err, check.IsNil)
889         f.Read(buf[:3])
890         pos, _ = f.Seek(0, io.SeekCurrent)
891         c.Check(pos, check.Equals, int64(3))
892         f.Write([]byte{7, 8, 9})
893         pos, err = f.Seek(0, io.SeekCurrent)
894         c.Check(pos, check.Equals, int64(9))
895         f.Close()
896
897         f, err = fs.OpenFile("wronly", os.O_CREATE|os.O_WRONLY, 0)
898         c.Assert(err, check.IsNil)
899         n, err = f.Write([]byte{3, 2, 1})
900         c.Check(n, check.Equals, 3)
901         c.Check(err, check.IsNil)
902         pos, _ = f.Seek(0, io.SeekCurrent)
903         c.Check(pos, check.Equals, int64(3))
904         pos, _ = f.Seek(0, io.SeekStart)
905         c.Check(pos, check.Equals, int64(0))
906         n, err = f.Read(buf)
907         c.Check(n, check.Equals, 0)
908         c.Check(err, check.ErrorMatches, `.*O_WRONLY.*`)
909         f, err = fs.OpenFile("wronly", os.O_RDONLY, 0)
910         c.Assert(err, check.IsNil)
911         n, _ = f.Read(buf)
912         c.Check(buf[:n], check.DeepEquals, []byte{3, 2, 1})
913
914         f, err = fs.OpenFile("unsupported", os.O_CREATE|os.O_SYNC, 0)
915         c.Check(f, check.IsNil)
916         c.Check(err, check.NotNil)
917
918         f, err = fs.OpenFile("append", os.O_RDWR|os.O_WRONLY, 0)
919         c.Check(f, check.IsNil)
920         c.Check(err, check.ErrorMatches, `invalid flag.*`)
921 }
922
923 func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
924         maxBlockSize = 1024
925         defer func() { maxBlockSize = 2 << 26 }()
926
927         fs, err := (&Collection{}).FileSystem(s.client, s.kc)
928         c.Assert(err, check.IsNil)
929         f, err := fs.OpenFile("50K", os.O_WRONLY|os.O_CREATE, 0)
930         c.Assert(err, check.IsNil)
931         defer f.Close()
932
933         data := make([]byte, 500)
934         rand.Read(data)
935
936         for i := 0; i < 100; i++ {
937                 n, err := f.Write(data)
938                 c.Assert(n, check.Equals, len(data))
939                 c.Assert(err, check.IsNil)
940         }
941
942         currentMemExtents := func() (memExtents []int) {
943                 for idx, e := range f.(*filehandle).inode.(*filenode).segments {
944                         switch e.(type) {
945                         case *memSegment:
946                                 memExtents = append(memExtents, idx)
947                         }
948                 }
949                 return
950         }
951         c.Check(currentMemExtents(), check.HasLen, 1)
952
953         m, err := fs.MarshalManifest(".")
954         c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
955         c.Check(err, check.IsNil)
956         c.Check(currentMemExtents(), check.HasLen, 0)
957 }
958
959 func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
960         for _, txt := range []string{
961                 "\n",
962                 ".\n",
963                 ". \n",
964                 ". d41d8cd98f00b204e9800998ecf8427e+0\n",
965                 ". d41d8cd98f00b204e9800998ecf8427e+0 \n",
966                 ". 0:0:foo\n",
967                 ".  0:0:foo\n",
968                 ". 0:0:foo 0:0:bar\n",
969                 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo\n",
970                 ". d41d8cd98f00b204e9800998ecf8427e+0 :0:0:foo\n",
971                 ". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
972                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
973                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
974                 ". d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n",
975                 "./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
976         } {
977                 c.Logf("<-%q", txt)
978                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
979                 c.Check(fs, check.IsNil)
980                 c.Logf("-> %s", err)
981                 c.Check(err, check.NotNil)
982         }
983 }
984
985 func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
986         for _, txt := range []string{
987                 "",
988                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
989                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
990                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
991                 ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
992         } {
993                 c.Logf("<-%q", txt)
994                 fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
995                 c.Check(err, check.IsNil)
996                 c.Check(fs, check.NotNil)
997         }
998 }
999
1000 func (s *CollectionFSSuite) checkMemSize(c *check.C, f File) {
1001         fn := f.(*filehandle).inode.(*filenode)
1002         var memsize int64
1003         for _, seg := range fn.segments {
1004                 if e, ok := seg.(*memSegment); ok {
1005                         memsize += int64(len(e.buf))
1006                 }
1007         }
1008         c.Check(fn.memsize, check.Equals, memsize)
1009 }
1010
1011 type CollectionFSUnitSuite struct{}
1012
1013 var _ = check.Suite(&CollectionFSUnitSuite{})
1014
1015 // expect ~2 seconds to load a manifest with 256K files
1016 func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
1017         const (
1018                 dirCount  = 512
1019                 fileCount = 512
1020         )
1021
1022         mb := bytes.NewBuffer(make([]byte, 0, 40000000))
1023         for i := 0; i < dirCount; i++ {
1024                 fmt.Fprintf(mb, "./dir%d", i)
1025                 for j := 0; j <= fileCount; j++ {
1026                         fmt.Fprintf(mb, " %032x+42+A%040x@%08x", j, j, j)
1027                 }
1028                 for j := 0; j < fileCount; j++ {
1029                         fmt.Fprintf(mb, " %d:%d:dir%d/file%d", j*42+21, 42, j, j)
1030                 }
1031                 mb.Write([]byte{'\n'})
1032         }
1033         coll := Collection{ManifestText: mb.String()}
1034         c.Logf("%s built", time.Now())
1035
1036         var memstats runtime.MemStats
1037         runtime.ReadMemStats(&memstats)
1038         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1039
1040         f, err := coll.FileSystem(nil, nil)
1041         c.Check(err, check.IsNil)
1042         c.Logf("%s loaded", time.Now())
1043
1044         for i := 0; i < dirCount; i++ {
1045                 for j := 0; j < fileCount; j++ {
1046                         f.Stat(fmt.Sprintf("./dir%d/dir%d/file%d", i, j, j))
1047                 }
1048         }
1049         c.Logf("%s Stat() x %d", time.Now(), dirCount*fileCount)
1050
1051         runtime.ReadMemStats(&memstats)
1052         c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
1053 }
1054
1055 // Gocheck boilerplate
1056 func Test(t *testing.T) {
1057         check.TestingT(t)
1058 }