package arvados
import (
+ "bytes"
"crypto/md5"
"errors"
"fmt"
"net/http"
"os"
"regexp"
+ "runtime"
"sync"
"testing"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
c.Check(ok, check.Equals, true)
}
+func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
+ fs, err := (&Collection{
+ ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
+ }).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+
+ f, err := fs.Open("/foo:foo")
+ c.Assert(err, check.IsNil)
+
+ fis, err := f.Readdir(0)
+ c.Check(err, check.IsNil)
+ c.Check(len(fis), check.Equals, 1)
+ c.Check(fis[0].Name(), check.Equals, "bar:bar")
+}
+
func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
f, err := s.fs.Open("/dir1")
c.Assert(err, check.IsNil)
}
func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
- f, err := s.fs.OpenFile("/newfile", os.O_RDWR|os.O_CREATE, 0)
+ f, err := s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE, 0)
c.Assert(err, check.IsNil)
st, err := f.Stat()
c.Assert(err, check.IsNil)
c.Check(f.Close(), check.IsNil)
- f, err = s.fs.OpenFile("/newfile", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
+ f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
c.Check(f, check.IsNil)
c.Assert(err, check.NotNil)
- f, err = s.fs.OpenFile("/newfile", os.O_RDWR, 0)
+ f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR, 0)
c.Assert(err, check.IsNil)
st, err = f.Stat()
c.Assert(err, check.IsNil)
c.Check(f.Close(), check.IsNil)
- // TODO: serialize to Collection, confirm manifest contents,
- // make new FileSystem, confirm file contents.
+ m, err := s.fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
}
func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
c.Check(err, check.Equals, io.EOF)
c.Check(string(buf[:3]), check.DeepEquals, "foo")
- pos, err := f.Seek(-2, os.SEEK_CUR)
+ pos, err := f.Seek(-2, io.SeekCurrent)
c.Check(pos, check.Equals, int64(1))
c.Check(err, check.IsNil)
c.Check(n, check.Equals, 1)
c.Check(err, check.IsNil)
- pos, err = f.Seek(0, os.SEEK_CUR)
+ pos, err = f.Seek(0, io.SeekCurrent)
c.Check(pos, check.Equals, int64(2))
c.Check(err, check.IsNil)
- pos, err = f.Seek(0, os.SEEK_SET)
+ pos, err = f.Seek(0, io.SeekStart)
c.Check(pos, check.Equals, int64(0))
c.Check(err, check.IsNil)
c.Check(string(rbuf), check.Equals, "f*o")
// Write multiple blocks in one call
- f.Seek(1, os.SEEK_SET)
+ f.Seek(1, io.SeekStart)
n, err = f.Write([]byte("0123456789abcdefg"))
c.Check(n, check.Equals, 17)
c.Check(err, check.IsNil)
- pos, err = f.Seek(0, os.SEEK_CUR)
+ pos, err = f.Seek(0, io.SeekCurrent)
c.Check(pos, check.Equals, int64(18))
- pos, err = f.Seek(-18, os.SEEK_CUR)
+ c.Check(err, check.IsNil)
+ pos, err = f.Seek(-18, io.SeekCurrent)
+ c.Check(pos, check.Equals, int64(0))
c.Check(err, check.IsNil)
n, err = io.ReadFull(f, buf)
c.Check(n, check.Equals, 18)
// truncate to current size
err = f.Truncate(18)
- f2.Seek(0, os.SEEK_SET)
+ c.Check(err, check.IsNil)
+ f2.Seek(0, io.SeekStart)
buf2, err = ioutil.ReadAll(f2)
c.Check(err, check.IsNil)
c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
// shrink to zero some data
f.Truncate(15)
- f2.Seek(0, os.SEEK_SET)
+ f2.Seek(0, io.SeekStart)
buf2, err = ioutil.ReadAll(f2)
c.Check(err, check.IsNil)
c.Check(string(buf2), check.Equals, "f0123456789abcd")
// grow to partial block/extent
f.Truncate(20)
- f2.Seek(0, os.SEEK_SET)
+ f2.Seek(0, io.SeekStart)
buf2, err = ioutil.ReadAll(f2)
c.Check(err, check.IsNil)
c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
f.Truncate(0)
+ f2.Seek(0, io.SeekStart)
f2.Write([]byte("12345678abcdefghijkl"))
// grow to block/extent boundary
f.Truncate(64)
- f2.Seek(0, os.SEEK_SET)
+ f2.Seek(0, io.SeekStart)
buf2, err = ioutil.ReadAll(f2)
c.Check(err, check.IsNil)
c.Check(len(buf2), check.Equals, 64)
- c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 8)
+ c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 8)
// shrink to block/extent boundary
err = f.Truncate(32)
- f2.Seek(0, os.SEEK_SET)
+ c.Check(err, check.IsNil)
+ f2.Seek(0, io.SeekStart)
buf2, err = ioutil.ReadAll(f2)
c.Check(err, check.IsNil)
c.Check(len(buf2), check.Equals, 32)
- c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 4)
+ c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 4)
// shrink to partial block/extent
err = f.Truncate(15)
- f2.Seek(0, os.SEEK_SET)
+ c.Check(err, check.IsNil)
+ f2.Seek(0, io.SeekStart)
buf2, err = ioutil.ReadAll(f2)
c.Check(err, check.IsNil)
c.Check(string(buf2), check.Equals, "12345678abcdefg")
- c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 2)
+ c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 2)
// Force flush to ensure the block "12345678" gets stored, so
// we know what to expect in the final manifest below.
buf2, err = ioutil.ReadAll(f2)
c.Check(err, check.IsNil)
c.Check(string(buf2), check.Equals, "")
- f2.Seek(0, os.SEEK_SET)
+ f2.Seek(0, io.SeekStart)
buf2, err = ioutil.ReadAll(f2)
c.Check(err, check.IsNil)
c.Check(string(buf2), check.Equals, "123")
- c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 1)
+ c.Check(len(f.(*filehandle).inode.(*filenode).segments), check.Equals, 1)
m, err := s.fs.MarshalManifest(".")
c.Check(err, check.IsNil)
c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 25d55ad283aa400af464c76d713c07ad+8 3:3:bar 6:3:foo\n")
}
+func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+
+ checkSize := func(size int64) {
+ fi, err := f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(fi.Size(), check.Equals, size)
+
+ f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ fi, err = f.Stat()
+ c.Check(err, check.IsNil)
+ c.Check(fi.Size(), check.Equals, size)
+ pos, err := f.Seek(0, io.SeekEnd)
+ c.Check(err, check.IsNil)
+ c.Check(pos, check.Equals, size)
+ }
+
+ f.Seek(2, io.SeekEnd)
+ checkSize(0)
+ f.Write([]byte{1})
+ checkSize(3)
+
+ f.Seek(2, io.SeekCurrent)
+ checkSize(3)
+ f.Write([]byte{})
+ checkSize(5)
+
+ f.Seek(8, io.SeekStart)
+ checkSize(5)
+ n, err := f.Read(make([]byte, 1))
+ c.Check(n, check.Equals, 0)
+ c.Check(err, check.Equals, io.EOF)
+ checkSize(5)
+ f.Write([]byte{1, 2, 3})
+ checkSize(11)
+}
+
func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
maxBlockSize = 8
defer func() { maxBlockSize = 2 << 26 }()
err = s.fs.Remove("foo/bar")
c.Check(err, check.IsNil)
- // mkdir succeds after the file is deleted
+ // mkdir succeeds after the file is deleted
err = s.fs.Mkdir("foo/bar", 0755)
c.Check(err, check.IsNil)
case 0:
f.Truncate(int64(rand.Intn(64)))
case 1:
- f.Seek(int64(rand.Intn(64)), os.SEEK_SET)
+ f.Seek(int64(rand.Intn(64)), io.SeekStart)
case 2:
_, err := f.Write([]byte("beep boop"))
c.Check(err, check.IsNil)
}
copy(expect[woff:], wbytes)
f.Truncate(int64(trunc))
- pos, err := f.Seek(int64(woff), os.SEEK_SET)
+ pos, err := f.Seek(int64(woff), io.SeekStart)
c.Check(pos, check.Equals, int64(woff))
c.Check(err, check.IsNil)
n, err := f.Write(wbytes)
c.Check(n, check.Equals, len(wbytes))
c.Check(err, check.IsNil)
- pos, err = f.Seek(0, os.SEEK_SET)
+ pos, err = f.Seek(0, io.SeekStart)
c.Check(pos, check.Equals, int64(0))
c.Check(err, check.IsNil)
buf, err := ioutil.ReadAll(f)
// TODO: check manifest content
}
+func (s *CollectionFSSuite) TestRemove(c *check.C) {
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+ err = fs.Mkdir("dir0", 0755)
+ c.Assert(err, check.IsNil)
+ err = fs.Mkdir("dir1", 0755)
+ c.Assert(err, check.IsNil)
+ err = fs.Mkdir("dir1/dir2", 0755)
+ c.Assert(err, check.IsNil)
+ err = fs.Mkdir("dir1/dir3", 0755)
+ c.Assert(err, check.IsNil)
+
+ err = fs.Remove("dir0")
+ c.Check(err, check.IsNil)
+ err = fs.Remove("dir0")
+ c.Check(err, check.Equals, os.ErrNotExist)
+
+ err = fs.Remove("dir1/dir2/.")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ err = fs.Remove("dir1/dir2/..")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ err = fs.Remove("dir1")
+ c.Check(err, check.Equals, ErrDirectoryNotEmpty)
+ err = fs.Remove("dir1/dir2/../../../dir1")
+ c.Check(err, check.Equals, ErrDirectoryNotEmpty)
+ err = fs.Remove("dir1/dir3/")
+ c.Check(err, check.IsNil)
+ err = fs.RemoveAll("dir1")
+ c.Check(err, check.IsNil)
+ err = fs.RemoveAll("dir1")
+ c.Check(err, check.IsNil)
+}
+
+func (s *CollectionFSSuite) TestRenameError(c *check.C) {
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+ err = fs.Mkdir("first", 0755)
+ c.Assert(err, check.IsNil)
+ err = fs.Mkdir("first/second", 0755)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("first/second/file", os.O_CREATE|os.O_WRONLY, 0755)
+ c.Assert(err, check.IsNil)
+ f.Write([]byte{1, 2, 3, 4, 5})
+ f.Close()
+ err = fs.Rename("first", "first/second/third")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ err = fs.Rename("first", "first/third")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ err = fs.Rename("first/second", "second")
+ c.Check(err, check.IsNil)
+ f, err = fs.OpenFile("second/file", 0, 0)
+ c.Assert(err, check.IsNil)
+ data, err := ioutil.ReadAll(f)
+ c.Check(err, check.IsNil)
+ c.Check(data, check.DeepEquals, []byte{1, 2, 3, 4, 5})
+}
+
+func (s *CollectionFSSuite) TestRename(c *check.C) {
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+ const (
+ outer = 16
+ inner = 16
+ )
+ for i := 0; i < outer; i++ {
+ err = fs.Mkdir(fmt.Sprintf("dir%d", i), 0755)
+ c.Assert(err, check.IsNil)
+ for j := 0; j < inner; j++ {
+ err = fs.Mkdir(fmt.Sprintf("dir%d/dir%d", i, j), 0755)
+ c.Assert(err, check.IsNil)
+ for _, fnm := range []string{
+ fmt.Sprintf("dir%d/file%d", i, j),
+ fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
+ } {
+ f, err := fs.OpenFile(fnm, os.O_CREATE|os.O_WRONLY, 0755)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write([]byte("beep"))
+ c.Assert(err, check.IsNil)
+ f.Close()
+ }
+ }
+ }
+ var wg sync.WaitGroup
+ for i := 0; i < outer; i++ {
+ for j := 0; j < inner; j++ {
+ wg.Add(1)
+ go func(i, j int) {
+ defer wg.Done()
+ oldname := fmt.Sprintf("dir%d/dir%d/file%d", i, j, j)
+ newname := fmt.Sprintf("dir%d/newfile%d", i, inner-j-1)
+ _, err := fs.Open(newname)
+ c.Check(err, check.Equals, os.ErrNotExist)
+ err = fs.Rename(oldname, newname)
+ c.Check(err, check.IsNil)
+ f, err := fs.Open(newname)
+ c.Check(err, check.IsNil)
+ f.Close()
+ }(i, j)
+
+ wg.Add(1)
+ go func(i, j int) {
+ defer wg.Done()
+ // oldname does not exist
+ err := fs.Rename(
+ fmt.Sprintf("dir%d/dir%d/missing", i, j),
+ fmt.Sprintf("dir%d/dir%d/file%d", outer-i-1, j, j))
+ c.Check(err, check.ErrorMatches, `.*does not exist`)
+
+ // newname parent dir does not exist
+ err = fs.Rename(
+ fmt.Sprintf("dir%d/dir%d", i, j),
+ fmt.Sprintf("dir%d/missing/irrelevant", outer-i-1))
+ c.Check(err, check.ErrorMatches, `.*does not exist`)
+
+ // oldname parent dir is a file
+ err = fs.Rename(
+ fmt.Sprintf("dir%d/file%d/patherror", i, j),
+ fmt.Sprintf("dir%d/irrelevant", i))
+ c.Check(err, check.ErrorMatches, `.*does not exist`)
+
+ // newname parent dir is a file
+ err = fs.Rename(
+ fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
+ fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
+ c.Check(err, check.ErrorMatches, `.*does not exist`)
+ }(i, j)
+ }
+ }
+ wg.Wait()
+
+ f, err := fs.OpenFile("dir1/newfile3", 0, 0)
+ c.Assert(err, check.IsNil)
+ c.Check(f.Size(), check.Equals, int64(4))
+ buf, err := ioutil.ReadAll(f)
+ c.Check(buf, check.DeepEquals, []byte("beep"))
+ c.Check(err, check.IsNil)
+ _, err = fs.Open("dir1/dir1/file1")
+ c.Check(err, check.Equals, os.ErrNotExist)
+}
+
func (s *CollectionFSSuite) TestPersist(c *check.C) {
maxBlockSize = 1024
defer func() { maxBlockSize = 2 << 26 }()
expect := map[string][]byte{
"0": nil,
- "00": []byte{},
- "one": []byte{1},
+ "00": {},
+ "one": {1},
"dir/0": nil,
- "dir/two": []byte{1, 2},
+ "dir/two": {1, 2},
"dir/zero": nil,
"dir/zerodir/zero": nil,
"zero/zero/zero": nil,
c.Assert(err, check.IsNil)
for name, data := range expect {
- f, err := persisted.Open("bogus-" + name)
+ _, err = persisted.Open("bogus-" + name)
c.Check(err, check.NotNil)
- f, err = persisted.Open(name)
+ f, err := persisted.Open(name)
c.Assert(err, check.IsNil)
if data == nil {
}
}
+func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+
+ f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
+ c.Check(f, check.IsNil)
+ c.Check(err, check.ErrorMatches, `file does not exist`)
+
+ f, err = fs.OpenFile("new", os.O_CREATE|os.O_RDONLY, 0)
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ n, err := f.Write([]byte{1, 2, 3})
+ c.Check(n, check.Equals, 0)
+ c.Check(err, check.ErrorMatches, `read-only file`)
+ n, err = f.Read(make([]byte, 1))
+ c.Check(n, check.Equals, 0)
+ c.Check(err, check.Equals, io.EOF)
+ f, err = fs.OpenFile("new", os.O_RDWR, 0)
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ _, err = f.Write([]byte{4, 5, 6})
+ c.Check(err, check.IsNil)
+ fi, err := f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(fi.Size(), check.Equals, int64(3))
+
+ f, err = fs.OpenFile("new", os.O_TRUNC|os.O_RDWR, 0)
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ pos, err := f.Seek(0, io.SeekEnd)
+ c.Check(pos, check.Equals, int64(0))
+ c.Check(err, check.IsNil)
+ fi, err = f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(fi.Size(), check.Equals, int64(0))
+ fs.Remove("new")
+
+ buf := make([]byte, 64)
+ f, err = fs.OpenFile("append", os.O_EXCL|os.O_CREATE|os.O_RDWR|os.O_APPEND, 0)
+ c.Assert(err, check.IsNil)
+ f.Write([]byte{1, 2, 3})
+ f.Seek(0, io.SeekStart)
+ n, _ = f.Read(buf[:1])
+ c.Check(n, check.Equals, 1)
+ c.Check(buf[:1], check.DeepEquals, []byte{1})
+ pos, err = f.Seek(0, io.SeekCurrent)
+ c.Assert(err, check.IsNil)
+ c.Check(pos, check.Equals, int64(1))
+ f.Write([]byte{4, 5, 6})
+ pos, err = f.Seek(0, io.SeekCurrent)
+ c.Assert(err, check.IsNil)
+ c.Check(pos, check.Equals, int64(6))
+ f.Seek(0, io.SeekStart)
+ n, err = f.Read(buf)
+ c.Check(buf[:n], check.DeepEquals, []byte{1, 2, 3, 4, 5, 6})
+ c.Check(err, check.Equals, io.EOF)
+ f.Close()
+
+ f, err = fs.OpenFile("append", os.O_RDWR|os.O_APPEND, 0)
+ c.Assert(err, check.IsNil)
+ pos, err = f.Seek(0, io.SeekCurrent)
+ c.Check(pos, check.Equals, int64(0))
+ c.Check(err, check.IsNil)
+ f.Read(buf[:3])
+ pos, _ = f.Seek(0, io.SeekCurrent)
+ c.Check(pos, check.Equals, int64(3))
+ f.Write([]byte{7, 8, 9})
+ pos, err = f.Seek(0, io.SeekCurrent)
+ c.Check(err, check.IsNil)
+ c.Check(pos, check.Equals, int64(9))
+ f.Close()
+
+ f, err = fs.OpenFile("wronly", os.O_CREATE|os.O_WRONLY, 0)
+ c.Assert(err, check.IsNil)
+ n, err = f.Write([]byte{3, 2, 1})
+ c.Check(n, check.Equals, 3)
+ c.Check(err, check.IsNil)
+ pos, _ = f.Seek(0, io.SeekCurrent)
+ c.Check(pos, check.Equals, int64(3))
+ pos, _ = f.Seek(0, io.SeekStart)
+ c.Check(pos, check.Equals, int64(0))
+ n, err = f.Read(buf)
+ c.Check(n, check.Equals, 0)
+ c.Check(err, check.ErrorMatches, `.*O_WRONLY.*`)
+ f, err = fs.OpenFile("wronly", os.O_RDONLY, 0)
+ c.Assert(err, check.IsNil)
+ n, _ = f.Read(buf)
+ c.Check(buf[:n], check.DeepEquals, []byte{3, 2, 1})
+
+ f, err = fs.OpenFile("unsupported", os.O_CREATE|os.O_SYNC, 0)
+ c.Check(f, check.IsNil)
+ c.Check(err, check.NotNil)
+
+ f, err = fs.OpenFile("append", os.O_RDWR|os.O_WRONLY, 0)
+ c.Check(f, check.IsNil)
+ c.Check(err, check.ErrorMatches, `invalid flag.*`)
+}
+
func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
maxBlockSize = 1024
defer func() { maxBlockSize = 2 << 26 }()
}
currentMemExtents := func() (memExtents []int) {
- for idx, e := range f.(*file).inode.(*filenode).extents {
+ for idx, e := range f.(*filehandle).inode.(*filenode).segments {
switch e.(type) {
- case *memExtent:
+ case *memSegment:
memExtents = append(memExtents, idx)
}
}
c.Check(currentMemExtents(), check.HasLen, 1)
m, err := fs.MarshalManifest(".")
- c.Check(m, check.Not(check.Equals), "")
+ c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
c.Check(err, check.IsNil)
c.Check(currentMemExtents(), check.HasLen, 0)
}
". 0:0:foo\n",
". 0:0:foo 0:0:bar\n",
". d41d8cd98f00b204e9800998ecf8427e 0:0:foo\n",
- ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo:bar\n",
+ ". d41d8cd98f00b204e9800998ecf8427e+0 :0:0:foo\n",
". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
}
func (s *CollectionFSSuite) checkMemSize(c *check.C, f File) {
- fn := f.(*file).inode.(*filenode)
+ fn := f.(*filehandle).inode.(*filenode)
var memsize int64
- for _, ext := range fn.extents {
- if e, ok := ext.(*memExtent); ok {
+ for _, seg := range fn.segments {
+ if e, ok := seg.(*memSegment); ok {
memsize += int64(len(e.buf))
}
}
c.Check(fn.memsize, check.Equals, memsize)
}
+type CollectionFSUnitSuite struct{}
+
+var _ = check.Suite(&CollectionFSUnitSuite{})
+
+// expect ~2 seconds to load a manifest with 256K files
+func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
+ const (
+ dirCount = 512
+ fileCount = 512
+ )
+
+ mb := bytes.NewBuffer(make([]byte, 0, 40000000))
+ for i := 0; i < dirCount; i++ {
+ fmt.Fprintf(mb, "./dir%d", i)
+ for j := 0; j <= fileCount; j++ {
+ fmt.Fprintf(mb, " %032x+42+A%040x@%08x", j, j, j)
+ }
+ for j := 0; j < fileCount; j++ {
+ fmt.Fprintf(mb, " %d:%d:dir%d/file%d", j*42+21, 42, j, j)
+ }
+ mb.Write([]byte{'\n'})
+ }
+ coll := Collection{ManifestText: mb.String()}
+ c.Logf("%s built", time.Now())
+
+ var memstats runtime.MemStats
+ runtime.ReadMemStats(&memstats)
+ c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
+
+ f, err := coll.FileSystem(nil, nil)
+ c.Check(err, check.IsNil)
+ c.Logf("%s loaded", time.Now())
+
+ for i := 0; i < dirCount; i++ {
+ for j := 0; j < fileCount; j++ {
+ f.Stat(fmt.Sprintf("./dir%d/dir%d/file%d", i, j, j))
+ }
+ }
+ c.Logf("%s Stat() x %d", time.Now(), dirCount*fileCount)
+
+ runtime.ReadMemStats(&memstats)
+ c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
+}
+
// Gocheck boilerplate
func Test(t *testing.T) {
check.TestingT(t)
projects / arvados.git / blobdiff