X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a0a02d2f4180f908a39293ef4adb00f927b7ad6f..c02ceff00fce94ec5794b53fe890f681acf31121:/sdk/go/arvados/fs_collection.go diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go index 36a92aff7e..fbd9775b0c 100644 --- a/sdk/go/arvados/fs_collection.go +++ b/sdk/go/arvados/fs_collection.go @@ -21,28 +21,6 @@ import ( var maxBlockSize = 1 << 26 -type fsBackend interface { - keepClient - apiClient -} - -// Ideally *Client would do everything; meanwhile keepBackend -// implements fsBackend by merging the two kinds of arvados client. -type keepBackend struct { - keepClient - apiClient -} - -type keepClient interface { - ReadAt(locator string, p []byte, off int) (int, error) - PutB(p []byte) (string, int, error) -} - -type apiClient interface { - RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error - UpdateBody(rsc resource) io.Reader -} - // A CollectionFileSystem is a FileSystem that can be serialized as a // manifest and stored as a collection. type CollectionFileSystem interface { @@ -55,6 +33,11 @@ type CollectionFileSystem interface { MarshalManifest(prefix string) (string, error) } +type collectionFileSystem struct { + fileSystem + uuid string +} + // FileSystem returns a CollectionFileSystem for the collection. func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) { var modTime time.Time @@ -80,50 +63,53 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile inodes: make(map[string]inode), }, } - root.SetParent(root) + root.SetParent(root, ".") if err := root.loadManifest(c.ManifestText); err != nil { return nil, err } + backdateTree(root, modTime) fs.root = root return fs, nil } -type collectionFileSystem struct { - fileSystem - uuid string +func backdateTree(n inode, modTime time.Time) { + switch n := n.(type) { + case *filenode: + n.fileinfo.modTime = modTime + case *dirnode: + n.fileinfo.modTime = modTime + for _, n := range n.inodes { + backdateTree(n, modTime) + } + } } -// Caller must have parent lock, and must have already ensured -// parent.Child(name,nil) is nil. -func (fs *collectionFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) { +func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) { if name == "" || name == "." || name == ".." { return nil, ErrInvalidArgument } - return &dirnode{ - fs: fs, - treenode: treenode{ + if perm.IsDir() { + return &dirnode{ + fs: fs, + treenode: treenode{ + fileinfo: fileinfo{ + name: name, + mode: perm | os.ModeDir, + modTime: modTime, + }, + inodes: make(map[string]inode), + }, + }, nil + } else { + return &filenode{ + fs: fs, fileinfo: fileinfo{ name: name, - mode: perm | os.ModeDir, + mode: perm & ^os.ModeDir, modTime: modTime, }, - inodes: make(map[string]inode), - }, - }, nil -} - -func (fs *collectionFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) { - if name == "" || name == "." || name == ".." { - return nil, ErrInvalidArgument + }, nil } - return &filenode{ - fs: fs, - fileinfo: fileinfo{ - name: name, - mode: perm & ^os.ModeDir, - modTime: modTime, - }, - }, nil } func (fs *collectionFileSystem) Sync() error { @@ -147,27 +133,6 @@ func (fs *collectionFileSystem) Sync() error { return err } -func (dn *dirnode) Child(name string, replace func(inode) inode) inode { - if dn == dn.fs.rootnode() && name == ".arvados#collection" { - gn := &getternode{Getter: func() ([]byte, error) { - var coll Collection - var err error - coll.ManifestText, err = dn.fs.MarshalManifest(".") - if err != nil { - return nil, err - } - data, err := json.Marshal(&coll) - if err == nil { - data = append(data, 10) - } - return data, err - }} - gn.SetParent(dn) - return gn - } - return dn.treenode.Child(name, replace) -} - func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) { fs.fileSystem.root.Lock() defer fs.fileSystem.root.Unlock() @@ -267,10 +232,11 @@ func (fn *filenode) appendSegment(e segment) { fn.fileinfo.size += int64(e.Len()) } -func (fn *filenode) SetParent(p inode) { - fn.RLock() - defer fn.RUnlock() +func (fn *filenode) SetParent(p inode, name string) { + fn.Lock() + defer fn.Unlock() fn.parent = p + fn.fileinfo.name = name } func (fn *filenode) Parent() inode { @@ -528,7 +494,7 @@ func (fn *filenode) pruneMemSegments() { if !ok || seg.Len() < maxBlockSize { continue } - locator, _, err := fn.parent.(fsBackend).PutB(seg.buf) + locator, _, err := fn.FS().PutB(seg.buf) if err != nil { // TODO: stall (or return errors from) // subsequent writes until flushing @@ -537,7 +503,7 @@ func (fn *filenode) pruneMemSegments() { } fn.memsize -= int64(seg.Len()) fn.segments[idx] = storedSegment{ - kc: fn.parent.FS(), + kc: fn.FS(), locator: locator, size: seg.Len(), offset: 0, @@ -555,6 +521,27 @@ func (dn *dirnode) FS() FileSystem { return dn.fs } +func (dn *dirnode) Child(name string, replace func(inode) inode) inode { + if dn == dn.fs.rootnode() && name == ".arvados#collection" { + gn := &getternode{Getter: func() ([]byte, error) { + var coll Collection + var err error + coll.ManifestText, err = dn.fs.MarshalManifest(".") + if err != nil { + return nil, err + } + data, err := json.Marshal(&coll) + if err == nil { + data = append(data, '\n') + } + return data, err + }} + gn.SetParent(dn, name) + return gn + } + return dn.treenode.Child(name, replace) +} + // sync flushes in-memory data (for all files in the tree rooted at // dn) to persistent storage. Caller must hold dn.Lock(). func (dn *dirnode) sync() error { @@ -852,8 +839,9 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) { } node.Child(name, func(child inode) inode { if child == nil { - node, err = node.FS().newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime()) - child = node + child, err = node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime()) + child.SetParent(node, name) + node = child } else if !child.IsDir() { err = ErrFileExists } else { @@ -868,7 +856,8 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) { node.Child(basename, func(child inode) inode { switch child := child.(type) { case nil: - child, err = node.FS().newFilenode(node, basename, 0755, node.FileInfo().ModTime()) + child, err = node.FS().newNode(basename, 0755, node.FileInfo().ModTime()) + child.SetParent(node, basename) fn = child.(*filenode) return child case *filenode: