"errors"
"fmt"
"io"
+ "log"
"net/http"
"os"
"path"
Readdir(int) ([]os.FileInfo, error)
Stat() (os.FileInfo, error)
Truncate(int64) error
+ Sync() error
}
// A FileSystem is an http.Filesystem plus Stat() and support for
// goroutines.
type FileSystem interface {
http.FileSystem
+ fsBackend
- inode
+ newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
+ newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
// analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
Remove(name string) error
RemoveAll(name string) error
Rename(oldname, newname string) error
+ Sync() error
}
type inode interface {
Parent() inode
+ FS() FileSystem
Read([]byte, filenodePtr) (int, filenodePtr, error)
Write([]byte, filenodePtr) (int, filenodePtr, error)
Truncate(int64) error
}
type treenode struct {
+ fs FileSystem
parent inode
inodes map[string]inode
fileinfo fileinfo
nullnode
}
+func (n *treenode) FS() FileSystem {
+ return n.fs
+}
+
func (n *treenode) Parent() inode {
n.RLock()
defer n.RUnlock()
}
type fileSystem struct {
- inode
+ root inode
+ fsBackend
}
// OpenFile is analogous to os.OpenFile().
}
func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- var dn inode = fs.inode
if flag&os.O_SYNC != 0 {
return nil, ErrSyncNotSupported
}
dirname, name := path.Split(name)
- parent := rlookup(dn, dirname)
+ parent := rlookup(fs.root, dirname)
if parent == nil {
return nil, os.ErrNotExist
}
}
var err error
n = parent.Child(name, func(inode) inode {
- var dn *dirnode
- switch parent := parent.(type) {
- case *dirnode:
- dn = parent
- case *collectionFileSystem:
- dn = parent.inode.(*dirnode)
- default:
- err = ErrInvalidArgument
- return nil
- }
if perm.IsDir() {
- n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
+ n, err = fs.newDirnode(parent, name, perm|0755, time.Now())
} else {
- n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
+ n, err = fs.newFilenode(parent, name, perm|0755, time.Now())
}
return n
})
} else if flag&os.O_TRUNC != 0 {
if !writable {
return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
+ } else if n.IsDir() {
return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
+ } else if err := n.Truncate(0); err != nil {
+ return nil, err
}
}
return &filehandle{
func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
dirname, name := path.Split(name)
- n := rlookup(fs.inode, dirname)
+ n := rlookup(fs.root, dirname)
if n == nil {
return os.ErrNotExist
}
if n.Child(name, nil) != nil {
return os.ErrExist
}
- dn, ok := n.(*dirnode)
- if !ok {
- return ErrInvalidArgument
- }
child := n.Child(name, func(inode) (child inode) {
- child, err = dn.newDirnode(dn, name, perm, time.Now())
+ child, err = fs.newDirnode(n, name, perm, time.Now())
return
})
if err != nil {
}
func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := rlookup(fs.inode, name)
+ node := rlookup(fs.root, name)
if node == nil {
err = os.ErrNotExist
} else {
for _, f := range []*filehandle{olddirf, newdirf} {
node := f.inode
needLock = append(needLock, node)
- for node.Parent() != node {
+ for node.Parent() != node && node.Parent().FS() == node.FS() {
node = node.Parent()
needLock = append(needLock, node)
}
}
}
- if _, ok := newdirf.inode.(*dirnode); !ok {
- return ErrInvalidOperation
- }
-
err = nil
olddirf.inode.Child(oldname, func(oldinode inode) inode {
if oldinode == nil {
}
oldinode.Lock()
defer oldinode.Unlock()
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
switch n := oldinode.(type) {
case *dirnode:
n.parent = newdirf.inode
- n.treenode.fileinfo.name = newname
+ n.fileinfo.name = newname
case *filenode:
- n.parent = newdn
+ n.parent = newdirf.inode
n.fileinfo.name = newname
default:
panic(fmt.Sprintf("bad inode type %T", n))
}
- olddn.treenode.fileinfo.modTime = time.Now()
- newdn.treenode.fileinfo.modTime = time.Now()
+ //TODO: olddirf.setModTime(time.Now())
+ //TODO: newdirf.setModTime(time.Now())
return nil
})
return err
if name == "" || name == "." || name == ".." {
return ErrInvalidArgument
}
- dir := rlookup(fs, dirname)
+ dir := rlookup(fs.root, dirname)
if dir == nil {
return os.ErrNotExist
}
})
return err
}
+
+// Caller must have parent lock, and must have already ensured
+// parent.Child(name,nil) is nil.
+func (fs *fileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &dirnode{
+ treenode: treenode{
+ fs: parent.FS(),
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
+}
+
+func (fs *fileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &filenode{
+ fs: parent.FS(),
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm & ^os.ModeDir,
+ modTime: modTime,
+ },
+ }, nil
+}
+
+func (fs *fileSystem) Sync() error {
+ log.Printf("TODO: sync fileSystem")
+ return ErrInvalidOperation
+}
+
+// rlookup (recursive lookup) returns the inode for the file/directory
+// with the given name (which may contain "/" separators). If no such
+// file/directory exists, the returned node is nil.
+func rlookup(start inode, path string) (node inode) {
+ node = start
+ for _, name := range strings.Split(path, "/") {
+ if node == nil {
+ break
+ }
+ if node.IsDir() {
+ if name == "." || name == "" {
+ continue
+ }
+ if name == ".." {
+ node = node.Parent()
+ continue
+ }
+ }
+ node = func() inode {
+ node.RLock()
+ defer node.RUnlock()
+ return node.Child(name, nil)
+ }()
+ }
+ return
+}
"encoding/json"
"fmt"
"io"
+ "log"
"os"
"path"
"regexp"
var maxBlockSize = 1 << 26
+type fsBackend interface {
+ keepClient
+ apiClient
+}
+
+// Ideally *Client would do everything; meanwhile keepBackend
+// implements fsBackend by merging the two kinds of arvados client.
+type keepBackend struct {
+ keepClient
+ apiClient
+}
+
type keepClient interface {
ReadAt(locator string, p []byte, off int) (int, error)
PutB(p []byte) (string, int, error)
}
+type apiClient interface {
+ RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
+ UpdateBody(rsc resource) io.Reader
+}
+
// A CollectionFileSystem is a FileSystem that can be serialized as a
// manifest and stored as a collection.
type CollectionFileSystem interface {
}
// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
+func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) {
var modTime time.Time
if c.ModifiedAt == nil {
modTime = time.Now()
} else {
modTime = *c.ModifiedAt
}
+ fs := &collectionFileSystem{
+ fileSystem: fileSystem{
+ fsBackend: keepBackend{apiClient: client, keepClient: kc},
+ },
+ uuid: c.UUID,
+ }
dn := &dirnode{
- client: client,
- kc: kc,
treenode: treenode{
+ fs: fs,
fileinfo: fileinfo{
name: ".",
mode: os.ModeDir | 0755,
modTime: modTime,
},
- parent: nil,
inodes: make(map[string]inode),
},
}
dn.parent = dn
- fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
+ fs.fileSystem.root = dn
if err := dn.loadManifest(c.ManifestText); err != nil {
return nil, err
}
type collectionFileSystem struct {
fileSystem
+ uuid string
}
-func (fs collectionFileSystem) Child(name string, replace func(inode) inode) inode {
+func (fs *collectionFileSystem) Sync() error {
+ log.Printf("cfs.Sync()")
+ if fs.uuid == "" {
+ return nil
+ }
+ txt, err := fs.MarshalManifest(".")
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ return err
+ }
+ coll := &Collection{
+ UUID: fs.uuid,
+ ManifestText: txt,
+ }
+ err = fs.RequestAndDecode(nil, "PUT", "arvados/v1/collections/"+fs.uuid, fs.UpdateBody(coll), map[string]interface{}{"select": []string{"uuid"}})
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ }
+ return err
+}
+
+func (fs *collectionFileSystem) Child(name string, replace func(inode) inode) inode {
if name == ".arvados#collection" {
return &getternode{Getter: func() ([]byte, error) {
var coll Collection
return data, err
}}
}
- return fs.fileSystem.Child(name, replace)
+ return fs.fileSystem.root.Child(name, replace)
}
-func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
- fs.fileSystem.inode.Lock()
- defer fs.fileSystem.inode.Unlock()
- return fs.fileSystem.inode.(*dirnode).marshalManifest(prefix)
+func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
+ fs.fileSystem.root.Lock()
+ defer fs.fileSystem.root.Unlock()
+ return fs.fileSystem.root.(*dirnode).marshalManifest(prefix)
}
// filenodePtr is an offset into a file that is (usually) efficient to
// filenode implements inode.
type filenode struct {
+ parent inode
+ fs FileSystem
fileinfo fileinfo
- parent *dirnode
segments []segment
// number of times `segments` has changed in a
// way that might invalidate a filenodePtr
return fn.parent
}
+func (fn *filenode) FS() FileSystem {
+ return fn.fs
+}
+
// Read reads file data from a single segment, starting at startPtr,
// into p. startPtr is assumed not to be up-to-date. Caller must have
// RLock or Lock.
if !ok || seg.Len() < maxBlockSize {
continue
}
- locator, _, err := fn.parent.kc.PutB(seg.buf)
+ locator, _, err := fn.parent.(fsBackend).PutB(seg.buf)
if err != nil {
// TODO: stall (or return errors from)
// subsequent writes until flushing
}
fn.memsize -= int64(seg.Len())
fn.segments[idx] = storedSegment{
- kc: fn.parent.kc,
+ kc: fn.parent.(fsBackend),
locator: locator,
size: seg.Len(),
offset: 0,
type dirnode struct {
treenode
- client *Client
- kc keepClient
}
// sync flushes in-memory data (for all files in the tree rooted at
for _, sb := range sbs {
block = append(block, sb.fn.segments[sb.idx].(*memSegment).buf...)
}
- locator, _, err := dn.kc.PutB(block)
+ locator, _, err := dn.fs.PutB(block)
if err != nil {
return err
}
for _, sb := range sbs {
data := sb.fn.segments[sb.idx].(*memSegment).buf
sb.fn.segments[sb.idx] = storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: locator,
size: len(block),
offset: off,
blkLen = int(offset + length - pos - int64(blkOff))
}
fnode.appendSegment(storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: seg.locator,
size: seg.size,
offset: blkOff,
// only safe to call from loadManifest -- no locking
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
- node := dn
+ var node inode = dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
if basename == "" || basename == "." || basename == ".." {
continue
}
node.Child(name, func(child inode) inode {
- switch child.(type) {
- case nil:
- node, err = dn.newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ if child == nil {
+ node, err = node.FS().newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
child = node
- case *dirnode:
- node = child.(*dirnode)
- case *filenode:
+ } else if !child.IsDir() {
err = ErrFileExists
- default:
- err = ErrInvalidOperation
+ } else {
+ node = child
}
return child
})
node.Child(basename, func(child inode) inode {
switch child := child.(type) {
case nil:
- fn, err = dn.newFilenode(node, basename, 0755, node.FileInfo().ModTime())
- return fn
+ child, err = node.FS().newFilenode(node, basename, 0755, node.FileInfo().ModTime())
+ fn = child.(*filenode)
+ return child
case *filenode:
fn = child
return child
return
}
-// rlookup (recursive lookup) returns the inode for the file/directory
-// with the given name (which may contain "/" separators). If no such
-// file/directory exists, the returned node is nil.
-func rlookup(start inode, path string) (node inode) {
- node = start
- for _, name := range strings.Split(path, "/") {
- if node == nil {
- break
- }
- if node.IsDir() {
- if name == "." || name == "" {
- continue
- }
- if name == ".." {
- node = node.Parent()
- continue
- }
- }
- node = func() inode {
- node.RLock()
- defer node.RUnlock()
- return node.Child(name, nil)
- }()
- }
- return
-}
-
-// Caller must have lock, and must have already ensured
-// Children(name,nil) is nil.
-func (dn *dirnode) newDirnode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *dirnode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &dirnode{
- client: dn.client,
- kc: dn.kc,
- treenode: treenode{
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm | os.ModeDir,
- modTime: modTime,
- },
- inodes: make(map[string]inode),
- },
- }, nil
-}
-
-func (dn *dirnode) newFilenode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *filenode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &filenode{
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
-}
-
type segment interface {
io.ReaderAt
Len() int
}
type storedSegment struct {
- kc keepClient
+ kc fsBackend
locator string
size int // size of stored block (also encoded in locator)
offset int // position of segment within the stored block
func (f *filehandle) Close() error {
return nil
}
+
+func (f *filehandle) Sync() error {
+ // Sync the containing filesystem.
+ return f.FS().Sync()
+}
// collections, these changes are not persistent or visible to other
// Arvados clients.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+ fs := &fileSystem{
+ fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ }
root := &treenode{
+ fs: fs,
fileinfo: fileinfo{
name: "/",
mode: os.ModeDir | 0755,
}
root.parent = root
root.Child("by_id", func(inode) inode {
- return &vdirnode{
+ var vn inode
+ vn = &vdirnode{
treenode: treenode{
+ fs: fs,
parent: root,
inodes: make(map[string]inode),
fileinfo: fileinfo{
},
},
create: func(name string) inode {
- return newEntByID(c, kc, name)
+ return newEntByID(vn, name)
},
}
+ return vn
})
- return &fileSystem{inode: root}
+ fs.root = root
+ return fs
}
-func newEntByID(c *Client, kc keepClient, id string) inode {
+func newEntByID(parent inode, id string) inode {
var coll Collection
- err := c.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ err := parent.FS().RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
return nil
}
- fs, err := coll.FileSystem(c, kc)
- fs.(*collectionFileSystem).inode.(*dirnode).fileinfo.name = id
+ fs, err := coll.FileSystem(parent.FS(), parent.FS())
if err != nil {
return nil
}
- return fs
+ root := fs.(*collectionFileSystem).root.(*dirnode)
+ root.fileinfo.name = id
+ root.parent = parent
+ return root
}
type vdirnode struct {
projects / arvados.git / commitdiff