projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
14199: Merge branch 'master' into 14199-copy-from-remote
[arvados.git]
/
sdk
/
go
/
arvados
/
fs_collection.go
diff --git
a/sdk/go/arvados/fs_collection.go
b/sdk/go/arvados/fs_collection.go
index 0121d2dccce31d745f36fbdf5f2bf7bf0b1463de..f6afadba5b47488dfff41a53ebca31a97e1dc940 100644
(file)
--- a/
sdk/go/arvados/fs_collection.go
+++ b/
sdk/go/arvados/fs_collection.go
@@
-31,6
+31,9
@@
type CollectionFileSystem interface {
// Prefix (normally ".") is a top level directory, effectively
// prepended to all paths in the returned manifest.
MarshalManifest(prefix string) (string, error)
// Prefix (normally ".") is a top level directory, effectively
// prepended to all paths in the returned manifest.
MarshalManifest(prefix string) (string, error)
+
+ // Total data bytes in all files.
+ Size() int64
}
type collectionFileSystem struct {
}
type collectionFileSystem struct {
@@
-67,10
+70,23
@@
func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
if err := root.loadManifest(c.ManifestText); err != nil {
return nil, err
}
if err := root.loadManifest(c.ManifestText); err != nil {
return nil, err
}
+ backdateTree(root, modTime)
fs.root = root
return fs, nil
}
fs.root = root
return fs, nil
}
+func backdateTree(n inode, modTime time.Time) {
+ switch n := n.(type) {
+ case *filenode:
+ n.fileinfo.modTime = modTime
+ case *dirnode:
+ n.fileinfo.modTime = modTime
+ for _, n := range n.inodes {
+ backdateTree(n, modTime)
+ }
+ }
+}
+
func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
if name == "" || name == "." || name == ".." {
return nil, ErrInvalidArgument
func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
if name == "" || name == "." || name == ".." {
return nil, ErrInvalidArgument
@@
-126,6
+142,10
@@
func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
return fs.fileSystem.root.(*dirnode).marshalManifest(prefix)
}
return fs.fileSystem.root.(*dirnode).marshalManifest(prefix)
}
+func (fs *collectionFileSystem) Size() int64 {
+ return fs.fileSystem.root.(*dirnode).TreeSize()
+}
+
// filenodePtr is an offset into a file that is (usually) efficient to
// seek to. Specifically, if filenode.repacked==filenodePtr.repacked
// then
// filenodePtr is an offset into a file that is (usually) efficient to
// seek to. Specifically, if filenode.repacked==filenodePtr.repacked
// then
@@
-508,7
+528,7
@@
func (dn *dirnode) FS() FileSystem {
return dn.fs
}
return dn.fs
}
-func (dn *dirnode) Child(name string, replace func(inode)
inode) inode
{
+func (dn *dirnode) Child(name string, replace func(inode)
(inode, error)) (inode, error)
{
if dn == dn.fs.rootnode() && name == ".arvados#collection" {
gn := &getternode{Getter: func() ([]byte, error) {
var coll Collection
if dn == dn.fs.rootnode() && name == ".arvados#collection" {
gn := &getternode{Getter: func() ([]byte, error) {
var coll Collection
@@
-519,19
+539,20
@@
func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
}
data, err := json.Marshal(&coll)
if err == nil {
}
data, err := json.Marshal(&coll)
if err == nil {
- data = append(data,
10
)
+ data = append(data,
'\n'
)
}
return data, err
}}
gn.SetParent(dn, name)
}
return data, err
}}
gn.SetParent(dn, name)
- return gn
+ return gn
, nil
}
return dn.treenode.Child(name, replace)
}
}
return dn.treenode.Child(name, replace)
}
-// sync flushes in-memory data (for all files in the tree rooted at
-// dn) to persistent storage. Caller must hold dn.Lock().
-func (dn *dirnode) sync() error {
+// sync flushes in-memory data (for the children with the given names,
+// which must be children of dn) to persistent storage. Caller must
+// have write lock on dn and the named children.
+func (dn *dirnode) sync(names []string) error {
type shortBlock struct {
fn *filenode
idx int
type shortBlock struct {
fn *filenode
idx int
@@
-567,19
+588,11
@@
func (dn *dirnode) sync() error {
return nil
}
return nil
}
- names := make([]string, 0, len(dn.inodes))
- for name := range dn.inodes {
- names = append(names, name)
- }
- sort.Strings(names)
-
for _, name := range names {
fn, ok := dn.inodes[name].(*filenode)
if !ok {
continue
}
for _, name := range names {
fn, ok := dn.inodes[name].(*filenode)
if !ok {
continue
}
- fn.Lock()
- defer fn.Unlock()
for idx, seg := range fn.segments {
seg, ok := seg.(*memSegment)
if !ok {
for idx, seg := range fn.segments {
seg, ok := seg.(*memSegment)
if !ok {
@@
-617,18
+630,19
@@
func (dn *dirnode) marshalManifest(prefix string) (string, error) {
var subdirs string
var blocks []string
var subdirs string
var blocks []string
- if err := dn.sync(); err != nil {
- return "", err
- }
-
names := make([]string, 0, len(dn.inodes))
names := make([]string, 0, len(dn.inodes))
- for name
, node
:= range dn.inodes {
+ for name := range dn.inodes {
names = append(names, name)
names = append(names, name)
+ }
+ sort.Strings(names)
+ for _, name := range names {
+ node := dn.inodes[name]
node.Lock()
defer node.Unlock()
}
node.Lock()
defer node.Unlock()
}
- sort.Strings(names)
-
+ if err := dn.sync(names); err != nil {
+ return "", err
+ }
for _, name := range names {
switch node := dn.inodes[name].(type) {
case *dirnode:
for _, name := range names {
switch node := dn.inodes[name].(type) {
case *dirnode:
@@
-808,8
+822,8
@@
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
var node inode = dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
var node inode = dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
- if
basename == "" || basename == "." || basename == ".."
{
- err = fmt.Errorf("invalid file
name"
)
+ if
!permittedName(basename)
{
+ err = fmt.Errorf("invalid file
part %q in path %q", basename, path
)
return
}
for _, name := range names[:len(names)-1] {
return
}
for _, name := range names[:len(names)-1] {
@@
-824,41
+838,60
@@
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
node = node.Parent()
continue
}
node = node.Parent()
continue
}
- node
.Child(name, func(child inode) inode
{
+ node
, err = node.Child(name, func(child inode) (inode, error)
{
if child == nil {
if child == nil {
- node, err = node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
- child = node
+ child, err := node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ if err != nil {
+ return nil, err
+ }
+ child.SetParent(node, name)
+ return child, nil
} else if !child.IsDir() {
} else if !child.IsDir() {
-
err =
ErrFileExists
+
return child,
ErrFileExists
} else {
} else {
- node = child
+ return child, nil
}
}
- return child
})
if err != nil {
return
}
}
})
if err != nil {
return
}
}
-
node.Child(basename, func(child inode) inode
{
+
_, err = node.Child(basename, func(child inode) (inode, error)
{
switch child := child.(type) {
case nil:
child, err = node.FS().newNode(basename, 0755, node.FileInfo().ModTime())
switch child := child.(type) {
case nil:
child, err = node.FS().newNode(basename, 0755, node.FileInfo().ModTime())
+ if err != nil {
+ return nil, err
+ }
+ child.SetParent(node, basename)
fn = child.(*filenode)
fn = child.(*filenode)
- return child
+ return child
, nil
case *filenode:
fn = child
case *filenode:
fn = child
- return child
+ return child
, nil
case *dirnode:
case *dirnode:
- err = ErrIsDirectory
- return child
+ return child, ErrIsDirectory
default:
default:
- err = ErrInvalidArgument
- return child
+ return child, ErrInvalidArgument
}
})
return
}
}
})
return
}
+func (dn *dirnode) TreeSize() (bytes int64) {
+ dn.RLock()
+ defer dn.RUnlock()
+ for _, i := range dn.inodes {
+ switch i := i.(type) {
+ case *filenode:
+ bytes += i.Size()
+ case *dirnode:
+ bytes += i.TreeSize()
+ }
+ }
+ return
+}
+
type segment interface {
io.ReaderAt
Len() int
type segment interface {
io.ReaderAt
Len() int