14259: Add method to keep client to request remote block copy via HEAD request.

[arvados.git] / sdk / go / arvados / fs_collection.go

diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index fbd9775b0cde0876e68328fd63a019485eeac4a9..f6afadba5b47488dfff41a53ebca31a97e1dc940 100644 (file)
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -31,6 +31,9 @@ type CollectionFileSystem interface {
        // Prefix (normally ".") is a top level directory, effectively
        // prepended to all paths in the returned manifest.
        MarshalManifest(prefix string) (string, error)
+
+       // Total data bytes in all files.
+       Size() int64
 }
 
 type collectionFileSystem struct {
@@ -139,6 +142,10 @@ func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
        return fs.fileSystem.root.(*dirnode).marshalManifest(prefix)
 }
 
+func (fs *collectionFileSystem) Size() int64 {
+       return fs.fileSystem.root.(*dirnode).TreeSize()
+}
+
 // filenodePtr is an offset into a file that is (usually) efficient to
 // seek to. Specifically, if filenode.repacked==filenodePtr.repacked
 // then
@@ -521,7 +528,7 @@ func (dn *dirnode) FS() FileSystem {
        return dn.fs
 }
 
-func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
+func (dn *dirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
        if dn == dn.fs.rootnode() && name == ".arvados#collection" {
                gn := &getternode{Getter: func() ([]byte, error) {
                        var coll Collection
@@ -537,14 +544,15 @@ func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
                        return data, err
                }}
                gn.SetParent(dn, name)
-               return gn
+               return gn, nil
        }
        return dn.treenode.Child(name, replace)
 }
 
-// sync flushes in-memory data (for all files in the tree rooted at
-// dn) to persistent storage. Caller must hold dn.Lock().
-func (dn *dirnode) sync() error {
+// sync flushes in-memory data (for the children with the given names,
+// which must be children of dn) to persistent storage. Caller must
+// have write lock on dn and the named children.
+func (dn *dirnode) sync(names []string) error {
        type shortBlock struct {
                fn  *filenode
                idx int
@@ -580,19 +588,11 @@ func (dn *dirnode) sync() error {
                return nil
        }
 
-       names := make([]string, 0, len(dn.inodes))
-       for name := range dn.inodes {
-               names = append(names, name)
-       }
-       sort.Strings(names)
-
        for _, name := range names {
                fn, ok := dn.inodes[name].(*filenode)
                if !ok {
                        continue
                }
-               fn.Lock()
-               defer fn.Unlock()
                for idx, seg := range fn.segments {
                        seg, ok := seg.(*memSegment)
                        if !ok {
@@ -630,18 +630,19 @@ func (dn *dirnode) marshalManifest(prefix string) (string, error) {
        var subdirs string
        var blocks []string
 
-       if err := dn.sync(); err != nil {
-               return "", err
-       }
-
        names := make([]string, 0, len(dn.inodes))
-       for name, node := range dn.inodes {
+       for name := range dn.inodes {
                names = append(names, name)
+       }
+       sort.Strings(names)
+       for _, name := range names {
+               node := dn.inodes[name]
                node.Lock()
                defer node.Unlock()
        }
-       sort.Strings(names)
-
+       if err := dn.sync(names); err != nil {
+               return "", err
+       }
        for _, name := range names {
                switch node := dn.inodes[name].(type) {
                case *dirnode:
@@ -821,8 +822,8 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
        var node inode = dn
        names := strings.Split(path, "/")
        basename := names[len(names)-1]
-       if basename == "" || basename == "." || basename == ".." {
-               err = fmt.Errorf("invalid filename")
+       if !permittedName(basename) {
+               err = fmt.Errorf("invalid file part %q in path %q", basename, path)
                return
        }
        for _, name := range names[:len(names)-1] {
@@ -837,43 +838,60 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
                        node = node.Parent()
                        continue
                }
-               node.Child(name, func(child inode) inode {
+               node, err = node.Child(name, func(child inode) (inode, error) {
                        if child == nil {
-                               child, err = node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+                               child, err := node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+                               if err != nil {
+                                       return nil, err
+                               }
                                child.SetParent(node, name)
-                               node = child
+                               return child, nil
                        } else if !child.IsDir() {
-                               err = ErrFileExists
+                               return child, ErrFileExists
                        } else {
-                               node = child
+                               return child, nil
                        }
-                       return child
                })
                if err != nil {
                        return
                }
        }
-       node.Child(basename, func(child inode) inode {
+       _, err = node.Child(basename, func(child inode) (inode, error) {
                switch child := child.(type) {
                case nil:
                        child, err = node.FS().newNode(basename, 0755, node.FileInfo().ModTime())
+                       if err != nil {
+                               return nil, err
+                       }
                        child.SetParent(node, basename)
                        fn = child.(*filenode)
-                       return child
+                       return child, nil
                case *filenode:
                        fn = child
-                       return child
+                       return child, nil
                case *dirnode:
-                       err = ErrIsDirectory
-                       return child
+                       return child, ErrIsDirectory
                default:
-                       err = ErrInvalidArgument
-                       return child
+                       return child, ErrInvalidArgument
                }
        })
        return
 }
 
+func (dn *dirnode) TreeSize() (bytes int64) {
+       dn.RLock()
+       defer dn.RUnlock()
+       for _, i := range dn.inodes {
+               switch i := i.(type) {
+               case *filenode:
+                       bytes += i.Size()
+               case *dirnode:
+                       bytes += i.TreeSize()
+               }
+       }
+       return
+}
+
 type segment interface {
        io.ReaderAt
        Len() int