Merge branch '19088-s3-properties-tags'
[arvados.git] / sdk / go / arvados / fs_collection.go
index d39805f3f3a773271825c68737645a0dcc7887fc..26012e240603d0be43a1019346c4e946e2821790 100644 (file)
@@ -43,10 +43,18 @@ type CollectionFileSystem interface {
 
 type collectionFileSystem struct {
        fileSystem
-       uuid              string
-       savedPDH          atomic.Value
-       replicas          int
-       storageClasses    []string
+       uuid           string
+       savedPDH       atomic.Value
+       replicas       int
+       storageClasses []string
+       // guessSignatureTTL tracks a lower bound for the server's
+       // configured BlobSigningTTL. The guess is initially zero, and
+       // increases when we come across a signature with an expiry
+       // time further in the future than the previous guess.
+       //
+       // When the guessed TTL is much smaller than the real TTL,
+       // preemptive signature refresh is delayed or missed entirely,
+       // which is OK.
        guessSignatureTTL time.Duration
        holdCheckChanges  time.Time
        lockCheckChanges  sync.Mutex
@@ -77,6 +85,7 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
                                name:    ".",
                                mode:    os.ModeDir | 0755,
                                modTime: modTime,
+                               sys:     func() interface{} { return c },
                        },
                        inodes: make(map[string]inode),
                },
@@ -156,8 +165,13 @@ func (fs *collectionFileSystem) signatureTimeLeft() (float64, time.Duration) {
        fs.fileSystem.root.Lock()
        {
                if ttl > fs.guessSignatureTTL {
+                       // ttl is closer to the real TTL than
+                       // guessSignatureTTL.
                        fs.guessSignatureTTL = ttl
                } else {
+                       // Use the previous best guess to compute the
+                       // portion remaining (below, after unlocking
+                       // mutex).
                        ttl = fs.guessSignatureTTL
                }
        }
@@ -444,6 +458,14 @@ func (fs *collectionFileSystem) Size() int64 {
        return fs.fileSystem.root.(*dirnode).TreeSize()
 }
 
+func (fs *collectionFileSystem) Snapshot() (inode, error) {
+       return fs.fileSystem.root.Snapshot()
+}
+
+func (fs *collectionFileSystem) Splice(r inode) error {
+       return fs.fileSystem.root.Splice(r)
+}
+
 // filenodePtr is an offset into a file that is (usually) efficient to
 // seek to. Specifically, if filenode.repacked==filenodePtr.repacked
 // then
@@ -863,6 +885,47 @@ func (fn *filenode) waitPrune() {
        }
 }
 
+func (fn *filenode) Snapshot() (inode, error) {
+       fn.RLock()
+       defer fn.RUnlock()
+       segments := make([]segment, 0, len(fn.segments))
+       for _, seg := range fn.segments {
+               segments = append(segments, seg.Slice(0, seg.Len()))
+       }
+       return &filenode{
+               fileinfo: fn.fileinfo,
+               segments: segments,
+       }, nil
+}
+
+func (fn *filenode) Splice(repl inode) error {
+       repl, err := repl.Snapshot()
+       if err != nil {
+               return err
+       }
+       fn.parent.Lock()
+       defer fn.parent.Unlock()
+       fn.Lock()
+       defer fn.Unlock()
+       _, err = fn.parent.Child(fn.fileinfo.name, func(inode) (inode, error) { return repl, nil })
+       if err != nil {
+               return err
+       }
+       switch repl := repl.(type) {
+       case *dirnode:
+               repl.parent = fn.parent
+               repl.fileinfo.name = fn.fileinfo.name
+               repl.setTreeFS(fn.fs)
+       case *filenode:
+               repl.parent = fn.parent
+               repl.fileinfo.name = fn.fileinfo.name
+               repl.fs = fn.fs
+       default:
+               return fmt.Errorf("cannot splice snapshot containing %T: %w", repl, ErrInvalidArgument)
+       }
+       return nil
+}
+
 type dirnode struct {
        fs *collectionFileSystem
        treenode
@@ -1097,15 +1160,17 @@ func (dn *dirnode) MemorySize() (size int64) {
                case *dirnode:
                        size += node.MemorySize()
                case *filenode:
+                       size += 64
                        for _, seg := range node.segments {
                                switch seg := seg.(type) {
                                case *memSegment:
                                        size += int64(seg.Len())
                                }
+                               size += 64
                        }
                }
        }
-       return
+       return 64 + size
 }
 
 // caller must have write lock.
@@ -1476,6 +1541,86 @@ func (dn *dirnode) TreeSize() (bytes int64) {
        return
 }
 
+func (dn *dirnode) Snapshot() (inode, error) {
+       return dn.snapshot()
+}
+
+func (dn *dirnode) snapshot() (*dirnode, error) {
+       dn.RLock()
+       defer dn.RUnlock()
+       snap := &dirnode{
+               treenode: treenode{
+                       inodes:   make(map[string]inode, len(dn.inodes)),
+                       fileinfo: dn.fileinfo,
+               },
+       }
+       for name, child := range dn.inodes {
+               dupchild, err := child.Snapshot()
+               if err != nil {
+                       return nil, err
+               }
+               snap.inodes[name] = dupchild
+               dupchild.SetParent(snap, name)
+       }
+       return snap, nil
+}
+
+func (dn *dirnode) Splice(repl inode) error {
+       repl, err := repl.Snapshot()
+       if err != nil {
+               return fmt.Errorf("cannot copy snapshot: %w", err)
+       }
+       switch repl := repl.(type) {
+       default:
+               return fmt.Errorf("cannot splice snapshot containing %T: %w", repl, ErrInvalidArgument)
+       case *dirnode:
+               dn.Lock()
+               defer dn.Unlock()
+               dn.inodes = repl.inodes
+               dn.setTreeFS(dn.fs)
+       case *filenode:
+               dn.parent.Lock()
+               defer dn.parent.Unlock()
+               removing, err := dn.parent.Child(dn.fileinfo.name, nil)
+               if err != nil {
+                       return fmt.Errorf("cannot use Splice to replace a top-level directory with a file: %w", ErrInvalidOperation)
+               } else if removing != dn {
+                       // If ../thisdirname is not this dirnode, it
+                       // must be an inode that wraps a dirnode, like
+                       // a collectionFileSystem or deferrednode.
+                       if deferred, ok := removing.(*deferrednode); ok {
+                               // More useful to report the type of
+                               // the wrapped node rather than just
+                               // *deferrednode. (We know the real
+                               // inode is already loaded because dn
+                               // is inside it.)
+                               removing = deferred.realinode()
+                       }
+                       return fmt.Errorf("cannot use Splice to attach a file at top level of %T: %w", removing, ErrInvalidOperation)
+               }
+               dn.Lock()
+               defer dn.Unlock()
+               _, err = dn.parent.Child(dn.fileinfo.name, func(inode) (inode, error) { return repl, nil })
+               if err != nil {
+                       return fmt.Errorf("error replacing filenode: dn.parent.Child(): %w", err)
+               }
+               repl.fs = dn.fs
+       }
+       return nil
+}
+
+func (dn *dirnode) setTreeFS(fs *collectionFileSystem) {
+       dn.fs = fs
+       for _, child := range dn.inodes {
+               switch child := child.(type) {
+               case *dirnode:
+                       child.setTreeFS(fs)
+               case *filenode:
+                       child.fs = fs
+               }
+       }
+}
+
 type segment interface {
        io.ReaderAt
        Len() int