Merge branch '18600-copy-by-ref'
authorTom Clegg <tom@curii.com>
Thu, 17 Feb 2022 17:52:27 +0000 (12:52 -0500)
committerTom Clegg <tom@curii.com>
Thu, 17 Feb 2022 17:52:27 +0000 (12:52 -0500)
refs #18600

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

13 files changed:
lib/crunchrun/crunchrun_test.go
lib/mount/fs.go
sdk/go/arvados/fs_base.go
sdk/go/arvados/fs_collection.go
sdk/go/arvados/fs_deferred.go
sdk/go/arvados/fs_filehandle.go
sdk/go/arvados/fs_getternode.go
sdk/go/arvados/fs_lookup.go
sdk/go/arvados/fs_project_test.go
sdk/go/arvados/fs_site.go
sdk/go/arvados/fs_site_test.go
services/keep-web/s3.go
services/keep-web/s3_test.go

index 806a83eef9aa789beeefaf5849051281428e8d81..26f78d2bf7d537c7a44f0b8dd57eda4355211b5b 100644 (file)
@@ -375,6 +375,14 @@ func (fw FileWrapper) Sync() error {
        return errors.New("not implemented")
 }
 
+func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
+       return nil, errors.New("not implemented")
+}
+
+func (fw FileWrapper) Splice(*arvados.Subtree) error {
+       return errors.New("not implemented")
+}
+
 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
        if filename == hwImageID+".tar" {
                rdr := ioutil.NopCloser(&bytes.Buffer{})
index c008b96af664ffa605592a1f61e14e30eb4677d3..3c2e628d0115e361f58150f589060ee14bc57f1b 100644 (file)
@@ -5,6 +5,7 @@
 package mount
 
 import (
+       "errors"
        "io"
        "log"
        "os"
@@ -121,23 +122,25 @@ func (fs *keepFS) Utimens(path string, tmsp []fuse.Timespec) int {
 }
 
 func (fs *keepFS) errCode(err error) int {
-       if os.IsNotExist(err) {
+       if err == nil {
+               return 0
+       }
+       if errors.Is(err, os.ErrNotExist) {
                return -fuse.ENOENT
        }
-       switch err {
-       case os.ErrExist:
+       if errors.Is(err, os.ErrExist) {
                return -fuse.EEXIST
-       case arvados.ErrInvalidArgument:
+       }
+       if errors.Is(err, arvados.ErrInvalidArgument) {
                return -fuse.EINVAL
-       case arvados.ErrInvalidOperation:
+       }
+       if errors.Is(err, arvados.ErrInvalidOperation) {
                return -fuse.ENOSYS
-       case arvados.ErrDirectoryNotEmpty:
+       }
+       if errors.Is(err, arvados.ErrDirectoryNotEmpty) {
                return -fuse.ENOTEMPTY
-       case nil:
-               return 0
-       default:
-               return -fuse.EIO
        }
+       return -fuse.EIO
 }
 
 func (fs *keepFS) Mkdir(path string, mode uint32) int {
index 5f2747ac9ada9225455331daee0f10640202907f..80b803729349dd500de0e7832288f593c41ab60c 100644 (file)
@@ -77,6 +77,24 @@ type File interface {
        Stat() (os.FileInfo, error)
        Truncate(int64) error
        Sync() error
+       // Create a snapshot of a file or directory tree, which can
+       // then be spliced onto a different path or a different
+       // collection.
+       Snapshot() (*Subtree, error)
+       // Replace this file or directory with the given snapshot.
+       // The target must be inside a collection: Splice returns an
+       // error if the File is a virtual file or directory like
+       // by_id, a project directory, .arvados#collection,
+       // etc. Splice can replace directories with regular files and
+       // vice versa, except it cannot replace the root directory of
+       // a collection with a regular file.
+       Splice(snapshot *Subtree) error
+}
+
+// A Subtree is a detached part of a filesystem tree that can be
+// spliced into a filesystem via (File)Splice().
+type Subtree struct {
+       inode inode
 }
 
 // A FileSystem is an http.Filesystem plus Stat() and support for
@@ -152,6 +170,12 @@ type inode interface {
        Readdir() ([]os.FileInfo, error)
        Size() int64
        FileInfo() os.FileInfo
+       // Create a snapshot of this node and its descendants.
+       Snapshot() (inode, error)
+       // Replace this node with a copy of the provided snapshot.
+       // Caller may provide the same snapshot to multiple Splice
+       // calls, but must not modify the snapshot concurrently.
+       Splice(inode) error
 
        // Child() performs lookups and updates of named child nodes.
        //
@@ -270,6 +294,14 @@ func (*nullnode) MemorySize() int64 {
        return 64
 }
 
+func (*nullnode) Snapshot() (inode, error) {
+       return nil, ErrInvalidOperation
+}
+
+func (*nullnode) Splice(inode) error {
+       return ErrInvalidOperation
+}
+
 type treenode struct {
        fs       FileSystem
        parent   inode
@@ -559,7 +591,7 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
                // supported. Locking inodes from different
                // filesystems could deadlock, so we must error out
                // now.
-               return ErrInvalidArgument
+               return ErrInvalidOperation
        }
 
        // To ensure we can test reliably whether we're about to move
@@ -697,3 +729,32 @@ func rlookup(start inode, path string) (node inode, err error) {
 func permittedName(name string) bool {
        return name != "" && name != "." && name != ".." && !strings.Contains(name, "/")
 }
+
+// Snapshot returns a Subtree that's a copy of the given path. It
+// returns an error if the path is not inside a collection.
+func Snapshot(fs FileSystem, path string) (*Subtree, error) {
+       f, err := fs.OpenFile(path, os.O_RDONLY, 0)
+       if err != nil {
+               return nil, err
+       }
+       defer f.Close()
+       return f.Snapshot()
+}
+
+// Splice inserts newsubtree at the indicated target path.
+//
+// Splice returns an error if target is not inside a collection.
+//
+// Splice returns an error if target is the root of a collection and
+// newsubtree is a snapshot of a file.
+func Splice(fs FileSystem, target string, newsubtree *Subtree) error {
+       f, err := fs.OpenFile(target, os.O_WRONLY, 0)
+       if os.IsNotExist(err) {
+               f, err = fs.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0700)
+       }
+       if err != nil {
+               return err
+       }
+       defer f.Close()
+       return f.Splice(newsubtree)
+}
index d087fd09441bc3938411d803de007541f506ba9f..0c5819721e0cb4e8762d6c1e1e3b70427a46a7da 100644 (file)
@@ -457,6 +457,14 @@ func (fs *collectionFileSystem) Size() int64 {
        return fs.fileSystem.root.(*dirnode).TreeSize()
 }
 
+func (fs *collectionFileSystem) Snapshot() (inode, error) {
+       return fs.fileSystem.root.Snapshot()
+}
+
+func (fs *collectionFileSystem) Splice(r inode) error {
+       return fs.fileSystem.root.Splice(r)
+}
+
 // filenodePtr is an offset into a file that is (usually) efficient to
 // seek to. Specifically, if filenode.repacked==filenodePtr.repacked
 // then
@@ -876,6 +884,47 @@ func (fn *filenode) waitPrune() {
        }
 }
 
+func (fn *filenode) Snapshot() (inode, error) {
+       fn.RLock()
+       defer fn.RUnlock()
+       segments := make([]segment, 0, len(fn.segments))
+       for _, seg := range fn.segments {
+               segments = append(segments, seg.Slice(0, seg.Len()))
+       }
+       return &filenode{
+               fileinfo: fn.fileinfo,
+               segments: segments,
+       }, nil
+}
+
+func (fn *filenode) Splice(repl inode) error {
+       repl, err := repl.Snapshot()
+       if err != nil {
+               return err
+       }
+       fn.parent.Lock()
+       defer fn.parent.Unlock()
+       fn.Lock()
+       defer fn.Unlock()
+       _, err = fn.parent.Child(fn.fileinfo.name, func(inode) (inode, error) { return repl, nil })
+       if err != nil {
+               return err
+       }
+       switch repl := repl.(type) {
+       case *dirnode:
+               repl.parent = fn.parent
+               repl.fileinfo.name = fn.fileinfo.name
+               repl.setTreeFS(fn.fs)
+       case *filenode:
+               repl.parent = fn.parent
+               repl.fileinfo.name = fn.fileinfo.name
+               repl.fs = fn.fs
+       default:
+               return fmt.Errorf("cannot splice snapshot containing %T: %w", repl, ErrInvalidArgument)
+       }
+       return nil
+}
+
 type dirnode struct {
        fs *collectionFileSystem
        treenode
@@ -1489,6 +1538,86 @@ func (dn *dirnode) TreeSize() (bytes int64) {
        return
 }
 
+func (dn *dirnode) Snapshot() (inode, error) {
+       return dn.snapshot()
+}
+
+func (dn *dirnode) snapshot() (*dirnode, error) {
+       dn.RLock()
+       defer dn.RUnlock()
+       snap := &dirnode{
+               treenode: treenode{
+                       inodes:   make(map[string]inode, len(dn.inodes)),
+                       fileinfo: dn.fileinfo,
+               },
+       }
+       for name, child := range dn.inodes {
+               dupchild, err := child.Snapshot()
+               if err != nil {
+                       return nil, err
+               }
+               snap.inodes[name] = dupchild
+               dupchild.SetParent(snap, name)
+       }
+       return snap, nil
+}
+
+func (dn *dirnode) Splice(repl inode) error {
+       repl, err := repl.Snapshot()
+       if err != nil {
+               return err
+       }
+       switch repl := repl.(type) {
+       default:
+               return fmt.Errorf("cannot splice snapshot containing %T: %w", repl, ErrInvalidArgument)
+       case *dirnode:
+               dn.Lock()
+               defer dn.Unlock()
+               dn.inodes = repl.inodes
+               dn.setTreeFS(dn.fs)
+       case *filenode:
+               dn.parent.Lock()
+               defer dn.parent.Unlock()
+               removing, err := dn.parent.Child(dn.fileinfo.name, nil)
+               if err != nil {
+                       return fmt.Errorf("cannot use Splice to replace a top-level directory with a file: %w", ErrInvalidOperation)
+               } else if removing != dn {
+                       // If ../thisdirname is not this dirnode, it
+                       // must be an inode that wraps a dirnode, like
+                       // a collectionFileSystem or deferrednode.
+                       if deferred, ok := removing.(*deferrednode); ok {
+                               // More useful to report the type of
+                               // the wrapped node rather than just
+                               // *deferrednode. (We know the real
+                               // inode is already loaded because dn
+                               // is inside it.)
+                               removing = deferred.realinode()
+                       }
+                       return fmt.Errorf("cannot use Splice to attach a file at top level of %T: %w", removing, ErrInvalidOperation)
+               }
+               dn.Lock()
+               defer dn.Unlock()
+               _, err = dn.parent.Child(dn.fileinfo.name, func(inode) (inode, error) { return repl, nil })
+               if err != nil {
+                       return err
+               }
+               repl.fs = dn.fs
+       }
+       return nil
+}
+
+func (dn *dirnode) setTreeFS(fs *collectionFileSystem) {
+       dn.fs = fs
+       for _, child := range dn.inodes {
+               switch child := child.(type) {
+               case *dirnode:
+                       child.setTreeFS(fs)
+               case *filenode:
+                       child.fs = fs
+               }
+       }
+}
+
 type segment interface {
        io.ReaderAt
        Len() int
index bb6c7a26263e23b1cdb733d6d1d00a9ee129175a..66a126a39c12a45620a93c59a9047ac3d4ae1fe8 100644 (file)
@@ -113,3 +113,5 @@ func (dn *deferrednode) RUnlock()                        { dn.realinode().RUnloc
 func (dn *deferrednode) FS() FileSystem                  { return dn.currentinode().FS() }
 func (dn *deferrednode) Parent() inode                   { return dn.currentinode().Parent() }
 func (dn *deferrednode) MemorySize() int64               { return dn.currentinode().MemorySize() }
+func (dn *deferrednode) Snapshot() (inode, error)        { return dn.realinode().Snapshot() }
+func (dn *deferrednode) Splice(repl inode) error         { return dn.realinode().Splice(repl) }
index 9af8d0ad405828b0c8e9906575cb1b77752eaa63..4530a7b06a4d58231f2b4d95287ee792c977431a 100644 (file)
@@ -110,3 +110,18 @@ func (f *filehandle) Sync() error {
        // Sync the containing filesystem.
        return f.FS().Sync()
 }
+
+func (f *filehandle) Snapshot() (*Subtree, error) {
+       if !f.readable {
+               return nil, ErrInvalidOperation
+       }
+       node, err := f.inode.Snapshot()
+       return &Subtree{inode: node}, err
+}
+
+func (f *filehandle) Splice(r *Subtree) error {
+       if !f.writable {
+               return ErrReadOnlyFile
+       }
+       return f.inode.Splice(r.inode)
+}
index 966fe9d5c04187829ebab9f065ea765355386fff..ddff0c52e709b50ba00db31225c5bd549866ea9c 100644 (file)
@@ -24,7 +24,7 @@ func (*getternode) IsDir() bool {
 }
 
 func (*getternode) Child(string, func(inode) (inode, error)) (inode, error) {
-       return nil, ErrInvalidArgument
+       return nil, ErrInvalidOperation
 }
 
 func (gn *getternode) get() error {
index 021e8241cfeda647334781499ac60396e0961ac1..471dc69c82f75318d290eaccfa7b8d4a542540f1 100644 (file)
@@ -68,7 +68,7 @@ func (ln *lookupnode) Readdir() ([]os.FileInfo, error) {
        return ln.treenode.Readdir()
 }
 
-// Child rejects (with ErrInvalidArgument) calls to add/replace
+// Child rejects (with ErrInvalidOperation) calls to add/replace
 // children, instead calling loadOne when a non-existing child is
 // looked up.
 func (ln *lookupnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
@@ -97,12 +97,12 @@ func (ln *lookupnode) Child(name string, replace func(inode) (inode, error)) (in
        if replace != nil {
                // Let the callback try to delete or replace the
                // existing node; if it does, return
-               // ErrInvalidArgument.
+               // ErrInvalidOperation.
                if tryRepl, err := replace(existing); err != nil {
                        // Propagate error from callback
                        return existing, err
                } else if tryRepl != existing {
-                       return existing, ErrInvalidArgument
+                       return existing, ErrInvalidOperation
                }
        }
        // Return original error from ln.treenode.Child() (it might be
index 89435132713b0a14a8438d1ebe07390beb4a8893..8e7f58815629478f243396472fb92b258fe3ac98 100644 (file)
@@ -7,6 +7,7 @@ package arvados
 import (
        "bytes"
        "encoding/json"
+       "errors"
        "io"
        "os"
        "path/filepath"
@@ -311,17 +312,37 @@ func (s *SiteFSSuite) TestProjectUnsupportedOperations(c *check.C) {
        s.fs.MountProject("home", "")
 
        _, err := s.fs.OpenFile("/home/A Project/newfilename", os.O_CREATE|os.O_RDWR, 0)
-       c.Check(err, check.ErrorMatches, "invalid argument")
+       c.Check(err, ErrorIs, ErrInvalidOperation)
 
        err = s.fs.Mkdir("/home/A Project/newdirname", 0)
-       c.Check(err, check.ErrorMatches, "invalid argument")
+       c.Check(err, ErrorIs, ErrInvalidOperation)
 
        err = s.fs.Mkdir("/by_id/newdirname", 0)
-       c.Check(err, check.ErrorMatches, "invalid argument")
+       c.Check(err, ErrorIs, ErrInvalidOperation)
 
        err = s.fs.Mkdir("/by_id/"+fixtureAProjectUUID+"/newdirname", 0)
-       c.Check(err, check.ErrorMatches, "invalid argument")
+       c.Check(err, ErrorIs, ErrInvalidOperation)
 
        _, err = s.fs.OpenFile("/home/A Project", 0, 0)
        c.Check(err, check.IsNil)
 }
+
+type errorIsChecker struct {
+       *check.CheckerInfo
+}
+
+var ErrorIs check.Checker = errorIsChecker{
+       &check.CheckerInfo{Name: "ErrorIs", Params: []string{"value", "target"}},
+}
+
+func (checker errorIsChecker) Check(params []interface{}, names []string) (result bool, errStr string) {
+       err, ok := params[0].(error)
+       if !ok {
+               return false, ""
+       }
+       target, ok := params[1].(error)
+       if !ok {
+               return false, ""
+       }
+       return errors.Is(err, target), ""
+}
index 5225df59ee58ec4c2017054f59ca68ee7936e41e..3892be1e9a97610522a1fc219d5c0fb807788e4c 100644 (file)
@@ -133,7 +133,7 @@ func (fs *customFileSystem) Stale(t time.Time) bool {
 }
 
 func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
-       return nil, ErrInvalidArgument
+       return nil, ErrInvalidOperation
 }
 
 func (fs *customFileSystem) mountByID(parent inode, id string) inode {
@@ -179,7 +179,7 @@ func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode
        }
 }
 
-// vdirnode wraps an inode by rejecting (with ErrInvalidArgument)
+// vdirnode wraps an inode by rejecting (with ErrInvalidOperation)
 // calls that add/replace children directly, instead calling a
 // create() func when a non-existing child is looked up.
 //
@@ -204,7 +204,7 @@ func (vn *vdirnode) Child(name string, replace func(inode) (inode, error)) (inod
                } else if tryRepl, err := replace(existing); err != nil {
                        return existing, err
                } else if tryRepl != existing {
-                       return existing, ErrInvalidArgument
+                       return existing, ErrInvalidOperation
                } else {
                        return existing, nil
                }
index 51ca88764e6625dd25428a526f325e6acf138af8..59fa5fc17616f692e29a7565972c63b4cbfb88cc 100644 (file)
@@ -5,8 +5,12 @@
 package arvados
 
 import (
+       "fmt"
+       "io"
+       "io/ioutil"
        "net/http"
        "os"
+       "syscall"
        "time"
 
        check "gopkg.in/check.v1"
@@ -131,16 +135,236 @@ func (s *SiteFSSuite) TestByUUIDAndPDH(c *check.C) {
        c.Check(names, check.DeepEquals, []string{"baz"})
 
        _, err = s.fs.OpenFile("/by_id/"+fixtureNonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
-       c.Check(err, check.Equals, ErrInvalidArgument)
+       c.Check(err, ErrorIs, ErrInvalidOperation)
        err = s.fs.Rename("/by_id/"+fixtureFooCollection, "/by_id/beep")
-       c.Check(err, check.Equals, ErrInvalidArgument)
+       c.Check(err, ErrorIs, ErrInvalidOperation)
        err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/beep")
-       c.Check(err, check.Equals, ErrInvalidArgument)
+       c.Check(err, ErrorIs, ErrInvalidOperation)
        _, err = s.fs.Stat("/by_id/beep")
        c.Check(err, check.Equals, os.ErrNotExist)
        err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/"+fixtureFooCollection+"/bar")
        c.Check(err, check.IsNil)
 
        err = s.fs.Rename("/by_id", "/beep")
-       c.Check(err, check.Equals, ErrInvalidArgument)
+       c.Check(err, ErrorIs, ErrInvalidOperation)
+}
+
+// Copy subtree from OS src to dst path inside fs. If src is a
+// directory, dst must exist and be a directory.
+func copyFromOS(fs FileSystem, dst, src string) error {
+       inf, err := os.Open(src)
+       if err != nil {
+               return err
+       }
+       defer inf.Close()
+       dirents, err := inf.Readdir(-1)
+       if e, ok := err.(*os.PathError); ok {
+               if e, ok := e.Err.(syscall.Errno); ok {
+                       if e == syscall.ENOTDIR {
+                               err = syscall.ENOTDIR
+                       }
+               }
+       }
+       if err == syscall.ENOTDIR {
+               outf, err := fs.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_TRUNC|os.O_WRONLY, 0700)
+               if err != nil {
+                       return fmt.Errorf("open %s: %s", dst, err)
+               }
+               defer outf.Close()
+               _, err = io.Copy(outf, inf)
+               if err != nil {
+                       return fmt.Errorf("%s: copying data from %s: %s", dst, src, err)
+               }
+               err = outf.Close()
+               if err != nil {
+                       return err
+               }
+       } else if err != nil {
+               return fmt.Errorf("%s: readdir: %T %s", src, err, err)
+       } else {
+               {
+                       d, err := fs.Open(dst)
+                       if err != nil {
+                               return fmt.Errorf("opendir(%s): %s", dst, err)
+                       }
+                       d.Close()
+               }
+               for _, ent := range dirents {
+                       if ent.Name() == "." || ent.Name() == ".." {
+                               continue
+                       }
+                       dstname := dst + "/" + ent.Name()
+                       if ent.IsDir() {
+                               err = fs.Mkdir(dstname, 0700)
+                               if err != nil {
+                                       return fmt.Errorf("mkdir %s: %s", dstname, err)
+                               }
+                       }
+                       err = copyFromOS(fs, dstname, src+"/"+ent.Name())
+                       if err != nil {
+                               return err
+                       }
+               }
+       }
+       return nil
+}
+
+func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
+       s.fs.MountProject("home", "")
+       thisfile, err := ioutil.ReadFile("fs_site_test.go")
+       c.Assert(err, check.IsNil)
+
+       var src1 Collection
+       err = s.client.RequestAndDecode(&src1, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+               "collection": map[string]string{
+                       "name":       "TestSnapshotSplice src1",
+                       "owner_uuid": fixtureAProjectUUID,
+               },
+       })
+       c.Assert(err, check.IsNil)
+       defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src1.UUID, nil, nil)
+       err = s.fs.Sync()
+       c.Assert(err, check.IsNil)
+       err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src1", "..") // arvados.git/sdk/go
+       c.Assert(err, check.IsNil)
+
+       var src2 Collection
+       err = s.client.RequestAndDecode(&src2, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+               "collection": map[string]string{
+                       "name":       "TestSnapshotSplice src2",
+                       "owner_uuid": fixtureAProjectUUID,
+               },
+       })
+       c.Assert(err, check.IsNil)
+       defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src2.UUID, nil, nil)
+       err = s.fs.Sync()
+       c.Assert(err, check.IsNil)
+       err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src2", "..") // arvados.git/sdk/go
+       c.Assert(err, check.IsNil)
+
+       var dst Collection
+       err = s.client.RequestAndDecode(&dst, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+               "collection": map[string]string{
+                       "name":       "TestSnapshotSplice dst",
+                       "owner_uuid": fixtureAProjectUUID,
+               },
+       })
+       c.Assert(err, check.IsNil)
+       defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+dst.UUID, nil, nil)
+       err = s.fs.Sync()
+       c.Assert(err, check.IsNil)
+
+       dstPath := "/home/A Project/TestSnapshotSplice dst"
+       err = copyFromOS(s.fs, dstPath, "..") // arvados.git/sdk/go
+       c.Assert(err, check.IsNil)
+
+       // Snapshot directory
+       snap1, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/ctxlog")
+       c.Check(err, check.IsNil)
+       // Attach same snapshot twice, at paths that didn't exist before
+       err = Splice(s.fs, dstPath+"/ctxlog-copy", snap1)
+       c.Check(err, check.IsNil)
+       err = Splice(s.fs, dstPath+"/ctxlog-copy2", snap1)
+       c.Check(err, check.IsNil)
+       // Splicing a snapshot twice results in two independent copies
+       err = s.fs.Rename(dstPath+"/ctxlog-copy2/log.go", dstPath+"/ctxlog-copy/log2.go")
+       c.Check(err, check.IsNil)
+       _, err = s.fs.Open(dstPath + "/ctxlog-copy2/log.go")
+       c.Check(err, check.Equals, os.ErrNotExist)
+       f, err := s.fs.Open(dstPath + "/ctxlog-copy/log.go")
+       if c.Check(err, check.IsNil) {
+               buf, err := ioutil.ReadAll(f)
+               c.Check(err, check.IsNil)
+               c.Check(string(buf), check.Not(check.Equals), "")
+               f.Close()
+       }
+
+       // Snapshot regular file
+       snapFile, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/arvados/fs_site_test.go")
+       c.Check(err, check.IsNil)
+       // Replace dir with file
+       err = Splice(s.fs, dstPath+"/ctxlog-copy2", snapFile)
+       c.Check(err, check.IsNil)
+       if f, err := s.fs.Open(dstPath + "/ctxlog-copy2"); c.Check(err, check.IsNil) {
+               buf, err := ioutil.ReadAll(f)
+               c.Check(err, check.IsNil)
+               c.Check(string(buf), check.Equals, string(thisfile))
+       }
+
+       // Cannot splice a file onto a collection root, or anywhere
+       // outside a collection
+       for _, badpath := range []string{
+               dstPath,
+               "/home/A Project/newnodename",
+               "/home/A Project",
+               "/home/newnodename",
+               "/home",
+               "/newnodename",
+       } {
+               err = Splice(s.fs, badpath, snapFile)
+               c.Check(err, check.NotNil)
+               c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %s"))
+               if badpath == dstPath {
+                       c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %s", badpath))
+                       continue
+               }
+               err = Splice(s.fs, badpath, snap1)
+               c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %s"))
+       }
+
+       // Destination cannot have trailing slash
+       for _, badpath := range []string{
+               dstPath + "/ctxlog/",
+               dstPath + "/",
+               "/home/A Project/",
+               "/home/",
+               "/",
+               "",
+       } {
+               err = Splice(s.fs, badpath, snap1)
+               c.Check(err, ErrorIs, ErrInvalidArgument, check.Commentf("badpath %s", badpath))
+               err = Splice(s.fs, badpath, snapFile)
+               c.Check(err, ErrorIs, ErrInvalidArgument, check.Commentf("badpath %s", badpath))
+       }
+
+       // Destination's parent must already exist
+       for _, badpath := range []string{
+               dstPath + "/newdirname/",
+               dstPath + "/newdirname/foobar",
+               "/foo/bar",
+       } {
+               err = Splice(s.fs, badpath, snap1)
+               c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
+               err = Splice(s.fs, badpath, snapFile)
+               c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
+       }
+
+       snap2, err := Snapshot(s.fs, dstPath+"/ctxlog-copy")
+       c.Check(err, check.IsNil)
+       err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
+       c.Check(err, check.IsNil)
+
+       // Snapshot entire collection, splice into same collection at
+       // a new path, remove file from original location, verify
+       // spliced content survives
+       snapDst, err := Snapshot(s.fs, dstPath+"")
+       c.Check(err, check.IsNil)
+       err = Splice(s.fs, dstPath+"", snapDst)
+       c.Check(err, check.IsNil)
+       err = Splice(s.fs, dstPath+"/copy1", snapDst)
+       c.Check(err, check.IsNil)
+       err = Splice(s.fs, dstPath+"/copy2", snapDst)
+       c.Check(err, check.IsNil)
+       err = s.fs.RemoveAll(dstPath + "/arvados/fs_site_test.go")
+       c.Check(err, check.IsNil)
+       err = s.fs.RemoveAll(dstPath + "/arvados")
+       c.Check(err, check.IsNil)
+       _, err = s.fs.Open(dstPath + "/arvados/fs_site_test.go")
+       c.Check(err, check.Equals, os.ErrNotExist)
+       f, err = s.fs.Open(dstPath + "/copy2/arvados/fs_site_test.go")
+       c.Check(err, check.IsNil)
+       defer f.Close()
+       buf, err := ioutil.ReadAll(f)
+       c.Check(err, check.IsNil)
+       c.Check(string(buf), check.Equals, string(thisfile))
 }
index e6262374d640f9ed526ef38c816fa785bce1d3d5..5af7ebb5d52cfe17022ce46ed1e50fed507ca93f 100644 (file)
@@ -471,7 +471,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
                                        return true
                                }
                                err = fs.Mkdir(dir, 0755)
-                               if err == arvados.ErrInvalidArgument {
+                               if errors.Is(err, arvados.ErrInvalidArgument) || errors.Is(err, arvados.ErrInvalidOperation) {
                                        // Cannot create a directory
                                        // here.
                                        err = fmt.Errorf("mkdir %q failed: %w", dir, err)
index f411fde871cdba0cc9bbb2584be9a9bb878c6c1e..966f39c28dece0a4e3b5807d44f24e33a2cbe7c9 100644 (file)
@@ -332,7 +332,7 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
                err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
                c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
                c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
-               c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
+               c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid (argument|operation)`)
 
                _, err = bucket.GetReader(trial.path)
                c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)