X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/9a966d7d7e4cc104037534f0d4e65c81961eb7a1..469ea187586ea8017e26874c2d80414ce7571fae:/lib/controller/localdb/collection_test.go

diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index efcc830f1e..dac8b769fe 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -6,16 +6,22 @@ package localdb
 
 import (
 	"context"
+	"io/fs"
+	"path/filepath"
 	"regexp"
+	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/lib/controller/rpc"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
 	"git.arvados.org/arvados.git/sdk/go/arvadostest"
 	"git.arvados.org/arvados.git/sdk/go/auth"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"git.arvados.org/arvados.git/sdk/go/keepclient"
 	check "gopkg.in/check.v1"
 )
 
@@ -119,6 +125,180 @@ func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C
 	}
 }
 
+func (s *CollectionSuite) TestCollectionReplaceFiles(c *check.C) {
+	ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AdminToken}})
+	foo, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
+		Attrs: map[string]interface{}{
+			"owner_uuid":    arvadostest.ActiveUserUUID,
+			"manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
+		}})
+	c.Assert(err, check.IsNil)
+	s.localdb.signCollection(ctx, &foo)
+	foobarbaz, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
+		Attrs: map[string]interface{}{
+			"owner_uuid":    arvadostest.ActiveUserUUID,
+			"manifest_text": "./foo/bar 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
+		}})
+	c.Assert(err, check.IsNil)
+	s.localdb.signCollection(ctx, &foobarbaz)
+	wazqux, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
+		Attrs: map[string]interface{}{
+			"owner_uuid":    arvadostest.ActiveUserUUID,
+			"manifest_text": "./waz d85b1213473c2fd7c2045020a6b9c62b+3 0:3:qux.txt\n",
+		}})
+	c.Assert(err, check.IsNil)
+	s.localdb.signCollection(ctx, &wazqux)
+
+	ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+	// Create using content from existing collections
+	dst, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+		ReplaceFiles: map[string]string{
+			"/f": foo.PortableDataHash + "/foo.txt",
+			"/b": foobarbaz.PortableDataHash + "/foo/bar",
+			"/q": wazqux.PortableDataHash + "/",
+			"/w": wazqux.PortableDataHash + "/waz",
+		},
+		Attrs: map[string]interface{}{
+			"owner_uuid": arvadostest.ActiveUserUUID,
+		}})
+	c.Assert(err, check.IsNil)
+	s.expectFiles(c, dst, "f", "b/baz.txt", "q/waz/qux.txt", "w/qux.txt")
+
+	// Delete a file and a directory
+	dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+		UUID: dst.UUID,
+		ReplaceFiles: map[string]string{
+			"/f":     "",
+			"/q/waz": "",
+		}})
+	c.Assert(err, check.IsNil)
+	s.expectFiles(c, dst, "b/baz.txt", "q/", "w/qux.txt")
+
+	// Move and copy content within collection
+	dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+		UUID: dst.UUID,
+		ReplaceFiles: map[string]string{
+			// Note splicing content to /b/corge.txt but
+			// removing everything else from /b
+			"/b":              "",
+			"/b/corge.txt":    dst.PortableDataHash + "/b/baz.txt",
+			"/quux/corge.txt": dst.PortableDataHash + "/b/baz.txt",
+		}})
+	c.Assert(err, check.IsNil)
+	s.expectFiles(c, dst, "b/corge.txt", "q/", "w/qux.txt", "quux/corge.txt")
+
+	// Remove everything except one file
+	dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+		UUID: dst.UUID,
+		ReplaceFiles: map[string]string{
+			"/":            "",
+			"/b/corge.txt": dst.PortableDataHash + "/b/corge.txt",
+		}})
+	c.Assert(err, check.IsNil)
+	s.expectFiles(c, dst, "b/corge.txt")
+
+	// Copy entire collection to root
+	dstcopy, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+		ReplaceFiles: map[string]string{
+			"/": dst.PortableDataHash,
+		}})
+	c.Check(err, check.IsNil)
+	c.Check(dstcopy.PortableDataHash, check.Equals, dst.PortableDataHash)
+	s.expectFiles(c, dstcopy, "b/corge.txt")
+
+	// Check invalid targets, sources, and combinations
+	for _, badrepl := range []map[string]string{
+		{
+			"/foo/nope": dst.PortableDataHash + "/b",
+			"/foo":      dst.PortableDataHash + "/b",
+		},
+		{
+			"/foo":      dst.PortableDataHash + "/b",
+			"/foo/nope": "",
+		},
+		{
+			"/":     dst.PortableDataHash + "/",
+			"/nope": "",
+		},
+		{
+			"/":     dst.PortableDataHash + "/",
+			"/nope": dst.PortableDataHash + "/b",
+		},
+		{"/bad/": ""},
+		{"/./bad": ""},
+		{"/b/./ad": ""},
+		{"/b/../ad": ""},
+		{"/b/.": ""},
+		{".": ""},
+		{"bad": ""},
+		{"": ""},
+		{"/bad": "/b"},
+		{"/bad": "bad/b"},
+		{"/bad": dst.UUID + "/b"},
+	} {
+		_, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+			UUID:         dst.UUID,
+			ReplaceFiles: badrepl,
+		})
+		c.Logf("badrepl %#v\n... got err: %s", badrepl, err)
+		c.Check(err, check.NotNil)
+	}
+
+	// Check conflicting replace_files and manifest_text
+	_, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+		UUID:         dst.UUID,
+		ReplaceFiles: map[string]string{"/": ""},
+		Attrs: map[string]interface{}{
+			"manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:z\n",
+		}})
+	c.Logf("replace_files+manifest_text\n... got err: %s", err)
+	c.Check(err, check.ErrorMatches, "ambiguous request: both.*replace_files.*manifest_text.*")
+}
+
+// expectFiles checks coll's directory structure against the given
+// list of expected files and empty directories. An expected path with
+// a trailing slash indicates an empty directory.
+func (s *CollectionSuite) expectFiles(c *check.C, coll arvados.Collection, expected ...string) {
+	client := arvados.NewClientFromEnv()
+	ac, err := arvadosclient.New(client)
+	c.Assert(err, check.IsNil)
+	kc, err := keepclient.MakeKeepClient(ac)
+	c.Assert(err, check.IsNil)
+	cfs, err := coll.FileSystem(arvados.NewClientFromEnv(), kc)
+	c.Assert(err, check.IsNil)
+	var found []string
+	nonemptydirs := map[string]bool{}
+	fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
+		dir, _ := filepath.Split(path)
+		nonemptydirs[dir] = true
+		if d.IsDir() {
+			if path != "/" {
+				path += "/"
+			}
+			if !nonemptydirs[path] {
+				nonemptydirs[path] = false
+			}
+		} else {
+			found = append(found, path)
+		}
+		return nil
+	})
+	for d, nonempty := range nonemptydirs {
+		if !nonempty {
+			found = append(found, d)
+		}
+	}
+	for i, path := range found {
+		if path != "/" {
+			found[i] = strings.TrimPrefix(path, "/")
+		}
+	}
+	sort.Strings(found)
+	sort.Strings(expected)
+	c.Check(found, check.DeepEquals, expected)
+}
+
 func (s *CollectionSuite) TestSignatures(c *check.C) {
 	ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})