X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/72d7d41944006d1f48f570784dafe56b9812b0c8..55723c549ef42d76584a9791036a12c68ffb95eb:/lib/controller/localdb/collection.go?ds=sidebyside diff --git a/lib/controller/localdb/collection.go b/lib/controller/localdb/collection.go index d81dd812bf..868e466e9e 100644 --- a/lib/controller/localdb/collection.go +++ b/lib/controller/localdb/collection.go @@ -6,10 +6,17 @@ package localdb import ( "context" + "fmt" + "net/http" + "os" + "sort" + "strings" "time" "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/auth" + "git.arvados.org/arvados.git/sdk/go/httpserver" ) // CollectionGet defers to railsProxy for everything except blob @@ -49,14 +56,21 @@ func (conn *Conn) CollectionList(ctx context.Context, opts arvados.ListOptions) } // CollectionCreate defers to railsProxy for everything except blob -// signatures. +// signatures and vocabulary checking. func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Collection, error) { + err := conn.checkProperties(ctx, opts.Attrs["properties"]) + if err != nil { + return arvados.Collection{}, err + } if len(opts.Select) > 0 { // We need to know IsTrashed and TrashAt to implement // signing properly, even if the caller doesn't want // them. opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...) } + if opts.Attrs, err = conn.applyReplaceFilesOption(ctx, "", opts.Attrs, opts.ReplaceFiles); err != nil { + return arvados.Collection{}, err + } resp, err := conn.railsProxy.CollectionCreate(ctx, opts) if err != nil { return resp, err @@ -66,14 +80,21 @@ func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptio } // CollectionUpdate defers to railsProxy for everything except blob -// signatures. +// signatures and vocabulary checking. func (conn *Conn) CollectionUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Collection, error) { + err := conn.checkProperties(ctx, opts.Attrs["properties"]) + if err != nil { + return arvados.Collection{}, err + } if len(opts.Select) > 0 { // We need to know IsTrashed and TrashAt to implement // signing properly, even if the caller doesn't want // them. opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...) } + if opts.Attrs, err = conn.applyReplaceFilesOption(ctx, opts.UUID, opts.Attrs, opts.ReplaceFiles); err != nil { + return arvados.Collection{}, err + } resp, err := conn.railsProxy.CollectionUpdate(ctx, opts) if err != nil { return resp, err @@ -100,3 +121,147 @@ func (conn *Conn) signCollection(ctx context.Context, coll *arvados.Collection) } coll.ManifestText = arvados.SignManifest(coll.ManifestText, token, exp, ttl, []byte(conn.cluster.Collections.BlobSigningKey)) } + +// If replaceFiles is non-empty, populate attrs["manifest_text"] by +// starting with the content of fromUUID (or an empty collection if +// fromUUID is empty) and applying the specified file/directory +// replacements. +// +// Return value is the (possibly modified) attrs map. +func (conn *Conn) applyReplaceFilesOption(ctx context.Context, fromUUID string, attrs map[string]interface{}, replaceFiles map[string]string) (map[string]interface{}, error) { + if len(replaceFiles) == 0 { + return attrs, nil + } else if mtxt, ok := attrs["manifest_text"].(string); ok && len(mtxt) > 0 { + return nil, httpserver.Errorf(http.StatusBadRequest, "ambiguous request: both 'replace_files' and attrs['manifest_text'] values provided") + } + + // Load the current collection (if any) and set up an + // in-memory filesystem. + var dst arvados.Collection + if _, replacingRoot := replaceFiles["/"]; !replacingRoot && fromUUID != "" { + src, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: fromUUID}) + if err != nil { + return nil, err + } + dst = src + } + dstfs, err := dst.FileSystem(&arvados.StubClient{}, &arvados.StubClient{}) + if err != nil { + return nil, err + } + + // Sort replacements by source collection to avoid redundant + // reloads when a source collection is used more than + // once. Note empty sources (which mean "delete target path") + // sort first. + dstTodo := make([]string, 0, len(replaceFiles)) + { + srcid := make(map[string]string, len(replaceFiles)) + for dst, src := range replaceFiles { + dstTodo = append(dstTodo, dst) + if i := strings.IndexRune(src, '/'); i > 0 { + srcid[dst] = src[:i] + } + } + sort.Slice(dstTodo, func(i, j int) bool { + return srcid[dstTodo[i]] < srcid[dstTodo[j]] + }) + } + + // Reject attempt to replace a node as well as its descendant + // (e.g., a/ and a/b/), which is unsupported, except where the + // source for a/ is empty (i.e., delete). + for _, dst := range dstTodo { + if dst != "/" && (strings.HasSuffix(dst, "/") || + strings.HasSuffix(dst, "/.") || + strings.HasSuffix(dst, "/..") || + strings.Contains(dst, "//") || + strings.Contains(dst, "/./") || + strings.Contains(dst, "/../") || + !strings.HasPrefix(dst, "/")) { + return nil, httpserver.Errorf(http.StatusBadRequest, "invalid replace_files target: %q", dst) + } + for i := 0; i < len(dst)-1; i++ { + if dst[i] != '/' { + continue + } + outerdst := dst[:i] + if outerdst == "" { + outerdst = "/" + } + if outersrc := replaceFiles[outerdst]; outersrc != "" { + return nil, httpserver.Errorf(http.StatusBadRequest, "replace_files: cannot operate on target %q inside non-empty target %q", dst, outerdst) + } + } + } + + var srcidloaded string + var srcfs arvados.FileSystem + // Apply the requested replacements. + for _, dst := range dstTodo { + src := replaceFiles[dst] + if src == "" { + if dst == "/" { + // In this case we started with a + // blank manifest, so there can't be + // anything to delete. + continue + } + err := dstfs.RemoveAll(dst) + if err != nil { + return nil, fmt.Errorf("RemoveAll(%s): %w", dst, err) + } + continue + } + srcspec := strings.SplitN(src, "/", 2) + srcid, srcpath := srcspec[0], "/" + if !arvadosclient.PDHMatch(srcid) { + return nil, httpserver.Errorf(http.StatusBadRequest, "invalid source %q for replace_files[%q]: must be \"\" or \"PDH\" or \"PDH/path\"", src, dst) + } + if len(srcspec) == 2 && srcspec[1] != "" { + srcpath = srcspec[1] + } + if srcidloaded != srcid { + srcfs = nil + srccoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: srcid}) + if err != nil { + return nil, err + } + // We use StubClient here because we don't + // want srcfs to read/write any file data or + // sync collection state to/from the database. + srcfs, err = srccoll.FileSystem(&arvados.StubClient{}, &arvados.StubClient{}) + if err != nil { + return nil, err + } + srcidloaded = srcid + } + snap, err := arvados.Snapshot(srcfs, srcpath) + if err != nil { + return nil, httpserver.Errorf(http.StatusBadRequest, "error getting snapshot of %q from %q: %w", srcpath, srcid, err) + } + // Create intermediate dirs, in case dst is + // "newdir1/newdir2/dst". + for i := 1; i < len(dst)-1; i++ { + if dst[i] == '/' { + err = dstfs.Mkdir(dst[:i], 0777) + if err != nil && !os.IsExist(err) { + return nil, httpserver.Errorf(http.StatusBadRequest, "error creating parent dirs for %q: %w", dst, err) + } + } + } + err = arvados.Splice(dstfs, dst, snap) + if err != nil { + return nil, fmt.Errorf("error splicing snapshot onto path %q: %w", dst, err) + } + } + mtxt, err := dstfs.MarshalManifest(".") + if err != nil { + return nil, err + } + if attrs == nil { + attrs = make(map[string]interface{}, 1) + } + attrs["manifest_text"] = mtxt + return attrs, nil +}