import (
"context"
+ "fmt"
+ "net/http"
+ "os"
+ "sort"
+ "strings"
"time"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
)
// CollectionGet defers to railsProxy for everything except blob
}
// CollectionCreate defers to railsProxy for everything except blob
-// signatures.
+// signatures and vocabulary checking.
func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Collection, error) {
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
+ if err != nil {
+ return arvados.Collection{}, err
+ }
if len(opts.Select) > 0 {
// We need to know IsTrashed and TrashAt to implement
// signing properly, even if the caller doesn't want
// them.
opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
}
+ if opts.Attrs, err = conn.applyReplaceFilesOption(ctx, "", opts.Attrs, opts.ReplaceFiles); err != nil {
+ return arvados.Collection{}, err
+ }
resp, err := conn.railsProxy.CollectionCreate(ctx, opts)
if err != nil {
return resp, err
}
// CollectionUpdate defers to railsProxy for everything except blob
-// signatures.
+// signatures and vocabulary checking.
func (conn *Conn) CollectionUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Collection, error) {
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
+ if err != nil {
+ return arvados.Collection{}, err
+ }
if len(opts.Select) > 0 {
// We need to know IsTrashed and TrashAt to implement
// signing properly, even if the caller doesn't want
// them.
opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
}
+ if opts.Attrs, err = conn.applyReplaceFilesOption(ctx, opts.UUID, opts.Attrs, opts.ReplaceFiles); err != nil {
+ return arvados.Collection{}, err
+ }
resp, err := conn.railsProxy.CollectionUpdate(ctx, opts)
if err != nil {
return resp, err
}
coll.ManifestText = arvados.SignManifest(coll.ManifestText, token, exp, ttl, []byte(conn.cluster.Collections.BlobSigningKey))
}
+
+// If replaceFiles is non-empty, populate attrs["manifest_text"] by
+// starting with the content of fromUUID (or an empty collection if
+// fromUUID is empty) and applying the specified file/directory
+// replacements.
+//
+// Return value is the (possibly modified) attrs map.
+func (conn *Conn) applyReplaceFilesOption(ctx context.Context, fromUUID string, attrs map[string]interface{}, replaceFiles map[string]string) (map[string]interface{}, error) {
+ if len(replaceFiles) == 0 {
+ return attrs, nil
+ } else if mtxt, ok := attrs["manifest_text"].(string); ok && len(mtxt) > 0 {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "ambiguous request: both 'replace_files' and attrs['manifest_text'] values provided")
+ }
+
+ // Load the current collection (if any) and set up an
+ // in-memory filesystem.
+ var dst arvados.Collection
+ if _, replacingRoot := replaceFiles["/"]; !replacingRoot && fromUUID != "" {
+ src, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: fromUUID})
+ if err != nil {
+ return nil, err
+ }
+ dst = src
+ }
+ dstfs, err := dst.FileSystem(&arvados.StubClient{}, &arvados.StubClient{})
+ if err != nil {
+ return nil, err
+ }
+
+ // Sort replacements by source collection to avoid redundant
+ // reloads when a source collection is used more than
+ // once. Note empty sources (which mean "delete target path")
+ // sort first.
+ dstTodo := make([]string, 0, len(replaceFiles))
+ {
+ srcid := make(map[string]string, len(replaceFiles))
+ for dst, src := range replaceFiles {
+ dstTodo = append(dstTodo, dst)
+ if i := strings.IndexRune(src, '/'); i > 0 {
+ srcid[dst] = src[:i]
+ }
+ }
+ sort.Slice(dstTodo, func(i, j int) bool {
+ return srcid[dstTodo[i]] < srcid[dstTodo[j]]
+ })
+ }
+
+ // Reject attempt to replace a node as well as its descendant
+ // (e.g., a/ and a/b/), which is unsupported, except where the
+ // source for a/ is empty (i.e., delete).
+ for _, dst := range dstTodo {
+ if dst != "/" && (strings.HasSuffix(dst, "/") ||
+ strings.HasSuffix(dst, "/.") ||
+ strings.HasSuffix(dst, "/..") ||
+ strings.Contains(dst, "//") ||
+ strings.Contains(dst, "/./") ||
+ strings.Contains(dst, "/../") ||
+ !strings.HasPrefix(dst, "/")) {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "invalid replace_files target: %q", dst)
+ }
+ for i := 0; i < len(dst)-1; i++ {
+ if dst[i] != '/' {
+ continue
+ }
+ outerdst := dst[:i]
+ if outerdst == "" {
+ outerdst = "/"
+ }
+ if outersrc := replaceFiles[outerdst]; outersrc != "" {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "replace_files: cannot operate on target %q inside non-empty target %q", dst, outerdst)
+ }
+ }
+ }
+
+ var srcidloaded string
+ var srcfs arvados.FileSystem
+ // Apply the requested replacements.
+ for _, dst := range dstTodo {
+ src := replaceFiles[dst]
+ if src == "" {
+ if dst == "/" {
+ // In this case we started with a
+ // blank manifest, so there can't be
+ // anything to delete.
+ continue
+ }
+ err := dstfs.RemoveAll(dst)
+ if err != nil {
+ return nil, fmt.Errorf("RemoveAll(%s): %w", dst, err)
+ }
+ continue
+ }
+ srcspec := strings.SplitN(src, "/", 2)
+ srcid, srcpath := srcspec[0], "/"
+ if !arvadosclient.PDHMatch(srcid) {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "invalid source %q for replace_files[%q]: must be \"\" or \"PDH\" or \"PDH/path\"", src, dst)
+ }
+ if len(srcspec) == 2 && srcspec[1] != "" {
+ srcpath = srcspec[1]
+ }
+ if srcidloaded != srcid {
+ srcfs = nil
+ srccoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: srcid})
+ if err != nil {
+ return nil, err
+ }
+ // We use StubClient here because we don't
+ // want srcfs to read/write any file data or
+ // sync collection state to/from the database.
+ srcfs, err = srccoll.FileSystem(&arvados.StubClient{}, &arvados.StubClient{})
+ if err != nil {
+ return nil, err
+ }
+ srcidloaded = srcid
+ }
+ snap, err := arvados.Snapshot(srcfs, srcpath)
+ if err != nil {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "error getting snapshot of %q from %q: %w", srcpath, srcid, err)
+ }
+ // Create intermediate dirs, in case dst is
+ // "newdir1/newdir2/dst".
+ for i := 1; i < len(dst)-1; i++ {
+ if dst[i] == '/' {
+ err = dstfs.Mkdir(dst[:i], 0777)
+ if err != nil && !os.IsExist(err) {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "error creating parent dirs for %q: %w", dst, err)
+ }
+ }
+ }
+ err = arvados.Splice(dstfs, dst, snap)
+ if err != nil {
+ return nil, fmt.Errorf("error splicing snapshot onto path %q: %w", dst, err)
+ }
+ }
+ mtxt, err := dstfs.MarshalManifest(".")
+ if err != nil {
+ return nil, err
+ }
+ if attrs == nil {
+ attrs = make(map[string]interface{}, 1)
+ }
+ attrs["manifest_text"] = mtxt
+ return attrs, nil
+}
projects / arvados.git / blobdiff