1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
16 "git.arvados.org/arvados.git/sdk/go/arvados"
17 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
18 "git.arvados.org/arvados.git/sdk/go/auth"
19 "git.arvados.org/arvados.git/sdk/go/httpserver"
22 // CollectionGet defers to railsProxy for everything except blob
24 func (conn *Conn) CollectionGet(ctx context.Context, opts arvados.GetOptions) (arvados.Collection, error) {
25 if len(opts.Select) > 0 {
26 // We need to know IsTrashed and TrashAt to implement
27 // signing properly, even if the caller doesn't want
29 opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
31 resp, err := conn.railsProxy.CollectionGet(ctx, opts)
35 conn.signCollection(ctx, &resp)
39 // CollectionList defers to railsProxy for everything except blob
41 func (conn *Conn) CollectionList(ctx context.Context, opts arvados.ListOptions) (arvados.CollectionList, error) {
42 if len(opts.Select) > 0 {
43 // We need to know IsTrashed and TrashAt to implement
44 // signing properly, even if the caller doesn't want
46 opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
48 resp, err := conn.railsProxy.CollectionList(ctx, opts)
52 for i := range resp.Items {
53 conn.signCollection(ctx, &resp.Items[i])
58 // CollectionCreate defers to railsProxy for everything except blob
59 // signatures and vocabulary checking.
60 func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Collection, error) {
61 err := conn.checkProperties(ctx, opts.Attrs["properties"])
63 return arvados.Collection{}, err
65 if len(opts.Select) > 0 {
66 // We need to know IsTrashed and TrashAt to implement
67 // signing properly, even if the caller doesn't want
69 opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
71 if opts.Attrs, err = conn.applyReplaceFilesOption(ctx, "", opts.Attrs, opts.ReplaceFiles); err != nil {
72 return arvados.Collection{}, err
74 resp, err := conn.railsProxy.CollectionCreate(ctx, opts)
78 conn.signCollection(ctx, &resp)
82 // CollectionUpdate defers to railsProxy for everything except blob
83 // signatures and vocabulary checking.
84 func (conn *Conn) CollectionUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Collection, error) {
85 err := conn.checkProperties(ctx, opts.Attrs["properties"])
87 return arvados.Collection{}, err
89 if len(opts.Select) > 0 {
90 // We need to know IsTrashed and TrashAt to implement
91 // signing properly, even if the caller doesn't want
93 opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
95 if opts.Attrs, err = conn.applyReplaceFilesOption(ctx, opts.UUID, opts.Attrs, opts.ReplaceFiles); err != nil {
96 return arvados.Collection{}, err
98 resp, err := conn.railsProxy.CollectionUpdate(ctx, opts)
102 conn.signCollection(ctx, &resp)
106 func (conn *Conn) signCollection(ctx context.Context, coll *arvados.Collection) {
107 if coll.IsTrashed || coll.ManifestText == "" || !conn.cluster.Collections.BlobSigning {
111 if creds, ok := auth.FromContext(ctx); ok && len(creds.Tokens) > 0 {
112 token = creds.Tokens[0]
117 ttl := conn.cluster.Collections.BlobSigningTTL.Duration()
118 exp := time.Now().Add(ttl)
119 if coll.TrashAt != nil && !coll.TrashAt.IsZero() && coll.TrashAt.Before(exp) {
122 coll.ManifestText = arvados.SignManifest(coll.ManifestText, token, exp, ttl, []byte(conn.cluster.Collections.BlobSigningKey))
125 // If replaceFiles is non-empty, populate attrs["manifest_text"] by
126 // starting with the content of fromUUID (or an empty collection if
127 // fromUUID is empty) and applying the specified file/directory
130 // Return value is the (possibly modified) attrs map.
131 func (conn *Conn) applyReplaceFilesOption(ctx context.Context, fromUUID string, attrs map[string]interface{}, replaceFiles map[string]string) (map[string]interface{}, error) {
132 if len(replaceFiles) == 0 {
134 } else if mtxt, ok := attrs["manifest_text"].(string); ok && len(mtxt) > 0 {
135 return nil, httpserver.Errorf(http.StatusBadRequest, "ambiguous request: both 'replace_files' and attrs['manifest_text'] values provided")
138 // Load the current collection (if any) and set up an
139 // in-memory filesystem.
140 var dst arvados.Collection
141 if _, replacingRoot := replaceFiles["/"]; !replacingRoot && fromUUID != "" {
142 src, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: fromUUID})
148 dstfs, err := dst.FileSystem(&arvados.StubClient{}, &arvados.StubClient{})
153 // Sort replacements by source collection to avoid redundant
154 // reloads when a source collection is used more than
155 // once. Note empty sources (which mean "delete target path")
157 dstTodo := make([]string, 0, len(replaceFiles))
159 srcid := make(map[string]string, len(replaceFiles))
160 for dst, src := range replaceFiles {
161 dstTodo = append(dstTodo, dst)
162 if i := strings.IndexRune(src, '/'); i > 0 {
166 sort.Slice(dstTodo, func(i, j int) bool {
167 return srcid[dstTodo[i]] < srcid[dstTodo[j]]
171 // Reject attempt to replace a node as well as its descendant
172 // (e.g., a/ and a/b/), which is unsupported, except where the
173 // source for a/ is empty (i.e., delete).
174 for _, dst := range dstTodo {
175 if dst != "/" && (strings.HasSuffix(dst, "/") ||
176 strings.HasSuffix(dst, "/.") ||
177 strings.HasSuffix(dst, "/..") ||
178 strings.Contains(dst, "//") ||
179 strings.Contains(dst, "/./") ||
180 strings.Contains(dst, "/../") ||
181 !strings.HasPrefix(dst, "/")) {
182 return nil, httpserver.Errorf(http.StatusBadRequest, "invalid replace_files target: %q", dst)
184 for i := 0; i < len(dst)-1; i++ {
192 if outersrc := replaceFiles[outerdst]; outersrc != "" {
193 return nil, httpserver.Errorf(http.StatusBadRequest, "replace_files: cannot operate on target %q inside non-empty target %q", dst, outerdst)
198 var srcidloaded string
199 var srcfs arvados.FileSystem
200 // Apply the requested replacements.
201 for _, dst := range dstTodo {
202 src := replaceFiles[dst]
205 // In this case we started with a
206 // blank manifest, so there can't be
207 // anything to delete.
210 err := dstfs.RemoveAll(dst)
212 return nil, fmt.Errorf("RemoveAll(%s): %w", dst, err)
216 srcspec := strings.SplitN(src, "/", 2)
217 srcid, srcpath := srcspec[0], "/"
218 if !arvadosclient.PDHMatch(srcid) {
219 return nil, httpserver.Errorf(http.StatusBadRequest, "invalid source %q for replace_files[%q]: must be \"\" or \"PDH\" or \"PDH/path\"", src, dst)
221 if len(srcspec) == 2 && srcspec[1] != "" {
224 if srcidloaded != srcid {
226 srccoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: srcid})
230 // We use StubClient here because we don't
231 // want srcfs to read/write any file data or
232 // sync collection state to/from the database.
233 srcfs, err = srccoll.FileSystem(&arvados.StubClient{}, &arvados.StubClient{})
239 snap, err := arvados.Snapshot(srcfs, srcpath)
241 return nil, httpserver.Errorf(http.StatusBadRequest, "error getting snapshot of %q from %q: %w", srcpath, srcid, err)
243 // Create intermediate dirs, in case dst is
244 // "newdir1/newdir2/dst".
245 for i := 1; i < len(dst)-1; i++ {
247 err = dstfs.Mkdir(dst[:i], 0777)
248 if err != nil && !os.IsExist(err) {
249 return nil, httpserver.Errorf(http.StatusBadRequest, "error creating parent dirs for %q: %w", dst, err)
253 err = arvados.Splice(dstfs, dst, snap)
255 return nil, fmt.Errorf("error splicing snapshot onto path %q: %w", dst, err)
258 mtxt, err := dstfs.MarshalManifest(".")
263 attrs = make(map[string]interface{}, 1)
265 attrs["manifest_text"] = mtxt