Merge branch '18691-freeze-project'
[arvados.git] / lib / controller / localdb / collection_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package localdb
6
7 import (
8         "context"
9         "io/fs"
10         "path/filepath"
11         "regexp"
12         "sort"
13         "strconv"
14         "strings"
15         "time"
16
17         "git.arvados.org/arvados.git/lib/config"
18         "git.arvados.org/arvados.git/lib/controller/rpc"
19         "git.arvados.org/arvados.git/sdk/go/arvados"
20         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
21         "git.arvados.org/arvados.git/sdk/go/arvadostest"
22         "git.arvados.org/arvados.git/sdk/go/auth"
23         "git.arvados.org/arvados.git/sdk/go/ctxlog"
24         "git.arvados.org/arvados.git/sdk/go/keepclient"
25         check "gopkg.in/check.v1"
26 )
27
28 var _ = check.Suite(&CollectionSuite{})
29
30 type CollectionSuite struct {
31         cluster  *arvados.Cluster
32         localdb  *Conn
33         railsSpy *arvadostest.Proxy
34 }
35
36 func (s *CollectionSuite) TearDownSuite(c *check.C) {
37         // Undo any changes/additions to the user database so they
38         // don't affect subsequent tests.
39         arvadostest.ResetEnv()
40         c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
41 }
42
43 func (s *CollectionSuite) SetUpTest(c *check.C) {
44         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
45         c.Assert(err, check.IsNil)
46         s.cluster, err = cfg.GetCluster("")
47         c.Assert(err, check.IsNil)
48         s.localdb = NewConn(s.cluster)
49         s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
50         *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
51 }
52
53 func (s *CollectionSuite) TearDownTest(c *check.C) {
54         s.railsSpy.Close()
55 }
56
57 func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) {
58         if testVocabulary == "" {
59                 testVocabulary = `{
60                         "strict_tags": false,
61                         "tags": {
62                                 "IDTAGIMPORTANCES": {
63                                         "strict": true,
64                                         "labels": [{"label": "Importance"}, {"label": "Priority"}],
65                                         "values": {
66                                                 "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
67                                                 "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
68                                                 "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
69                                         }
70                                 }
71                         }
72                 }`
73         }
74         voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
75         c.Assert(err, check.IsNil)
76         s.cluster.API.VocabularyPath = "foo"
77         s.localdb.vocabularyCache = voc
78 }
79
80 func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C) {
81         s.setUpVocabulary(c, "")
82         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
83
84         tests := []struct {
85                 name    string
86                 props   map[string]interface{}
87                 success bool
88         }{
89                 {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
90                 {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
91                 {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
92                 {"Empty properties", map[string]interface{}{}, true},
93         }
94         for _, tt := range tests {
95                 c.Log(c.TestName()+" ", tt.name)
96
97                 // Create with properties
98                 coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
99                         Select: []string{"uuid", "properties"},
100                         Attrs: map[string]interface{}{
101                                 "properties": tt.props,
102                         }})
103                 if tt.success {
104                         c.Assert(err, check.IsNil)
105                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
106                 } else {
107                         c.Assert(err, check.NotNil)
108                 }
109
110                 // Create, then update with properties
111                 coll, err = s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
112                 c.Assert(err, check.IsNil)
113                 coll, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
114                         UUID:   coll.UUID,
115                         Select: []string{"uuid", "properties"},
116                         Attrs: map[string]interface{}{
117                                 "properties": tt.props,
118                         }})
119                 if tt.success {
120                         c.Assert(err, check.IsNil)
121                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
122                 } else {
123                         c.Assert(err, check.NotNil)
124                 }
125         }
126 }
127
128 func (s *CollectionSuite) TestCollectionReplaceFiles(c *check.C) {
129         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AdminToken}})
130         foo, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
131                 Attrs: map[string]interface{}{
132                         "owner_uuid":    arvadostest.ActiveUserUUID,
133                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
134                 }})
135         c.Assert(err, check.IsNil)
136         s.localdb.signCollection(ctx, &foo)
137         foobarbaz, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
138                 Attrs: map[string]interface{}{
139                         "owner_uuid":    arvadostest.ActiveUserUUID,
140                         "manifest_text": "./foo/bar 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
141                 }})
142         c.Assert(err, check.IsNil)
143         s.localdb.signCollection(ctx, &foobarbaz)
144         wazqux, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
145                 Attrs: map[string]interface{}{
146                         "owner_uuid":    arvadostest.ActiveUserUUID,
147                         "manifest_text": "./waz d85b1213473c2fd7c2045020a6b9c62b+3 0:3:qux.txt\n",
148                 }})
149         c.Assert(err, check.IsNil)
150         s.localdb.signCollection(ctx, &wazqux)
151
152         ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
153
154         // Create using content from existing collections
155         dst, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
156                 ReplaceFiles: map[string]string{
157                         "/f": foo.PortableDataHash + "/foo.txt",
158                         "/b": foobarbaz.PortableDataHash + "/foo/bar",
159                         "/q": wazqux.PortableDataHash + "/",
160                         "/w": wazqux.PortableDataHash + "/waz",
161                 },
162                 Attrs: map[string]interface{}{
163                         "owner_uuid": arvadostest.ActiveUserUUID,
164                 }})
165         c.Assert(err, check.IsNil)
166         s.expectFiles(c, dst, "f", "b/baz.txt", "q/waz/qux.txt", "w/qux.txt")
167
168         // Delete a file and a directory
169         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
170                 UUID: dst.UUID,
171                 ReplaceFiles: map[string]string{
172                         "/f":     "",
173                         "/q/waz": "",
174                 }})
175         c.Assert(err, check.IsNil)
176         s.expectFiles(c, dst, "b/baz.txt", "q/", "w/qux.txt")
177
178         // Move and copy content within collection
179         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
180                 UUID: dst.UUID,
181                 ReplaceFiles: map[string]string{
182                         // Note splicing content to /b/corge.txt but
183                         // removing everything else from /b
184                         "/b":              "",
185                         "/b/corge.txt":    dst.PortableDataHash + "/b/baz.txt",
186                         "/quux/corge.txt": dst.PortableDataHash + "/b/baz.txt",
187                 }})
188         c.Assert(err, check.IsNil)
189         s.expectFiles(c, dst, "b/corge.txt", "q/", "w/qux.txt", "quux/corge.txt")
190
191         // Remove everything except one file
192         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
193                 UUID: dst.UUID,
194                 ReplaceFiles: map[string]string{
195                         "/":            "",
196                         "/b/corge.txt": dst.PortableDataHash + "/b/corge.txt",
197                 }})
198         c.Assert(err, check.IsNil)
199         s.expectFiles(c, dst, "b/corge.txt")
200
201         // Copy entire collection to root
202         dstcopy, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
203                 ReplaceFiles: map[string]string{
204                         "/": dst.PortableDataHash,
205                 }})
206         c.Check(err, check.IsNil)
207         c.Check(dstcopy.PortableDataHash, check.Equals, dst.PortableDataHash)
208         s.expectFiles(c, dstcopy, "b/corge.txt")
209
210         // Check invalid targets, sources, and combinations
211         for _, badrepl := range []map[string]string{
212                 {
213                         "/foo/nope": dst.PortableDataHash + "/b",
214                         "/foo":      dst.PortableDataHash + "/b",
215                 },
216                 {
217                         "/foo":      dst.PortableDataHash + "/b",
218                         "/foo/nope": "",
219                 },
220                 {
221                         "/":     dst.PortableDataHash + "/",
222                         "/nope": "",
223                 },
224                 {
225                         "/":     dst.PortableDataHash + "/",
226                         "/nope": dst.PortableDataHash + "/b",
227                 },
228                 {"/bad/": ""},
229                 {"/./bad": ""},
230                 {"/b/./ad": ""},
231                 {"/b/../ad": ""},
232                 {"/b/.": ""},
233                 {".": ""},
234                 {"bad": ""},
235                 {"": ""},
236                 {"/bad": "/b"},
237                 {"/bad": "bad/b"},
238                 {"/bad": dst.UUID + "/b"},
239         } {
240                 _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
241                         UUID:         dst.UUID,
242                         ReplaceFiles: badrepl,
243                 })
244                 c.Logf("badrepl %#v\n... got err: %s", badrepl, err)
245                 c.Check(err, check.NotNil)
246         }
247
248         // Check conflicting replace_files and manifest_text
249         _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
250                 UUID:         dst.UUID,
251                 ReplaceFiles: map[string]string{"/": ""},
252                 Attrs: map[string]interface{}{
253                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:z\n",
254                 }})
255         c.Logf("replace_files+manifest_text\n... got err: %s", err)
256         c.Check(err, check.ErrorMatches, "ambiguous request: both.*replace_files.*manifest_text.*")
257 }
258
259 // expectFiles checks coll's directory structure against the given
260 // list of expected files and empty directories. An expected path with
261 // a trailing slash indicates an empty directory.
262 func (s *CollectionSuite) expectFiles(c *check.C, coll arvados.Collection, expected ...string) {
263         client := arvados.NewClientFromEnv()
264         ac, err := arvadosclient.New(client)
265         c.Assert(err, check.IsNil)
266         kc, err := keepclient.MakeKeepClient(ac)
267         c.Assert(err, check.IsNil)
268         cfs, err := coll.FileSystem(arvados.NewClientFromEnv(), kc)
269         c.Assert(err, check.IsNil)
270         var found []string
271         nonemptydirs := map[string]bool{}
272         fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
273                 dir, _ := filepath.Split(path)
274                 nonemptydirs[dir] = true
275                 if d.IsDir() {
276                         if path != "/" {
277                                 path += "/"
278                         }
279                         if !nonemptydirs[path] {
280                                 nonemptydirs[path] = false
281                         }
282                 } else {
283                         found = append(found, path)
284                 }
285                 return nil
286         })
287         for d, nonempty := range nonemptydirs {
288                 if !nonempty {
289                         found = append(found, d)
290                 }
291         }
292         for i, path := range found {
293                 if path != "/" {
294                         found[i] = strings.TrimPrefix(path, "/")
295                 }
296         }
297         sort.Strings(found)
298         sort.Strings(expected)
299         c.Check(found, check.DeepEquals, expected)
300 }
301
302 func (s *CollectionSuite) TestSignatures(c *check.C) {
303         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
304
305         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
306         c.Check(err, check.IsNil)
307         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
308         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
309
310         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
311         c.Check(err, check.IsNil)
312         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
313
314         lresp, err := s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}})
315         c.Check(err, check.IsNil)
316         if c.Check(lresp.Items, check.HasLen, 1) {
317                 c.Check(lresp.Items[0].UUID, check.Equals, arvadostest.FooCollection)
318                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
319                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
320         }
321
322         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"manifest_text"}})
323         c.Check(err, check.IsNil)
324         if c.Check(lresp.Items, check.HasLen, 1) {
325                 c.Check(lresp.Items[0].ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
326                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
327         }
328
329         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"unsigned_manifest_text"}})
330         c.Check(err, check.IsNil)
331         if c.Check(lresp.Items, check.HasLen, 1) {
332                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
333                 c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
334         }
335
336         // early trash date causes lower signature TTL (even if
337         // trash_at and is_trashed fields are unselected)
338         trashed, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
339                 Select: []string{"uuid", "manifest_text"},
340                 Attrs: map[string]interface{}{
341                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
342                         "trash_at":      time.Now().UTC().Add(time.Hour),
343                 }})
344         c.Assert(err, check.IsNil)
345         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour)
346         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
347         c.Assert(err, check.IsNil)
348         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour)
349
350         // distant future trash date does not cause higher signature TTL
351         trashed, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
352                 UUID: trashed.UUID,
353                 Attrs: map[string]interface{}{
354                         "trash_at": time.Now().UTC().Add(time.Hour * 24 * 365),
355                 }})
356         c.Assert(err, check.IsNil)
357         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour*24*7*2)
358         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
359         c.Assert(err, check.IsNil)
360         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
361
362         // Make sure groups/contents doesn't return manifest_text with
363         // collections (if it did, we'd need to sign it).
364         gresp, err := s.localdb.GroupContents(ctx, arvados.GroupContentsOptions{
365                 Limit:   -1,
366                 Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}},
367                 Select:  []string{"uuid", "manifest_text"},
368         })
369         if err != nil {
370                 c.Check(err, check.ErrorMatches, `.*Invalid attribute.*manifest_text.*`)
371         } else if c.Check(gresp.Items, check.HasLen, 1) {
372                 c.Check(gresp.Items[0].(map[string]interface{})["uuid"], check.Equals, arvadostest.FooCollection)
373                 c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
374         }
375 }
376
377 func (s *CollectionSuite) checkSignatureExpiry(c *check.C, manifestText string, expectedTTL time.Duration) {
378         m := regexp.MustCompile(`@([[:xdigit:]]+)`).FindStringSubmatch(manifestText)
379         c.Assert(m, check.HasLen, 2)
380         sigexp, err := strconv.ParseInt(m[1], 16, 64)
381         c.Assert(err, check.IsNil)
382         expectedExp := time.Now().Add(expectedTTL).Unix()
383         c.Check(sigexp > expectedExp-60, check.Equals, true)
384         c.Check(sigexp <= expectedExp, check.Equals, true)
385 }
386
387 func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
388         s.localdb.cluster.Collections.BlobSigning = false
389         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
390
391         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
392         c.Check(err, check.IsNil)
393         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ +]*\+3 0:.*`)
394 }