lib/controller/localdb/collection_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "context"
   9         "io/fs"
  10         "path/filepath"
  11         "regexp"
  12         "sort"
  13         "strconv"
  14         "strings"
  15         "time"
  16
  17         "git.arvados.org/arvados.git/lib/config"
  18         "git.arvados.org/arvados.git/lib/controller/rpc"
  19         "git.arvados.org/arvados.git/sdk/go/arvados"
  20         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  21         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  22         "git.arvados.org/arvados.git/sdk/go/auth"
  23         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  24         "git.arvados.org/arvados.git/sdk/go/keepclient"
  25         check "gopkg.in/check.v1"
  26 )
  27
  28 var _ = check.Suite(&CollectionSuite{})
  29
  30 type CollectionSuite struct {
  31         cluster  *arvados.Cluster
  32         localdb  *Conn
  33         railsSpy *arvadostest.Proxy
  34 }
  35
  36 func (s *CollectionSuite) TearDownSuite(c *check.C) {
  37         // Undo any changes/additions to the user database so they
  38         // don't affect subsequent tests.
  39         arvadostest.ResetEnv()
  40         c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
  41 }
  42
  43 func (s *CollectionSuite) SetUpTest(c *check.C) {
  44         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
  45         c.Assert(err, check.IsNil)
  46         s.cluster, err = cfg.GetCluster("")
  47         c.Assert(err, check.IsNil)
  48         s.localdb = NewConn(s.cluster)
  49         s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
  50         *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
  51 }
  52
  53 func (s *CollectionSuite) TearDownTest(c *check.C) {
  54         s.railsSpy.Close()
  55 }
  56
  57 func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) {
  58         if testVocabulary == "" {
  59                 testVocabulary = `{
  60                         "strict_tags": false,
  61                         "tags": {
  62                                 "IDTAGIMPORTANCES": {
  63                                         "strict": true,
  64                                         "labels": [{"label": "Importance"}, {"label": "Priority"}],
  65                                         "values": {
  66                                                 "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
  67                                                 "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
  68                                                 "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
  69                                         }
  70                                 }
  71                         }
  72                 }`
  73         }
  74         voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
  75         c.Assert(err, check.IsNil)
  76         s.cluster.API.VocabularyPath = "foo"
  77         s.localdb.vocabularyCache = voc
  78 }
  79
  80 func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C) {
  81         s.setUpVocabulary(c, "")
  82         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
  83
  84         tests := []struct {
  85                 name    string
  86                 props   map[string]interface{}
  87                 success bool
  88         }{
  89                 {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
  90                 {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
  91                 {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
  92                 {"Empty properties", map[string]interface{}{}, true},
  93         }
  94         for _, tt := range tests {
  95                 c.Log(c.TestName()+" ", tt.name)
  96
  97                 // Create with properties
  98                 coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
  99                         Select: []string{"uuid", "properties"},
 100                         Attrs: map[string]interface{}{
 101                                 "properties": tt.props,
 102                         }})
 103                 if tt.success {
 104                         c.Assert(err, check.IsNil)
 105                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
 106                 } else {
 107                         c.Assert(err, check.NotNil)
 108                 }
 109
 110                 // Create, then update with properties
 111                 coll, err = s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
 112                 c.Assert(err, check.IsNil)
 113                 coll, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 114                         UUID:   coll.UUID,
 115                         Select: []string{"uuid", "properties"},
 116                         Attrs: map[string]interface{}{
 117                                 "properties": tt.props,
 118                         }})
 119                 if tt.success {
 120                         c.Assert(err, check.IsNil)
 121                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
 122                 } else {
 123                         c.Assert(err, check.NotNil)
 124                 }
 125         }
 126 }
 127
 128 func (s *CollectionSuite) TestCollectionReplaceFiles(c *check.C) {
 129         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AdminToken}})
 130         foo, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 131                 Attrs: map[string]interface{}{
 132                         "owner_uuid":    arvadostest.ActiveUserUUID,
 133                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
 134                 }})
 135         c.Assert(err, check.IsNil)
 136         s.localdb.signCollection(ctx, &foo)
 137         foobarbaz, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 138                 Attrs: map[string]interface{}{
 139                         "owner_uuid":    arvadostest.ActiveUserUUID,
 140                         "manifest_text": "./foo/bar 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
 141                 }})
 142         c.Assert(err, check.IsNil)
 143         s.localdb.signCollection(ctx, &foobarbaz)
 144         wazqux, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 145                 Attrs: map[string]interface{}{
 146                         "owner_uuid":    arvadostest.ActiveUserUUID,
 147                         "manifest_text": "./waz d85b1213473c2fd7c2045020a6b9c62b+3 0:3:qux.txt\n",
 148                 }})
 149         c.Assert(err, check.IsNil)
 150         s.localdb.signCollection(ctx, &wazqux)
 151
 152         ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 153
 154         // Create using content from existing collections
 155         dst, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 156                 ReplaceFiles: map[string]string{
 157                         "/f": foo.PortableDataHash + "/foo.txt",
 158                         "/b": foobarbaz.PortableDataHash + "/foo/bar",
 159                         "/q": wazqux.PortableDataHash + "/",
 160                         "/w": wazqux.PortableDataHash + "/waz",
 161                 },
 162                 Attrs: map[string]interface{}{
 163                         "owner_uuid": arvadostest.ActiveUserUUID,
 164                 }})
 165         c.Assert(err, check.IsNil)
 166         s.expectFiles(c, dst, "f", "b/baz.txt", "q/waz/qux.txt", "w/qux.txt")
 167
 168         // Delete a file and a directory
 169         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 170                 UUID: dst.UUID,
 171                 ReplaceFiles: map[string]string{
 172                         "/f":     "",
 173                         "/q/waz": "",
 174                 }})
 175         c.Assert(err, check.IsNil)
 176         s.expectFiles(c, dst, "b/baz.txt", "q/", "w/qux.txt")
 177
 178         // Move and copy content within collection
 179         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 180                 UUID: dst.UUID,
 181                 ReplaceFiles: map[string]string{
 182                         // Note splicing content to /b/corge.txt but
 183                         // removing everything else from /b
 184                         "/b":              "",
 185                         "/b/corge.txt":    dst.PortableDataHash + "/b/baz.txt",
 186                         "/quux/corge.txt": dst.PortableDataHash + "/b/baz.txt",
 187                 }})
 188         c.Assert(err, check.IsNil)
 189         s.expectFiles(c, dst, "b/corge.txt", "q/", "w/qux.txt", "quux/corge.txt")
 190
 191         // Remove everything except one file
 192         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 193                 UUID: dst.UUID,
 194                 ReplaceFiles: map[string]string{
 195                         "/":            "",
 196                         "/b/corge.txt": dst.PortableDataHash + "/b/corge.txt",
 197                 }})
 198         c.Assert(err, check.IsNil)
 199         s.expectFiles(c, dst, "b/corge.txt")
 200
 201         // Copy entire collection to root
 202         dstcopy, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 203                 ReplaceFiles: map[string]string{
 204                         "/": dst.PortableDataHash,
 205                 }})
 206         c.Check(err, check.IsNil)
 207         c.Check(dstcopy.PortableDataHash, check.Equals, dst.PortableDataHash)
 208         s.expectFiles(c, dstcopy, "b/corge.txt")
 209
 210         // Check invalid targets, sources, and combinations
 211         for _, badrepl := range []map[string]string{
 212                 {
 213                         "/foo/nope": dst.PortableDataHash + "/b",
 214                         "/foo":      dst.PortableDataHash + "/b",
 215                 },
 216                 {
 217                         "/foo":      dst.PortableDataHash + "/b",
 218                         "/foo/nope": "",
 219                 },
 220                 {
 221                         "/":     dst.PortableDataHash + "/",
 222                         "/nope": "",
 223                 },
 224                 {
 225                         "/":     dst.PortableDataHash + "/",
 226                         "/nope": dst.PortableDataHash + "/b",
 227                 },
 228                 {"/bad/": ""},
 229                 {"/./bad": ""},
 230                 {"/b/./ad": ""},
 231                 {"/b/../ad": ""},
 232                 {"/b/.": ""},
 233                 {".": ""},
 234                 {"bad": ""},
 235                 {"": ""},
 236                 {"/bad": "/b"},
 237                 {"/bad": "bad/b"},
 238                 {"/bad": dst.UUID + "/b"},
 239         } {
 240                 _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 241                         UUID:         dst.UUID,
 242                         ReplaceFiles: badrepl,
 243                 })
 244                 c.Logf("badrepl %#v\n... got err: %s", badrepl, err)
 245                 c.Check(err, check.NotNil)
 246         }
 247
 248         // Check conflicting replace_files and manifest_text
 249         _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 250                 UUID:         dst.UUID,
 251                 ReplaceFiles: map[string]string{"/": ""},
 252                 Attrs: map[string]interface{}{
 253                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:z\n",
 254                 }})
 255         c.Logf("replace_files+manifest_text\n... got err: %s", err)
 256         c.Check(err, check.ErrorMatches, "ambiguous request: both.*replace_files.*manifest_text.*")
 257 }
 258
 259 // expectFiles checks coll's directory structure against the given
 260 // list of expected files and empty directories. An expected path with
 261 // a trailing slash indicates an empty directory.
 262 func (s *CollectionSuite) expectFiles(c *check.C, coll arvados.Collection, expected ...string) {
 263         client := arvados.NewClientFromEnv()
 264         ac, err := arvadosclient.New(client)
 265         c.Assert(err, check.IsNil)
 266         kc, err := keepclient.MakeKeepClient(ac)
 267         c.Assert(err, check.IsNil)
 268         cfs, err := coll.FileSystem(arvados.NewClientFromEnv(), kc)
 269         c.Assert(err, check.IsNil)
 270         var found []string
 271         nonemptydirs := map[string]bool{}
 272         fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
 273                 dir, _ := filepath.Split(path)
 274                 nonemptydirs[dir] = true
 275                 if d.IsDir() {
 276                         if path != "/" {
 277                                 path += "/"
 278                         }
 279                         if !nonemptydirs[path] {
 280                                 nonemptydirs[path] = false
 281                         }
 282                 } else {
 283                         found = append(found, path)
 284                 }
 285                 return nil
 286         })
 287         for d, nonempty := range nonemptydirs {
 288                 if !nonempty {
 289                         found = append(found, d)
 290                 }
 291         }
 292         for i, path := range found {
 293                 if path != "/" {
 294                         found[i] = strings.TrimPrefix(path, "/")
 295                 }
 296         }
 297         sort.Strings(found)
 298         sort.Strings(expected)
 299         c.Check(found, check.DeepEquals, expected)
 300 }
 301
 302 func (s *CollectionSuite) TestSignatures(c *check.C) {
 303         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 304
 305         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 306         c.Check(err, check.IsNil)
 307         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 308         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 309
 310         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
 311         c.Check(err, check.IsNil)
 312         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 313
 314         lresp, err := s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}})
 315         c.Check(err, check.IsNil)
 316         if c.Check(lresp.Items, check.HasLen, 1) {
 317                 c.Check(lresp.Items[0].UUID, check.Equals, arvadostest.FooCollection)
 318                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 319                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 320         }
 321
 322         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"manifest_text"}})
 323         c.Check(err, check.IsNil)
 324         if c.Check(lresp.Items, check.HasLen, 1) {
 325                 c.Check(lresp.Items[0].ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 326                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 327         }
 328
 329         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"unsigned_manifest_text"}})
 330         c.Check(err, check.IsNil)
 331         if c.Check(lresp.Items, check.HasLen, 1) {
 332                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 333                 c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
 334         }
 335
 336         // early trash date causes lower signature TTL (even if
 337         // trash_at and is_trashed fields are unselected)
 338         trashed, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 339                 Select: []string{"uuid", "manifest_text"},
 340                 Attrs: map[string]interface{}{
 341                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
 342                         "trash_at":      time.Now().UTC().Add(time.Hour),
 343                 }})
 344         c.Assert(err, check.IsNil)
 345         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour)
 346         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
 347         c.Assert(err, check.IsNil)
 348         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour)
 349
 350         // distant future trash date does not cause higher signature TTL
 351         trashed, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 352                 UUID: trashed.UUID,
 353                 Attrs: map[string]interface{}{
 354                         "trash_at": time.Now().UTC().Add(time.Hour * 24 * 365),
 355                 }})
 356         c.Assert(err, check.IsNil)
 357         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour*24*7*2)
 358         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
 359         c.Assert(err, check.IsNil)
 360         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 361
 362         // Make sure groups/contents doesn't return manifest_text with
 363         // collections (if it did, we'd need to sign it).
 364         gresp, err := s.localdb.GroupContents(ctx, arvados.GroupContentsOptions{
 365                 Limit:   -1,
 366                 Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}},
 367                 Select:  []string{"uuid", "manifest_text"},
 368         })
 369         if err != nil {
 370                 c.Check(err, check.ErrorMatches, `.*Invalid attribute.*manifest_text.*`)
 371         } else if c.Check(gresp.Items, check.HasLen, 1) {
 372                 c.Check(gresp.Items[0].(map[string]interface{})["uuid"], check.Equals, arvadostest.FooCollection)
 373                 c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
 374         }
 375 }
 376
 377 func (s *CollectionSuite) checkSignatureExpiry(c *check.C, manifestText string, expectedTTL time.Duration) {
 378         m := regexp.MustCompile(`@([[:xdigit:]]+)`).FindStringSubmatch(manifestText)
 379         c.Assert(m, check.HasLen, 2)
 380         sigexp, err := strconv.ParseInt(m[1], 16, 64)
 381         c.Assert(err, check.IsNil)
 382         expectedExp := time.Now().Add(expectedTTL).Unix()
 383         c.Check(sigexp > expectedExp-60, check.Equals, true)
 384         c.Check(sigexp <= expectedExp, check.Equals, true)
 385 }
 386
 387 func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
 388         s.localdb.cluster.Collections.BlobSigning = false
 389         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 390
 391         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 392         c.Check(err, check.IsNil)
 393         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ +]*\+3 0:.*`)
 394 }