lib/controller/localdb/collection_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "context"
   9         "io/fs"
  10         "path/filepath"
  11         "regexp"
  12         "sort"
  13         "strconv"
  14         "strings"
  15         "time"
  16
  17         "git.arvados.org/arvados.git/lib/config"
  18         "git.arvados.org/arvados.git/lib/controller/rpc"
  19         "git.arvados.org/arvados.git/sdk/go/arvados"
  20         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  21         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  22         "git.arvados.org/arvados.git/sdk/go/auth"
  23         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  24         "git.arvados.org/arvados.git/sdk/go/keepclient"
  25         check "gopkg.in/check.v1"
  26 )
  27
  28 var _ = check.Suite(&CollectionSuite{})
  29
  30 type CollectionSuite struct {
  31         cluster  *arvados.Cluster
  32         localdb  *Conn
  33         railsSpy *arvadostest.Proxy
  34 }
  35
  36 func (s *CollectionSuite) TearDownSuite(c *check.C) {
  37         // Undo any changes/additions to the user database so they
  38         // don't affect subsequent tests.
  39         arvadostest.ResetEnv()
  40         c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
  41 }
  42
  43 func (s *CollectionSuite) SetUpTest(c *check.C) {
  44         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
  45         c.Assert(err, check.IsNil)
  46         s.cluster, err = cfg.GetCluster("")
  47         c.Assert(err, check.IsNil)
  48         s.localdb = NewConn(s.cluster)
  49         s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
  50         *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
  51 }
  52
  53 func (s *CollectionSuite) TearDownTest(c *check.C) {
  54         s.railsSpy.Close()
  55 }
  56
  57 func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) {
  58         if testVocabulary == "" {
  59                 testVocabulary = `{
  60                         "strict_tags": false,
  61                         "tags": {
  62                                 "IDTAGIMPORTANCES": {
  63                                         "strict": true,
  64                                         "labels": [{"label": "Importance"}, {"label": "Priority"}],
  65                                         "values": {
  66                                                 "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
  67                                                 "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
  68                                                 "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
  69                                         }
  70                                 }
  71                         }
  72                 }`
  73         }
  74         voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
  75         c.Assert(err, check.IsNil)
  76         s.cluster.API.VocabularyPath = "foo"
  77         s.localdb.vocabularyCache = voc
  78 }
  79
  80 func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C) {
  81         s.setUpVocabulary(c, "")
  82         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
  83
  84         tests := []struct {
  85                 name    string
  86                 props   map[string]interface{}
  87                 success bool
  88         }{
  89                 {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
  90                 {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
  91                 {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
  92                 {"Empty properties", map[string]interface{}{}, true},
  93         }
  94         for _, tt := range tests {
  95                 c.Log(c.TestName()+" ", tt.name)
  96
  97                 // Create with properties
  98                 coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
  99                         Select: []string{"uuid", "properties"},
 100                         Attrs: map[string]interface{}{
 101                                 "properties": tt.props,
 102                         }})
 103                 if tt.success {
 104                         c.Assert(err, check.IsNil)
 105                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
 106                 } else {
 107                         c.Assert(err, check.NotNil)
 108                 }
 109
 110                 // Create, then update with properties
 111                 coll, err = s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
 112                 c.Assert(err, check.IsNil)
 113                 coll, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 114                         UUID:   coll.UUID,
 115                         Select: []string{"uuid", "properties"},
 116                         Attrs: map[string]interface{}{
 117                                 "properties": tt.props,
 118                         }})
 119                 if tt.success {
 120                         c.Assert(err, check.IsNil)
 121                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
 122                 } else {
 123                         c.Assert(err, check.NotNil)
 124                 }
 125         }
 126 }
 127
 128 func (s *CollectionSuite) TestCollectionUpdateFiles(c *check.C) {
 129         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AdminToken}})
 130         foo, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 131                 Attrs: map[string]interface{}{
 132                         "owner_uuid":    arvadostest.ActiveUserUUID,
 133                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
 134                 }})
 135         c.Assert(err, check.IsNil)
 136         s.localdb.signCollection(ctx, &foo)
 137         foobarbaz, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 138                 Attrs: map[string]interface{}{
 139                         "owner_uuid":    arvadostest.ActiveUserUUID,
 140                         "manifest_text": "./foo/bar 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
 141                 }})
 142         c.Assert(err, check.IsNil)
 143         s.localdb.signCollection(ctx, &foobarbaz)
 144         wazqux, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 145                 Attrs: map[string]interface{}{
 146                         "owner_uuid":    arvadostest.ActiveUserUUID,
 147                         "manifest_text": "./waz d85b1213473c2fd7c2045020a6b9c62b+3 0:3:qux.txt\n",
 148                 }})
 149         c.Assert(err, check.IsNil)
 150         s.localdb.signCollection(ctx, &wazqux)
 151
 152         ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 153
 154         // Create using content from existing collections
 155         dst, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 156                 Attrs: map[string]interface{}{
 157                         "owner_uuid": arvadostest.ActiveUserUUID,
 158                         "splices": map[string]string{
 159                                 "/f": foo.PortableDataHash + "/foo.txt",
 160                                 "/b": foobarbaz.PortableDataHash + "/foo/bar",
 161                                 "/q": wazqux.PortableDataHash + "/",
 162                                 "/w": wazqux.PortableDataHash + "/waz",
 163                         },
 164                 }})
 165         c.Assert(err, check.IsNil)
 166         s.expectFiles(c, dst, "f", "b/baz.txt", "q/waz/qux.txt", "w/qux.txt")
 167
 168         // Delete a file and a directory
 169         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 170                 UUID: dst.UUID,
 171                 Attrs: map[string]interface{}{
 172                         "splices": map[string]string{
 173                                 "/f":     "",
 174                                 "/q/waz": "",
 175                         },
 176                 }})
 177         c.Assert(err, check.IsNil)
 178         s.expectFiles(c, dst, "b/baz.txt", "q/", "w/qux.txt")
 179
 180         // Move and copy content within collection
 181         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 182                 UUID: dst.UUID,
 183                 Attrs: map[string]interface{}{
 184                         "splices": map[string]string{
 185                                 // Note splicing content to
 186                                 // /b/corge.txt but removing
 187                                 // everything else from /b
 188                                 "/b":              "",
 189                                 "/b/corge.txt":    dst.PortableDataHash + "/b/baz.txt",
 190                                 "/quux/corge.txt": dst.PortableDataHash + "/b/baz.txt",
 191                         },
 192                 }})
 193         c.Assert(err, check.IsNil)
 194         s.expectFiles(c, dst, "b/corge.txt", "q/", "w/qux.txt", "quux/corge.txt")
 195
 196         // Remove everything except one file
 197         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 198                 UUID: dst.UUID,
 199                 Attrs: map[string]interface{}{
 200                         "splices": map[string]string{
 201                                 "/":            "",
 202                                 "/b/corge.txt": dst.PortableDataHash + "/b/corge.txt",
 203                         },
 204                 }})
 205         c.Assert(err, check.IsNil)
 206         s.expectFiles(c, dst, "b/corge.txt")
 207
 208         // Copy entire collection to root
 209         dstcopy, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 210                 Attrs: map[string]interface{}{
 211                         // Note map[string]interface{} here, which is
 212                         // how lib/controller/router requests will
 213                         // look.
 214                         "splices": map[string]interface{}{
 215                                 "/": dst.PortableDataHash,
 216                         },
 217                 }})
 218         c.Check(err, check.IsNil)
 219         c.Check(dstcopy.PortableDataHash, check.Equals, dst.PortableDataHash)
 220         s.expectFiles(c, dstcopy, "b/corge.txt")
 221
 222         // Check invalid targets, sources, and combinations
 223         for _, splices := range []map[string]string{
 224                 {
 225                         "/foo/nope": dst.PortableDataHash + "/b",
 226                         "/foo":      dst.PortableDataHash + "/b",
 227                 },
 228                 {
 229                         "/foo":      dst.PortableDataHash + "/b",
 230                         "/foo/nope": "",
 231                 },
 232                 {
 233                         "/":     dst.PortableDataHash + "/",
 234                         "/nope": "",
 235                 },
 236                 {
 237                         "/":     dst.PortableDataHash + "/",
 238                         "/nope": dst.PortableDataHash + "/b",
 239                 },
 240                 {"/bad/": ""},
 241                 {"/./bad": ""},
 242                 {"/b/./ad": ""},
 243                 {"/b/../ad": ""},
 244                 {"/b/.": ""},
 245                 {".": ""},
 246                 {"bad": ""},
 247                 {"": ""},
 248                 {"/bad": "/b"},
 249                 {"/bad": "bad/b"},
 250                 {"/bad": dst.UUID + "/b"},
 251         } {
 252                 _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 253                         UUID: dst.UUID,
 254                         Attrs: map[string]interface{}{
 255                                 "splices": splices,
 256                         }})
 257                 c.Logf("splices %#v\n... got err: %s", splices, err)
 258                 c.Check(err, check.NotNil)
 259         }
 260
 261         // Check "splices" value that isn't even the right type
 262         for _, splices := range []interface{}{
 263                 map[string]int{"foo": 1},
 264                 map[int]string{1: "foo"},
 265                 12345,
 266                 "foo",
 267                 []string{"foo"},
 268         } {
 269                 _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 270                         UUID: dst.UUID,
 271                         Attrs: map[string]interface{}{
 272                                 "splices": splices,
 273                         }})
 274                 c.Logf("splices %#v\n... got err: %s", splices, err)
 275                 c.Check(err, check.ErrorMatches, "invalid type .* for splices parameter")
 276         }
 277
 278         // Check conflicting splices and manifest_text
 279         _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 280                 UUID: dst.UUID,
 281                 Attrs: map[string]interface{}{
 282                         "splices":       map[string]string{"/": ""},
 283                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:z\n",
 284                 }})
 285         c.Logf("splices+manifest_text\n... got err: %s", err)
 286         c.Check(err, check.ErrorMatches, "ambiguous request: both.*splices.*manifest_text.*")
 287 }
 288
 289 // expectFiles checks coll's directory structure against the given
 290 // list of expected files and empty directories. An expected path with
 291 // a trailing slash indicates an empty directory.
 292 func (s *CollectionSuite) expectFiles(c *check.C, coll arvados.Collection, expected ...string) {
 293         client := arvados.NewClientFromEnv()
 294         ac, err := arvadosclient.New(client)
 295         c.Assert(err, check.IsNil)
 296         kc, err := keepclient.MakeKeepClient(ac)
 297         c.Assert(err, check.IsNil)
 298         cfs, err := coll.FileSystem(arvados.NewClientFromEnv(), kc)
 299         c.Assert(err, check.IsNil)
 300         var found []string
 301         nonemptydirs := map[string]bool{}
 302         fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
 303                 dir, _ := filepath.Split(path)
 304                 nonemptydirs[dir] = true
 305                 if d.IsDir() {
 306                         if path != "/" {
 307                                 path += "/"
 308                         }
 309                         if !nonemptydirs[path] {
 310                                 nonemptydirs[path] = false
 311                         }
 312                 } else {
 313                         found = append(found, path)
 314                 }
 315                 return nil
 316         })
 317         for d, nonempty := range nonemptydirs {
 318                 if !nonempty {
 319                         found = append(found, d)
 320                 }
 321         }
 322         for i, path := range found {
 323                 if path != "/" {
 324                         found[i] = strings.TrimPrefix(path, "/")
 325                 }
 326         }
 327         sort.Strings(found)
 328         sort.Strings(expected)
 329         c.Check(found, check.DeepEquals, expected)
 330 }
 331
 332 func (s *CollectionSuite) TestSignatures(c *check.C) {
 333         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 334
 335         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 336         c.Check(err, check.IsNil)
 337         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 338         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 339
 340         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
 341         c.Check(err, check.IsNil)
 342         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 343
 344         lresp, err := s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}})
 345         c.Check(err, check.IsNil)
 346         if c.Check(lresp.Items, check.HasLen, 1) {
 347                 c.Check(lresp.Items[0].UUID, check.Equals, arvadostest.FooCollection)
 348                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 349                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 350         }
 351
 352         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"manifest_text"}})
 353         c.Check(err, check.IsNil)
 354         if c.Check(lresp.Items, check.HasLen, 1) {
 355                 c.Check(lresp.Items[0].ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 356                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 357         }
 358
 359         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"unsigned_manifest_text"}})
 360         c.Check(err, check.IsNil)
 361         if c.Check(lresp.Items, check.HasLen, 1) {
 362                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 363                 c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
 364         }
 365
 366         // early trash date causes lower signature TTL (even if
 367         // trash_at and is_trashed fields are unselected)
 368         trashed, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 369                 Select: []string{"uuid", "manifest_text"},
 370                 Attrs: map[string]interface{}{
 371                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
 372                         "trash_at":      time.Now().UTC().Add(time.Hour),
 373                 }})
 374         c.Assert(err, check.IsNil)
 375         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour)
 376         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
 377         c.Assert(err, check.IsNil)
 378         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour)
 379
 380         // distant future trash date does not cause higher signature TTL
 381         trashed, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 382                 UUID: trashed.UUID,
 383                 Attrs: map[string]interface{}{
 384                         "trash_at": time.Now().UTC().Add(time.Hour * 24 * 365),
 385                 }})
 386         c.Assert(err, check.IsNil)
 387         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour*24*7*2)
 388         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
 389         c.Assert(err, check.IsNil)
 390         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 391
 392         // Make sure groups/contents doesn't return manifest_text with
 393         // collections (if it did, we'd need to sign it).
 394         gresp, err := s.localdb.GroupContents(ctx, arvados.GroupContentsOptions{
 395                 Limit:   -1,
 396                 Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}},
 397                 Select:  []string{"uuid", "manifest_text"},
 398         })
 399         if err != nil {
 400                 c.Check(err, check.ErrorMatches, `.*Invalid attribute.*manifest_text.*`)
 401         } else if c.Check(gresp.Items, check.HasLen, 1) {
 402                 c.Check(gresp.Items[0].(map[string]interface{})["uuid"], check.Equals, arvadostest.FooCollection)
 403                 c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
 404         }
 405 }
 406
 407 func (s *CollectionSuite) checkSignatureExpiry(c *check.C, manifestText string, expectedTTL time.Duration) {
 408         m := regexp.MustCompile(`@([[:xdigit:]]+)`).FindStringSubmatch(manifestText)
 409         c.Assert(m, check.HasLen, 2)
 410         sigexp, err := strconv.ParseInt(m[1], 16, 64)
 411         c.Assert(err, check.IsNil)
 412         expectedExp := time.Now().Add(expectedTTL).Unix()
 413         c.Check(sigexp > expectedExp-60, check.Equals, true)
 414         c.Check(sigexp <= expectedExp, check.Equals, true)
 415 }
 416
 417 func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
 418         s.localdb.cluster.Collections.BlobSigning = false
 419         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 420
 421         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 422         c.Check(err, check.IsNil)
 423         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ +]*\+3 0:.*`)
 424 }