lib/controller/localdb/collection_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "context"
   9         "io/fs"
  10         "path/filepath"
  11         "regexp"
  12         "sort"
  13         "strconv"
  14         "strings"
  15         "time"
  16
  17         "git.arvados.org/arvados.git/lib/config"
  18         "git.arvados.org/arvados.git/lib/controller/rpc"
  19         "git.arvados.org/arvados.git/sdk/go/arvados"
  20         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  21         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  22         "git.arvados.org/arvados.git/sdk/go/auth"
  23         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  24         "git.arvados.org/arvados.git/sdk/go/keepclient"
  25         check "gopkg.in/check.v1"
  26 )
  27
  28 var _ = check.Suite(&CollectionSuite{})
  29
  30 type CollectionSuite struct {
  31         cluster  *arvados.Cluster
  32         localdb  *Conn
  33         railsSpy *arvadostest.Proxy
  34 }
  35
  36 func (s *CollectionSuite) TearDownSuite(c *check.C) {
  37         // Undo any changes/additions to the user database so they
  38         // don't affect subsequent tests.
  39         arvadostest.ResetEnv()
  40         c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
  41 }
  42
  43 func (s *CollectionSuite) SetUpTest(c *check.C) {
  44         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
  45         c.Assert(err, check.IsNil)
  46         s.cluster, err = cfg.GetCluster("")
  47         c.Assert(err, check.IsNil)
  48         s.localdb = NewConn(s.cluster)
  49         s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
  50         *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
  51 }
  52
  53 func (s *CollectionSuite) TearDownTest(c *check.C) {
  54         s.railsSpy.Close()
  55 }
  56
  57 func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) {
  58         if testVocabulary == "" {
  59                 testVocabulary = `{
  60                         "strict_tags": false,
  61                         "tags": {
  62                                 "IDTAGIMPORTANCES": {
  63                                         "strict": true,
  64                                         "labels": [{"label": "Importance"}, {"label": "Priority"}],
  65                                         "values": {
  66                                                 "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
  67                                                 "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
  68                                                 "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
  69                                         }
  70                                 }
  71                         }
  72                 }`
  73         }
  74         voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
  75         c.Assert(err, check.IsNil)
  76         s.cluster.API.VocabularyPath = "foo"
  77         s.localdb.vocabularyCache = voc
  78 }
  79
  80 func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C) {
  81         s.setUpVocabulary(c, "")
  82         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
  83
  84         tests := []struct {
  85                 name    string
  86                 props   map[string]interface{}
  87                 success bool
  88         }{
  89                 {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
  90                 {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
  91                 {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
  92                 {"Empty properties", map[string]interface{}{}, true},
  93         }
  94         for _, tt := range tests {
  95                 c.Log(c.TestName()+" ", tt.name)
  96
  97                 // Create with properties
  98                 coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
  99                         Select: []string{"uuid", "properties"},
 100                         Attrs: map[string]interface{}{
 101                                 "properties": tt.props,
 102                         }})
 103                 if tt.success {
 104                         c.Assert(err, check.IsNil)
 105                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
 106                 } else {
 107                         c.Assert(err, check.NotNil)
 108                 }
 109
 110                 // Create, then update with properties
 111                 coll, err = s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
 112                 c.Assert(err, check.IsNil)
 113                 coll, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 114                         UUID:   coll.UUID,
 115                         Select: []string{"uuid", "properties"},
 116                         Attrs: map[string]interface{}{
 117                                 "properties": tt.props,
 118                         }})
 119                 if tt.success {
 120                         c.Assert(err, check.IsNil)
 121                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
 122                 } else {
 123                         c.Assert(err, check.NotNil)
 124                 }
 125         }
 126 }
 127
 128 func (s *CollectionSuite) TestCollectionUpdateFiles(c *check.C) {
 129         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AdminToken}})
 130         foo, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 131                 Attrs: map[string]interface{}{
 132                         "owner_uuid":    arvadostest.ActiveUserUUID,
 133                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
 134                 }})
 135         c.Assert(err, check.IsNil)
 136         s.localdb.signCollection(ctx, &foo)
 137         foobarbaz, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 138                 Attrs: map[string]interface{}{
 139                         "owner_uuid":    arvadostest.ActiveUserUUID,
 140                         "manifest_text": "./foo/bar 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
 141                 }})
 142         c.Assert(err, check.IsNil)
 143         s.localdb.signCollection(ctx, &foobarbaz)
 144         wazqux, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 145                 Attrs: map[string]interface{}{
 146                         "owner_uuid":    arvadostest.ActiveUserUUID,
 147                         "manifest_text": "./waz d85b1213473c2fd7c2045020a6b9c62b+3 0:3:qux.txt\n",
 148                 }})
 149         c.Assert(err, check.IsNil)
 150         s.localdb.signCollection(ctx, &wazqux)
 151
 152         ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 153
 154         // Create using content from existing collections
 155         dst, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 156                 Attrs: map[string]interface{}{
 157                         "owner_uuid": arvadostest.ActiveUserUUID,
 158                         "splices": map[string]string{
 159                                 "/f": foo.PortableDataHash + "/foo.txt",
 160                                 "/b": foobarbaz.PortableDataHash + "/foo/bar",
 161                                 "/q": wazqux.PortableDataHash + "/",
 162                                 "/w": wazqux.PortableDataHash + "/waz",
 163                         },
 164                 }})
 165         c.Assert(err, check.IsNil)
 166         s.expectFiles(c, dst, "f", "b/baz.txt", "q/waz/qux.txt", "w/qux.txt")
 167
 168         // Delete a file and a directory
 169         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 170                 UUID: dst.UUID,
 171                 Attrs: map[string]interface{}{
 172                         "splices": map[string]string{
 173                                 "/f":     "",
 174                                 "/q/waz": "",
 175                         },
 176                 }})
 177         c.Assert(err, check.IsNil)
 178         s.expectFiles(c, dst, "b/baz.txt", "q/", "w/qux.txt")
 179
 180         // Move and copy content within collection
 181         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 182                 UUID: dst.UUID,
 183                 Attrs: map[string]interface{}{
 184                         "splices": map[string]string{
 185                                 // Note splicing content to
 186                                 // /b/corge.txt but removing
 187                                 // everything else from /b
 188                                 "/b":              "",
 189                                 "/b/corge.txt":    dst.PortableDataHash + "/b/baz.txt",
 190                                 "/quux/corge.txt": dst.PortableDataHash + "/b/baz.txt",
 191                         },
 192                 }})
 193         c.Assert(err, check.IsNil)
 194         s.expectFiles(c, dst, "b/corge.txt", "q/", "w/qux.txt", "quux/corge.txt")
 195
 196         // Remove everything except one file
 197         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 198                 UUID: dst.UUID,
 199                 Attrs: map[string]interface{}{
 200                         "splices": map[string]string{
 201                                 "/":            "",
 202                                 "/b/corge.txt": dst.PortableDataHash + "/b/corge.txt",
 203                         },
 204                 }})
 205         c.Assert(err, check.IsNil)
 206         s.expectFiles(c, dst, "b/corge.txt")
 207
 208         // Copy entire collection to root
 209         dstcopy, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 210                 Attrs: map[string]interface{}{
 211                         // Note map[string]interface{} here, which is
 212                         // how lib/controller/router requests will
 213                         // look.
 214                         "splices": map[string]interface{}{
 215                                 "/": dst.PortableDataHash,
 216                         },
 217                 }})
 218         c.Check(err, check.IsNil)
 219         c.Check(dstcopy.PortableDataHash, check.Equals, dst.PortableDataHash)
 220         s.expectFiles(c, dstcopy, "b/corge.txt")
 221
 222         for _, splices := range []map[string]string{
 223                 {
 224                         "/foo/nope": dst.PortableDataHash + "/b",
 225                         "/foo":      dst.PortableDataHash + "/b",
 226                 },
 227                 {
 228                         "/foo":      dst.PortableDataHash + "/b",
 229                         "/foo/nope": "",
 230                 },
 231                 {
 232                         "/":     dst.PortableDataHash + "/",
 233                         "/nope": "",
 234                 },
 235                 {
 236                         "/":     dst.PortableDataHash + "/",
 237                         "/nope": dst.PortableDataHash + "/b",
 238                 },
 239                 {"/bad/": ""},
 240                 {"/./bad": ""},
 241                 {"/b/./ad": ""},
 242                 {"/b/../ad": ""},
 243                 {"/b/.": ""},
 244                 {".": ""},
 245                 {"bad": ""},
 246                 {"": ""},
 247                 {"/bad": "/b"},
 248                 {"/bad": "bad/b"},
 249                 {"/bad": dst.UUID + "/b"},
 250         } {
 251                 _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 252                         UUID: dst.UUID,
 253                         Attrs: map[string]interface{}{
 254                                 "splices": splices,
 255                         }})
 256                 c.Logf("splices %#v\n... got err: %s", splices, err)
 257                 c.Check(err, check.NotNil)
 258         }
 259         for _, splices := range []interface{}{
 260                 map[string]int{"foo": 1},
 261                 map[int]string{1: "foo"},
 262         } {
 263                 _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 264                         UUID: dst.UUID,
 265                         Attrs: map[string]interface{}{
 266                                 "splices": splices,
 267                         }})
 268                 c.Logf("splices %#v\n... got err: %s", splices, err)
 269                 c.Check(err, check.NotNil)
 270         }
 271 }
 272
 273 // expectFiles checks coll's directory structure against the given
 274 // list of expected files and empty directories. An expected path with
 275 // a trailing slash indicates an empty directory.
 276 func (s *CollectionSuite) expectFiles(c *check.C, coll arvados.Collection, expected ...string) {
 277         client := arvados.NewClientFromEnv()
 278         ac, err := arvadosclient.New(client)
 279         c.Assert(err, check.IsNil)
 280         kc, err := keepclient.MakeKeepClient(ac)
 281         c.Assert(err, check.IsNil)
 282         cfs, err := coll.FileSystem(arvados.NewClientFromEnv(), kc)
 283         c.Assert(err, check.IsNil)
 284         var found []string
 285         nonemptydirs := map[string]bool{}
 286         fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
 287                 dir, _ := filepath.Split(path)
 288                 nonemptydirs[dir] = true
 289                 if d.IsDir() {
 290                         if path != "/" {
 291                                 path += "/"
 292                         }
 293                         if !nonemptydirs[path] {
 294                                 nonemptydirs[path] = false
 295                         }
 296                 } else {
 297                         found = append(found, path)
 298                 }
 299                 return nil
 300         })
 301         for d, nonempty := range nonemptydirs {
 302                 if !nonempty {
 303                         found = append(found, d)
 304                 }
 305         }
 306         for i, path := range found {
 307                 if path != "/" {
 308                         found[i] = strings.TrimPrefix(path, "/")
 309                 }
 310         }
 311         sort.Strings(found)
 312         sort.Strings(expected)
 313         c.Check(found, check.DeepEquals, expected)
 314 }
 315
 316 func (s *CollectionSuite) TestSignatures(c *check.C) {
 317         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 318
 319         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 320         c.Check(err, check.IsNil)
 321         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 322         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 323
 324         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
 325         c.Check(err, check.IsNil)
 326         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 327
 328         lresp, err := s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}})
 329         c.Check(err, check.IsNil)
 330         if c.Check(lresp.Items, check.HasLen, 1) {
 331                 c.Check(lresp.Items[0].UUID, check.Equals, arvadostest.FooCollection)
 332                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 333                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 334         }
 335
 336         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"manifest_text"}})
 337         c.Check(err, check.IsNil)
 338         if c.Check(lresp.Items, check.HasLen, 1) {
 339                 c.Check(lresp.Items[0].ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 340                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 341         }
 342
 343         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"unsigned_manifest_text"}})
 344         c.Check(err, check.IsNil)
 345         if c.Check(lresp.Items, check.HasLen, 1) {
 346                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 347                 c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
 348         }
 349
 350         // early trash date causes lower signature TTL (even if
 351         // trash_at and is_trashed fields are unselected)
 352         trashed, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 353                 Select: []string{"uuid", "manifest_text"},
 354                 Attrs: map[string]interface{}{
 355                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
 356                         "trash_at":      time.Now().UTC().Add(time.Hour),
 357                 }})
 358         c.Assert(err, check.IsNil)
 359         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour)
 360         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
 361         c.Assert(err, check.IsNil)
 362         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour)
 363
 364         // distant future trash date does not cause higher signature TTL
 365         trashed, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 366                 UUID: trashed.UUID,
 367                 Attrs: map[string]interface{}{
 368                         "trash_at": time.Now().UTC().Add(time.Hour * 24 * 365),
 369                 }})
 370         c.Assert(err, check.IsNil)
 371         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour*24*7*2)
 372         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
 373         c.Assert(err, check.IsNil)
 374         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 375
 376         // Make sure groups/contents doesn't return manifest_text with
 377         // collections (if it did, we'd need to sign it).
 378         gresp, err := s.localdb.GroupContents(ctx, arvados.GroupContentsOptions{
 379                 Limit:   -1,
 380                 Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}},
 381                 Select:  []string{"uuid", "manifest_text"},
 382         })
 383         if err != nil {
 384                 c.Check(err, check.ErrorMatches, `.*Invalid attribute.*manifest_text.*`)
 385         } else if c.Check(gresp.Items, check.HasLen, 1) {
 386                 c.Check(gresp.Items[0].(map[string]interface{})["uuid"], check.Equals, arvadostest.FooCollection)
 387                 c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
 388         }
 389 }
 390
 391 func (s *CollectionSuite) checkSignatureExpiry(c *check.C, manifestText string, expectedTTL time.Duration) {
 392         m := regexp.MustCompile(`@([[:xdigit:]]+)`).FindStringSubmatch(manifestText)
 393         c.Assert(m, check.HasLen, 2)
 394         sigexp, err := strconv.ParseInt(m[1], 16, 64)
 395         c.Assert(err, check.IsNil)
 396         expectedExp := time.Now().Add(expectedTTL).Unix()
 397         c.Check(sigexp > expectedExp-60, check.Equals, true)
 398         c.Check(sigexp <= expectedExp, check.Equals, true)
 399 }
 400
 401 func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
 402         s.localdb.cluster.Collections.BlobSigning = false
 403         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 404
 405         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 406         c.Check(err, check.IsNil)
 407         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ +]*\+3 0:.*`)
 408 }