lib/controller/localdb/collection_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "context"
   9         "io/fs"
  10         "regexp"
  11         "sort"
  12         "strconv"
  13         "time"
  14
  15         "git.arvados.org/arvados.git/lib/config"
  16         "git.arvados.org/arvados.git/lib/controller/rpc"
  17         "git.arvados.org/arvados.git/sdk/go/arvados"
  18         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  19         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  20         "git.arvados.org/arvados.git/sdk/go/auth"
  21         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  22         "git.arvados.org/arvados.git/sdk/go/keepclient"
  23         check "gopkg.in/check.v1"
  24 )
  25
  26 var _ = check.Suite(&CollectionSuite{})
  27
  28 type CollectionSuite struct {
  29         cluster  *arvados.Cluster
  30         localdb  *Conn
  31         railsSpy *arvadostest.Proxy
  32 }
  33
  34 func (s *CollectionSuite) TearDownSuite(c *check.C) {
  35         // Undo any changes/additions to the user database so they
  36         // don't affect subsequent tests.
  37         arvadostest.ResetEnv()
  38         c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
  39 }
  40
  41 func (s *CollectionSuite) SetUpTest(c *check.C) {
  42         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
  43         c.Assert(err, check.IsNil)
  44         s.cluster, err = cfg.GetCluster("")
  45         c.Assert(err, check.IsNil)
  46         s.localdb = NewConn(s.cluster)
  47         s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
  48         *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
  49 }
  50
  51 func (s *CollectionSuite) TearDownTest(c *check.C) {
  52         s.railsSpy.Close()
  53 }
  54
  55 func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) {
  56         if testVocabulary == "" {
  57                 testVocabulary = `{
  58                         "strict_tags": false,
  59                         "tags": {
  60                                 "IDTAGIMPORTANCES": {
  61                                         "strict": true,
  62                                         "labels": [{"label": "Importance"}, {"label": "Priority"}],
  63                                         "values": {
  64                                                 "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
  65                                                 "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
  66                                                 "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
  67                                         }
  68                                 }
  69                         }
  70                 }`
  71         }
  72         voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
  73         c.Assert(err, check.IsNil)
  74         s.cluster.API.VocabularyPath = "foo"
  75         s.localdb.vocabularyCache = voc
  76 }
  77
  78 func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C) {
  79         s.setUpVocabulary(c, "")
  80         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
  81
  82         tests := []struct {
  83                 name    string
  84                 props   map[string]interface{}
  85                 success bool
  86         }{
  87                 {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
  88                 {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
  89                 {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
  90                 {"Empty properties", map[string]interface{}{}, true},
  91         }
  92         for _, tt := range tests {
  93                 c.Log(c.TestName()+" ", tt.name)
  94
  95                 // Create with properties
  96                 coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
  97                         Select: []string{"uuid", "properties"},
  98                         Attrs: map[string]interface{}{
  99                                 "properties": tt.props,
 100                         }})
 101                 if tt.success {
 102                         c.Assert(err, check.IsNil)
 103                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
 104                 } else {
 105                         c.Assert(err, check.NotNil)
 106                 }
 107
 108                 // Create, then update with properties
 109                 coll, err = s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
 110                 c.Assert(err, check.IsNil)
 111                 coll, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 112                         UUID:   coll.UUID,
 113                         Select: []string{"uuid", "properties"},
 114                         Attrs: map[string]interface{}{
 115                                 "properties": tt.props,
 116                         }})
 117                 if tt.success {
 118                         c.Assert(err, check.IsNil)
 119                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
 120                 } else {
 121                         c.Assert(err, check.NotNil)
 122                 }
 123         }
 124 }
 125
 126 func (s *CollectionSuite) TestCollectionUpdateFiles(c *check.C) {
 127         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AdminToken}})
 128         foo, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 129                 Attrs: map[string]interface{}{
 130                         "owner_uuid":    arvadostest.ActiveUserUUID,
 131                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
 132                 }})
 133         c.Assert(err, check.IsNil)
 134         s.localdb.signCollection(ctx, &foo)
 135         foobarbaz, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 136                 Attrs: map[string]interface{}{
 137                         "owner_uuid":    arvadostest.ActiveUserUUID,
 138                         "manifest_text": "./foo/bar 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
 139                 }})
 140         c.Assert(err, check.IsNil)
 141         s.localdb.signCollection(ctx, &foobarbaz)
 142         wazqux, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
 143                 Attrs: map[string]interface{}{
 144                         "owner_uuid":    arvadostest.ActiveUserUUID,
 145                         "manifest_text": "./waz d85b1213473c2fd7c2045020a6b9c62b+3 0:3:qux.txt\n",
 146                 }})
 147         c.Assert(err, check.IsNil)
 148         s.localdb.signCollection(ctx, &wazqux)
 149
 150         ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 151
 152         // Create using content from existing collections
 153         dst, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 154                 Attrs: map[string]interface{}{
 155                         "owner_uuid": arvadostest.ActiveUserUUID,
 156                         "splices": map[string]string{
 157                                 "/f": foo.PortableDataHash + "/foo.txt",
 158                                 "/b": foobarbaz.PortableDataHash + "/foo/bar",
 159                                 "/q": wazqux.PortableDataHash + "/",
 160                                 "/w": wazqux.PortableDataHash + "/waz",
 161                         },
 162                 }})
 163         c.Assert(err, check.IsNil)
 164         s.expectFiles(c, dst, "f", "b/baz.txt", "q/waz/qux.txt", "w/qux.txt")
 165
 166         // Delete a file and a directory
 167         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 168                 UUID: dst.UUID,
 169                 Attrs: map[string]interface{}{
 170                         "splices": map[string]string{
 171                                 "/f":     "",
 172                                 "/q/waz": "",
 173                         },
 174                 }})
 175         c.Assert(err, check.IsNil)
 176         s.expectFiles(c, dst, "b/baz.txt", "q/", "w/qux.txt")
 177
 178         // Move content within collection
 179         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 180                 UUID: dst.UUID,
 181                 Attrs: map[string]interface{}{
 182                         "splices": map[string]string{
 183                                 "/b":              "",
 184                                 "/quux/corge.txt": dst.PortableDataHash + "/b/baz.txt",
 185                         },
 186                 }})
 187         c.Assert(err, check.IsNil)
 188         s.expectFiles(c, dst, "q/", "w/qux.txt", "quux/corge.txt")
 189 }
 190
 191 // Wrap arvados.FileSystem to satisfy the fs.FS interface (until the
 192 // SDK offers a neater solution) so we can use fs.WalkDir().
 193 type filesystemfs struct {
 194         arvados.FileSystem
 195 }
 196
 197 func (fs filesystemfs) Open(path string) (fs.File, error) {
 198         f, err := fs.FileSystem.Open(path)
 199         return f, err
 200 }
 201
 202 func (s *CollectionSuite) expectFiles(c *check.C, coll arvados.Collection, expected ...string) {
 203         client := arvados.NewClientFromEnv()
 204         ac, err := arvadosclient.New(client)
 205         c.Assert(err, check.IsNil)
 206         kc, err := keepclient.MakeKeepClient(ac)
 207         c.Assert(err, check.IsNil)
 208         cfs, err := coll.FileSystem(arvados.NewClientFromEnv(), kc)
 209         c.Assert(err, check.IsNil)
 210         var found []string
 211         fs.WalkDir(filesystemfs{cfs}, "/", func(path string, d fs.DirEntry, err error) error {
 212                 found = append(found, path)
 213                 return nil
 214         })
 215         sort.Strings(found)
 216         sort.Strings(expected)
 217         c.Check(found, check.DeepEquals, expected)
 218 }
 219
 220 func (s *CollectionSuite) TestSignatures(c *check.C) {
 221         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 222
 223         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 224         c.Check(err, check.IsNil)
 225         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 226         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 227
 228         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
 229         c.Check(err, check.IsNil)
 230         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 231
 232         lresp, err := s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}})
 233         c.Check(err, check.IsNil)
 234         if c.Check(lresp.Items, check.HasLen, 1) {
 235                 c.Check(lresp.Items[0].UUID, check.Equals, arvadostest.FooCollection)
 236                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 237                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 238         }
 239
 240         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"manifest_text"}})
 241         c.Check(err, check.IsNil)
 242         if c.Check(lresp.Items, check.HasLen, 1) {
 243                 c.Check(lresp.Items[0].ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 244                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 245         }
 246
 247         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"unsigned_manifest_text"}})
 248         c.Check(err, check.IsNil)
 249         if c.Check(lresp.Items, check.HasLen, 1) {
 250                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 251                 c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
 252         }
 253
 254         // early trash date causes lower signature TTL (even if
 255         // trash_at and is_trashed fields are unselected)
 256         trashed, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 257                 Select: []string{"uuid", "manifest_text"},
 258                 Attrs: map[string]interface{}{
 259                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
 260                         "trash_at":      time.Now().UTC().Add(time.Hour),
 261                 }})
 262         c.Assert(err, check.IsNil)
 263         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour)
 264         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
 265         c.Assert(err, check.IsNil)
 266         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour)
 267
 268         // distant future trash date does not cause higher signature TTL
 269         trashed, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 270                 UUID: trashed.UUID,
 271                 Attrs: map[string]interface{}{
 272                         "trash_at": time.Now().UTC().Add(time.Hour * 24 * 365),
 273                 }})
 274         c.Assert(err, check.IsNil)
 275         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour*24*7*2)
 276         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
 277         c.Assert(err, check.IsNil)
 278         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 279
 280         // Make sure groups/contents doesn't return manifest_text with
 281         // collections (if it did, we'd need to sign it).
 282         gresp, err := s.localdb.GroupContents(ctx, arvados.GroupContentsOptions{
 283                 Limit:   -1,
 284                 Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}},
 285                 Select:  []string{"uuid", "manifest_text"},
 286         })
 287         if err != nil {
 288                 c.Check(err, check.ErrorMatches, `.*Invalid attribute.*manifest_text.*`)
 289         } else if c.Check(gresp.Items, check.HasLen, 1) {
 290                 c.Check(gresp.Items[0].(map[string]interface{})["uuid"], check.Equals, arvadostest.FooCollection)
 291                 c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
 292         }
 293 }
 294
 295 func (s *CollectionSuite) checkSignatureExpiry(c *check.C, manifestText string, expectedTTL time.Duration) {
 296         m := regexp.MustCompile(`@([[:xdigit:]]+)`).FindStringSubmatch(manifestText)
 297         c.Assert(m, check.HasLen, 2)
 298         sigexp, err := strconv.ParseInt(m[1], 16, 64)
 299         c.Assert(err, check.IsNil)
 300         expectedExp := time.Now().Add(expectedTTL).Unix()
 301         c.Check(sigexp > expectedExp-60, check.Equals, true)
 302         c.Check(sigexp <= expectedExp, check.Equals, true)
 303 }
 304
 305 func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
 306         s.localdb.cluster.Collections.BlobSigning = false
 307         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 308
 309         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 310         c.Check(err, check.IsNil)
 311         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ +]*\+3 0:.*`)
 312 }