lib/controller/localdb/collection_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "io/fs"
   9         "path/filepath"
  10         "regexp"
  11         "sort"
  12         "strconv"
  13         "strings"
  14         "sync"
  15         "time"
  16
  17         "git.arvados.org/arvados.git/lib/ctrlctx"
  18         "git.arvados.org/arvados.git/sdk/go/arvados"
  19         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  20         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  21         "git.arvados.org/arvados.git/sdk/go/keepclient"
  22         check "gopkg.in/check.v1"
  23 )
  24
  25 var _ = check.Suite(&CollectionSuite{})
  26
  27 type CollectionSuite struct {
  28         localdbSuite
  29 }
  30
  31 func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C) {
  32         s.setUpVocabulary(c, "")
  33
  34         tests := []struct {
  35                 name    string
  36                 props   map[string]interface{}
  37                 success bool
  38         }{
  39                 {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
  40                 {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
  41                 {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
  42                 {"Empty properties", map[string]interface{}{}, true},
  43         }
  44         for _, tt := range tests {
  45                 c.Log(c.TestName()+" ", tt.name)
  46
  47                 // Create with properties
  48                 coll, err := s.localdb.CollectionCreate(s.userctx, arvados.CreateOptions{
  49                         Select: []string{"uuid", "properties"},
  50                         Attrs: map[string]interface{}{
  51                                 "properties": tt.props,
  52                         }})
  53                 if tt.success {
  54                         c.Assert(err, check.IsNil)
  55                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
  56                 } else {
  57                         c.Assert(err, check.NotNil)
  58                 }
  59
  60                 // Create, then update with properties
  61                 coll, err = s.localdb.CollectionCreate(s.userctx, arvados.CreateOptions{})
  62                 c.Assert(err, check.IsNil)
  63                 coll, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
  64                         UUID:   coll.UUID,
  65                         Select: []string{"uuid", "properties"},
  66                         Attrs: map[string]interface{}{
  67                                 "properties": tt.props,
  68                         }})
  69                 if tt.success {
  70                         c.Assert(err, check.IsNil)
  71                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
  72                 } else {
  73                         c.Assert(err, check.NotNil)
  74                 }
  75         }
  76 }
  77
  78 func (s *CollectionSuite) TestCollectionReplaceFiles(c *check.C) {
  79         adminctx := ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AdminToken)
  80         foo, err := s.localdb.railsProxy.CollectionCreate(adminctx, arvados.CreateOptions{
  81                 Attrs: map[string]interface{}{
  82                         "owner_uuid":    arvadostest.ActiveUserUUID,
  83                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
  84                 }})
  85         c.Assert(err, check.IsNil)
  86         s.localdb.signCollection(adminctx, &foo)
  87         foobarbaz, err := s.localdb.railsProxy.CollectionCreate(adminctx, arvados.CreateOptions{
  88                 Attrs: map[string]interface{}{
  89                         "owner_uuid":    arvadostest.ActiveUserUUID,
  90                         "manifest_text": "./foo/bar 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
  91                 }})
  92         c.Assert(err, check.IsNil)
  93         s.localdb.signCollection(adminctx, &foobarbaz)
  94         wazqux, err := s.localdb.railsProxy.CollectionCreate(adminctx, arvados.CreateOptions{
  95                 Attrs: map[string]interface{}{
  96                         "owner_uuid":    arvadostest.ActiveUserUUID,
  97                         "manifest_text": "./waz d85b1213473c2fd7c2045020a6b9c62b+3 0:3:qux.txt\n",
  98                 }})
  99         c.Assert(err, check.IsNil)
 100         s.localdb.signCollection(adminctx, &wazqux)
 101
 102         // Create using content from existing collections
 103         dst, err := s.localdb.CollectionCreate(s.userctx, arvados.CreateOptions{
 104                 ReplaceFiles: map[string]string{
 105                         "/f": foo.PortableDataHash + "/foo.txt",
 106                         "/b": foobarbaz.PortableDataHash + "/foo/bar",
 107                         "/q": wazqux.PortableDataHash + "/",
 108                         "/w": wazqux.PortableDataHash + "/waz",
 109                 },
 110                 Attrs: map[string]interface{}{
 111                         "owner_uuid": arvadostest.ActiveUserUUID,
 112                 }})
 113         c.Assert(err, check.IsNil)
 114         s.expectFiles(c, dst, "f", "b/baz.txt", "q/waz/qux.txt", "w/qux.txt")
 115
 116         // Delete a file and a directory
 117         dst, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 118                 UUID: dst.UUID,
 119                 ReplaceFiles: map[string]string{
 120                         "/f":     "",
 121                         "/q/waz": "",
 122                 }})
 123         c.Assert(err, check.IsNil)
 124         s.expectFiles(c, dst, "b/baz.txt", "q/", "w/qux.txt")
 125
 126         // Move and copy content within collection
 127         dst, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 128                 UUID: dst.UUID,
 129                 ReplaceFiles: map[string]string{
 130                         // Note splicing content to /b/corge.txt but
 131                         // removing everything else from /b
 132                         "/b":              "",
 133                         "/b/corge.txt":    dst.PortableDataHash + "/b/baz.txt",
 134                         "/quux/corge.txt": dst.PortableDataHash + "/b/baz.txt",
 135                 }})
 136         c.Assert(err, check.IsNil)
 137         s.expectFiles(c, dst, "b/corge.txt", "q/", "w/qux.txt", "quux/corge.txt")
 138
 139         // Remove everything except one file
 140         dst, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 141                 UUID: dst.UUID,
 142                 ReplaceFiles: map[string]string{
 143                         "/":            "",
 144                         "/b/corge.txt": dst.PortableDataHash + "/b/corge.txt",
 145                 }})
 146         c.Assert(err, check.IsNil)
 147         s.expectFiles(c, dst, "b/corge.txt")
 148
 149         // Copy entire collection to root
 150         dstcopy, err := s.localdb.CollectionCreate(s.userctx, arvados.CreateOptions{
 151                 ReplaceFiles: map[string]string{
 152                         "/": dst.PortableDataHash,
 153                 }})
 154         c.Check(err, check.IsNil)
 155         c.Check(dstcopy.PortableDataHash, check.Equals, dst.PortableDataHash)
 156         s.expectFiles(c, dstcopy, "b/corge.txt")
 157
 158         // Check invalid targets, sources, and combinations
 159         for _, badrepl := range []map[string]string{
 160                 {
 161                         "/foo/nope": dst.PortableDataHash + "/b",
 162                         "/foo":      dst.PortableDataHash + "/b",
 163                 },
 164                 {
 165                         "/foo":      dst.PortableDataHash + "/b",
 166                         "/foo/nope": "",
 167                 },
 168                 {
 169                         "/":     dst.PortableDataHash + "/",
 170                         "/nope": "",
 171                 },
 172                 {
 173                         "/":     dst.PortableDataHash + "/",
 174                         "/nope": dst.PortableDataHash + "/b",
 175                 },
 176                 {"/bad/": ""},
 177                 {"/./bad": ""},
 178                 {"/b/./ad": ""},
 179                 {"/b/../ad": ""},
 180                 {"/b/.": ""},
 181                 {".": ""},
 182                 {"bad": ""},
 183                 {"": ""},
 184                 {"/bad": "/b"},
 185                 {"/bad": "bad/b"},
 186                 {"/bad": dst.UUID + "/b"},
 187         } {
 188                 _, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 189                         UUID:         dst.UUID,
 190                         ReplaceFiles: badrepl,
 191                 })
 192                 c.Logf("badrepl %#v\n... got err: %s", badrepl, err)
 193                 c.Check(err, check.NotNil)
 194         }
 195
 196         // Check conflicting replace_files and manifest_text
 197         _, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 198                 UUID:         dst.UUID,
 199                 ReplaceFiles: map[string]string{"/": ""},
 200                 Attrs: map[string]interface{}{
 201                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:z\n",
 202                 }})
 203         c.Logf("replace_files+manifest_text\n... got err: %s", err)
 204         c.Check(err, check.ErrorMatches, "ambiguous request: both.*replace_files.*manifest_text.*")
 205 }
 206
 207 // expectFiles checks coll's directory structure against the given
 208 // list of expected files and empty directories. An expected path with
 209 // a trailing slash indicates an empty directory.
 210 func (s *CollectionSuite) expectFiles(c *check.C, coll arvados.Collection, expected ...string) {
 211         client := arvados.NewClientFromEnv()
 212         ac, err := arvadosclient.New(client)
 213         c.Assert(err, check.IsNil)
 214         kc, err := keepclient.MakeKeepClient(ac)
 215         c.Assert(err, check.IsNil)
 216         cfs, err := coll.FileSystem(client, kc)
 217         c.Assert(err, check.IsNil)
 218         var found []string
 219         nonemptydirs := map[string]bool{}
 220         fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
 221                 dir, _ := filepath.Split(path)
 222                 nonemptydirs[dir] = true
 223                 if d.IsDir() {
 224                         if path != "/" {
 225                                 path += "/"
 226                         }
 227                         if !nonemptydirs[path] {
 228                                 nonemptydirs[path] = false
 229                         }
 230                 } else {
 231                         found = append(found, path)
 232                 }
 233                 return nil
 234         })
 235         for d, nonempty := range nonemptydirs {
 236                 if !nonempty {
 237                         found = append(found, d)
 238                 }
 239         }
 240         for i, path := range found {
 241                 if path != "/" {
 242                         found[i] = strings.TrimPrefix(path, "/")
 243                 }
 244         }
 245         sort.Strings(found)
 246         sort.Strings(expected)
 247         c.Check(found, check.DeepEquals, expected)
 248 }
 249
 250 // Until #21701 it's hard to test from the outside whether the
 251 // uuid_lock mechanism is effectively serializing concurrent
 252 // replace_files updates to a single collection.  For now, we're
 253 // really just checking that it doesn't cause updates to deadlock or
 254 // anything like that.
 255 func (s *CollectionSuite) TestCollectionUpdateLock(c *check.C) {
 256         adminctx := ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AdminToken)
 257         foo, err := s.localdb.railsProxy.CollectionCreate(adminctx, arvados.CreateOptions{
 258                 Attrs: map[string]interface{}{
 259                         "owner_uuid":    arvadostest.ActiveUserUUID,
 260                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
 261                 }})
 262         c.Assert(err, check.IsNil)
 263         dst, err := s.localdb.CollectionCreate(s.userctx, arvados.CreateOptions{
 264                 ReplaceFiles: map[string]string{
 265                         "/foo.txt": foo.PortableDataHash + "/foo.txt",
 266                 },
 267                 Attrs: map[string]interface{}{
 268                         "owner_uuid": arvadostest.ActiveUserUUID,
 269                 }})
 270         c.Assert(err, check.IsNil)
 271         s.expectFiles(c, dst, "foo.txt")
 272
 273         var wg sync.WaitGroup
 274         for i := 0; i < 10; i++ {
 275                 name1, name2 := "a", "b"
 276                 if i&1 == 1 {
 277                         name1, name2 = "b", "a"
 278                 }
 279                 wg.Add(1)
 280                 go func() {
 281                         defer wg.Done()
 282                         upd, err := s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 283                                 UUID: dst.UUID,
 284                                 ReplaceFiles: map[string]string{
 285                                         "/" + name1: foo.PortableDataHash + "/foo.txt",
 286                                         "/" + name2: "",
 287                                         "/foo.txt":  "",
 288                                 }})
 289                         c.Assert(err, check.IsNil)
 290                         s.expectFiles(c, upd, name1)
 291                 }()
 292         }
 293         wg.Wait()
 294 }
 295
 296 func (s *CollectionSuite) TestSignatures(c *check.C) {
 297         resp, err := s.localdb.CollectionGet(s.userctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 298         c.Check(err, check.IsNil)
 299         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 300         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 301
 302         resp, err = s.localdb.CollectionGet(s.userctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
 303         c.Check(err, check.IsNil)
 304         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 305
 306         lresp, err := s.localdb.CollectionList(s.userctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}})
 307         c.Check(err, check.IsNil)
 308         if c.Check(lresp.Items, check.HasLen, 1) {
 309                 c.Check(lresp.Items[0].UUID, check.Equals, arvadostest.FooCollection)
 310                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 311                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 312         }
 313
 314         lresp, err = s.localdb.CollectionList(s.userctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"manifest_text"}})
 315         c.Check(err, check.IsNil)
 316         if c.Check(lresp.Items, check.HasLen, 1) {
 317                 c.Check(lresp.Items[0].ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 318                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 319         }
 320
 321         lresp, err = s.localdb.CollectionList(s.userctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"unsigned_manifest_text"}})
 322         c.Check(err, check.IsNil)
 323         if c.Check(lresp.Items, check.HasLen, 1) {
 324                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 325                 c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
 326         }
 327
 328         // early trash date causes lower signature TTL (even if
 329         // trash_at and is_trashed fields are unselected)
 330         trashed, err := s.localdb.CollectionCreate(s.userctx, arvados.CreateOptions{
 331                 Select: []string{"uuid", "manifest_text"},
 332                 Attrs: map[string]interface{}{
 333                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
 334                         "trash_at":      time.Now().UTC().Add(time.Hour),
 335                 }})
 336         c.Assert(err, check.IsNil)
 337         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour)
 338         resp, err = s.localdb.CollectionGet(s.userctx, arvados.GetOptions{UUID: trashed.UUID})
 339         c.Assert(err, check.IsNil)
 340         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour)
 341
 342         // distant future trash date does not cause higher signature TTL
 343         trashed, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 344                 UUID: trashed.UUID,
 345                 Attrs: map[string]interface{}{
 346                         "trash_at": time.Now().UTC().Add(time.Hour * 24 * 365),
 347                 }})
 348         c.Assert(err, check.IsNil)
 349         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour*24*7*2)
 350         resp, err = s.localdb.CollectionGet(s.userctx, arvados.GetOptions{UUID: trashed.UUID})
 351         c.Assert(err, check.IsNil)
 352         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 353
 354         // Make sure groups/contents doesn't return manifest_text with
 355         // collections (if it did, we'd need to sign it).
 356         gresp, err := s.localdb.GroupContents(s.userctx, arvados.GroupContentsOptions{
 357                 Limit:   -1,
 358                 Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}},
 359                 Select:  []string{"uuid", "manifest_text"},
 360         })
 361         if err != nil {
 362                 c.Check(err, check.ErrorMatches, `.*Invalid attribute.*manifest_text.*`)
 363         } else if c.Check(gresp.Items, check.HasLen, 1) {
 364                 c.Check(gresp.Items[0].(map[string]interface{})["uuid"], check.Equals, arvadostest.FooCollection)
 365                 c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
 366         }
 367 }
 368
 369 func (s *CollectionSuite) checkSignatureExpiry(c *check.C, manifestText string, expectedTTL time.Duration) {
 370         m := regexp.MustCompile(`@([[:xdigit:]]+)`).FindStringSubmatch(manifestText)
 371         c.Assert(m, check.HasLen, 2)
 372         sigexp, err := strconv.ParseInt(m[1], 16, 64)
 373         c.Assert(err, check.IsNil)
 374         expectedExp := time.Now().Add(expectedTTL).Unix()
 375         c.Check(sigexp > expectedExp-60, check.Equals, true)
 376         c.Check(sigexp <= expectedExp, check.Equals, true)
 377 }
 378
 379 func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
 380         s.localdb.cluster.Collections.BlobSigning = false
 381         resp, err := s.localdb.CollectionGet(s.userctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 382         c.Check(err, check.IsNil)
 383         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ +]*\+3 0:.*`)
 384 }