lib/controller/localdb/collection_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "io/fs"
   9         "path/filepath"
  10         "regexp"
  11         "sort"
  12         "strconv"
  13         "strings"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/lib/ctrlctx"
  17         "git.arvados.org/arvados.git/sdk/go/arvados"
  18         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  19         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  20         "git.arvados.org/arvados.git/sdk/go/keepclient"
  21         check "gopkg.in/check.v1"
  22 )
  23
  24 var _ = check.Suite(&CollectionSuite{})
  25
  26 type CollectionSuite struct {
  27         localdbSuite
  28 }
  29
  30 func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C) {
  31         s.setUpVocabulary(c, "")
  32
  33         tests := []struct {
  34                 name    string
  35                 props   map[string]interface{}
  36                 success bool
  37         }{
  38                 {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
  39                 {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
  40                 {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
  41                 {"Empty properties", map[string]interface{}{}, true},
  42         }
  43         for _, tt := range tests {
  44                 c.Log(c.TestName()+" ", tt.name)
  45
  46                 // Create with properties
  47                 coll, err := s.localdb.CollectionCreate(s.userctx, arvados.CreateOptions{
  48                         Select: []string{"uuid", "properties"},
  49                         Attrs: map[string]interface{}{
  50                                 "properties": tt.props,
  51                         }})
  52                 if tt.success {
  53                         c.Assert(err, check.IsNil)
  54                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
  55                 } else {
  56                         c.Assert(err, check.NotNil)
  57                 }
  58
  59                 // Create, then update with properties
  60                 coll, err = s.localdb.CollectionCreate(s.userctx, arvados.CreateOptions{})
  61                 c.Assert(err, check.IsNil)
  62                 coll, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
  63                         UUID:   coll.UUID,
  64                         Select: []string{"uuid", "properties"},
  65                         Attrs: map[string]interface{}{
  66                                 "properties": tt.props,
  67                         }})
  68                 if tt.success {
  69                         c.Assert(err, check.IsNil)
  70                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
  71                 } else {
  72                         c.Assert(err, check.NotNil)
  73                 }
  74         }
  75 }
  76
  77 func (s *CollectionSuite) TestCollectionReplaceFiles(c *check.C) {
  78         adminctx := ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AdminToken)
  79         foo, err := s.localdb.railsProxy.CollectionCreate(adminctx, arvados.CreateOptions{
  80                 Attrs: map[string]interface{}{
  81                         "owner_uuid":    arvadostest.ActiveUserUUID,
  82                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
  83                 }})
  84         c.Assert(err, check.IsNil)
  85         s.localdb.signCollection(adminctx, &foo)
  86         foobarbaz, err := s.localdb.railsProxy.CollectionCreate(adminctx, arvados.CreateOptions{
  87                 Attrs: map[string]interface{}{
  88                         "owner_uuid":    arvadostest.ActiveUserUUID,
  89                         "manifest_text": "./foo/bar 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
  90                 }})
  91         c.Assert(err, check.IsNil)
  92         s.localdb.signCollection(adminctx, &foobarbaz)
  93         wazqux, err := s.localdb.railsProxy.CollectionCreate(adminctx, arvados.CreateOptions{
  94                 Attrs: map[string]interface{}{
  95                         "owner_uuid":    arvadostest.ActiveUserUUID,
  96                         "manifest_text": "./waz d85b1213473c2fd7c2045020a6b9c62b+3 0:3:qux.txt\n",
  97                 }})
  98         c.Assert(err, check.IsNil)
  99         s.localdb.signCollection(adminctx, &wazqux)
 100
 101         // Create using content from existing collections
 102         dst, err := s.localdb.CollectionCreate(s.userctx, arvados.CreateOptions{
 103                 ReplaceFiles: map[string]string{
 104                         "/f": foo.PortableDataHash + "/foo.txt",
 105                         "/b": foobarbaz.PortableDataHash + "/foo/bar",
 106                         "/q": wazqux.PortableDataHash + "/",
 107                         "/w": wazqux.PortableDataHash + "/waz",
 108                 },
 109                 Attrs: map[string]interface{}{
 110                         "owner_uuid": arvadostest.ActiveUserUUID,
 111                 }})
 112         c.Assert(err, check.IsNil)
 113         s.expectFiles(c, dst, "f", "b/baz.txt", "q/waz/qux.txt", "w/qux.txt")
 114
 115         // Delete a file and a directory
 116         dst, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 117                 UUID: dst.UUID,
 118                 ReplaceFiles: map[string]string{
 119                         "/f":     "",
 120                         "/q/waz": "",
 121                 }})
 122         c.Assert(err, check.IsNil)
 123         s.expectFiles(c, dst, "b/baz.txt", "q/", "w/qux.txt")
 124
 125         // Move and copy content within collection
 126         dst, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 127                 UUID: dst.UUID,
 128                 ReplaceFiles: map[string]string{
 129                         // Note splicing content to /b/corge.txt but
 130                         // removing everything else from /b
 131                         "/b":              "",
 132                         "/b/corge.txt":    dst.PortableDataHash + "/b/baz.txt",
 133                         "/quux/corge.txt": dst.PortableDataHash + "/b/baz.txt",
 134                 }})
 135         c.Assert(err, check.IsNil)
 136         s.expectFiles(c, dst, "b/corge.txt", "q/", "w/qux.txt", "quux/corge.txt")
 137
 138         // Remove everything except one file
 139         dst, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 140                 UUID: dst.UUID,
 141                 ReplaceFiles: map[string]string{
 142                         "/":            "",
 143                         "/b/corge.txt": dst.PortableDataHash + "/b/corge.txt",
 144                 }})
 145         c.Assert(err, check.IsNil)
 146         s.expectFiles(c, dst, "b/corge.txt")
 147
 148         // Copy entire collection to root
 149         dstcopy, err := s.localdb.CollectionCreate(s.userctx, arvados.CreateOptions{
 150                 ReplaceFiles: map[string]string{
 151                         "/": dst.PortableDataHash,
 152                 }})
 153         c.Check(err, check.IsNil)
 154         c.Check(dstcopy.PortableDataHash, check.Equals, dst.PortableDataHash)
 155         s.expectFiles(c, dstcopy, "b/corge.txt")
 156
 157         // Check invalid targets, sources, and combinations
 158         for _, badrepl := range []map[string]string{
 159                 {
 160                         "/foo/nope": dst.PortableDataHash + "/b",
 161                         "/foo":      dst.PortableDataHash + "/b",
 162                 },
 163                 {
 164                         "/foo":      dst.PortableDataHash + "/b",
 165                         "/foo/nope": "",
 166                 },
 167                 {
 168                         "/":     dst.PortableDataHash + "/",
 169                         "/nope": "",
 170                 },
 171                 {
 172                         "/":     dst.PortableDataHash + "/",
 173                         "/nope": dst.PortableDataHash + "/b",
 174                 },
 175                 {"/bad/": ""},
 176                 {"/./bad": ""},
 177                 {"/b/./ad": ""},
 178                 {"/b/../ad": ""},
 179                 {"/b/.": ""},
 180                 {".": ""},
 181                 {"bad": ""},
 182                 {"": ""},
 183                 {"/bad": "/b"},
 184                 {"/bad": "bad/b"},
 185                 {"/bad": dst.UUID + "/b"},
 186         } {
 187                 _, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 188                         UUID:         dst.UUID,
 189                         ReplaceFiles: badrepl,
 190                 })
 191                 c.Logf("badrepl %#v\n... got err: %s", badrepl, err)
 192                 c.Check(err, check.NotNil)
 193         }
 194
 195         // Check conflicting replace_files and manifest_text
 196         _, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 197                 UUID:         dst.UUID,
 198                 ReplaceFiles: map[string]string{"/": ""},
 199                 Attrs: map[string]interface{}{
 200                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:z\n",
 201                 }})
 202         c.Logf("replace_files+manifest_text\n... got err: %s", err)
 203         c.Check(err, check.ErrorMatches, "ambiguous request: both.*replace_files.*manifest_text.*")
 204 }
 205
 206 // expectFiles checks coll's directory structure against the given
 207 // list of expected files and empty directories. An expected path with
 208 // a trailing slash indicates an empty directory.
 209 func (s *CollectionSuite) expectFiles(c *check.C, coll arvados.Collection, expected ...string) {
 210         client := arvados.NewClientFromEnv()
 211         ac, err := arvadosclient.New(client)
 212         c.Assert(err, check.IsNil)
 213         kc, err := keepclient.MakeKeepClient(ac)
 214         c.Assert(err, check.IsNil)
 215         cfs, err := coll.FileSystem(arvados.NewClientFromEnv(), kc)
 216         c.Assert(err, check.IsNil)
 217         var found []string
 218         nonemptydirs := map[string]bool{}
 219         fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
 220                 dir, _ := filepath.Split(path)
 221                 nonemptydirs[dir] = true
 222                 if d.IsDir() {
 223                         if path != "/" {
 224                                 path += "/"
 225                         }
 226                         if !nonemptydirs[path] {
 227                                 nonemptydirs[path] = false
 228                         }
 229                 } else {
 230                         found = append(found, path)
 231                 }
 232                 return nil
 233         })
 234         for d, nonempty := range nonemptydirs {
 235                 if !nonempty {
 236                         found = append(found, d)
 237                 }
 238         }
 239         for i, path := range found {
 240                 if path != "/" {
 241                         found[i] = strings.TrimPrefix(path, "/")
 242                 }
 243         }
 244         sort.Strings(found)
 245         sort.Strings(expected)
 246         c.Check(found, check.DeepEquals, expected)
 247 }
 248
 249 func (s *CollectionSuite) TestSignatures(c *check.C) {
 250         resp, err := s.localdb.CollectionGet(s.userctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 251         c.Check(err, check.IsNil)
 252         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 253         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 254
 255         resp, err = s.localdb.CollectionGet(s.userctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
 256         c.Check(err, check.IsNil)
 257         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 258
 259         lresp, err := s.localdb.CollectionList(s.userctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}})
 260         c.Check(err, check.IsNil)
 261         if c.Check(lresp.Items, check.HasLen, 1) {
 262                 c.Check(lresp.Items[0].UUID, check.Equals, arvadostest.FooCollection)
 263                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 264                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 265         }
 266
 267         lresp, err = s.localdb.CollectionList(s.userctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"manifest_text"}})
 268         c.Check(err, check.IsNil)
 269         if c.Check(lresp.Items, check.HasLen, 1) {
 270                 c.Check(lresp.Items[0].ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 271                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 272         }
 273
 274         lresp, err = s.localdb.CollectionList(s.userctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"unsigned_manifest_text"}})
 275         c.Check(err, check.IsNil)
 276         if c.Check(lresp.Items, check.HasLen, 1) {
 277                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 278                 c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
 279         }
 280
 281         // early trash date causes lower signature TTL (even if
 282         // trash_at and is_trashed fields are unselected)
 283         trashed, err := s.localdb.CollectionCreate(s.userctx, arvados.CreateOptions{
 284                 Select: []string{"uuid", "manifest_text"},
 285                 Attrs: map[string]interface{}{
 286                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
 287                         "trash_at":      time.Now().UTC().Add(time.Hour),
 288                 }})
 289         c.Assert(err, check.IsNil)
 290         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour)
 291         resp, err = s.localdb.CollectionGet(s.userctx, arvados.GetOptions{UUID: trashed.UUID})
 292         c.Assert(err, check.IsNil)
 293         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour)
 294
 295         // distant future trash date does not cause higher signature TTL
 296         trashed, err = s.localdb.CollectionUpdate(s.userctx, arvados.UpdateOptions{
 297                 UUID: trashed.UUID,
 298                 Attrs: map[string]interface{}{
 299                         "trash_at": time.Now().UTC().Add(time.Hour * 24 * 365),
 300                 }})
 301         c.Assert(err, check.IsNil)
 302         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour*24*7*2)
 303         resp, err = s.localdb.CollectionGet(s.userctx, arvados.GetOptions{UUID: trashed.UUID})
 304         c.Assert(err, check.IsNil)
 305         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 306
 307         // Make sure groups/contents doesn't return manifest_text with
 308         // collections (if it did, we'd need to sign it).
 309         gresp, err := s.localdb.GroupContents(s.userctx, arvados.GroupContentsOptions{
 310                 Limit:   -1,
 311                 Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}},
 312                 Select:  []string{"uuid", "manifest_text"},
 313         })
 314         if err != nil {
 315                 c.Check(err, check.ErrorMatches, `.*Invalid attribute.*manifest_text.*`)
 316         } else if c.Check(gresp.Items, check.HasLen, 1) {
 317                 c.Check(gresp.Items[0].(map[string]interface{})["uuid"], check.Equals, arvadostest.FooCollection)
 318                 c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
 319         }
 320 }
 321
 322 func (s *CollectionSuite) checkSignatureExpiry(c *check.C, manifestText string, expectedTTL time.Duration) {
 323         m := regexp.MustCompile(`@([[:xdigit:]]+)`).FindStringSubmatch(manifestText)
 324         c.Assert(m, check.HasLen, 2)
 325         sigexp, err := strconv.ParseInt(m[1], 16, 64)
 326         c.Assert(err, check.IsNil)
 327         expectedExp := time.Now().Add(expectedTTL).Unix()
 328         c.Check(sigexp > expectedExp-60, check.Equals, true)
 329         c.Check(sigexp <= expectedExp, check.Equals, true)
 330 }
 331
 332 func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
 333         s.localdb.cluster.Collections.BlobSigning = false
 334         resp, err := s.localdb.CollectionGet(s.userctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 335         c.Check(err, check.IsNil)
 336         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ +]*\+3 0:.*`)
 337 }