lib/controller/localdb/collection_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "context"
   9         "io/fs"
  10         "path/filepath"
  11         "regexp"
  12         "sort"
  13         "strconv"
  14         "strings"
  15         "time"
  16
  17         "git.arvados.org/arvados.git/sdk/go/arvados"
  18         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  19         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  20         "git.arvados.org/arvados.git/sdk/go/auth"
  21         "git.arvados.org/arvados.git/sdk/go/keepclient"
  22         check "gopkg.in/check.v1"
  23 )
  24
  25 var _ = check.Suite(&CollectionSuite{})
  26
  27 type CollectionSuite struct {
  28         localdbSuite
  29 }
  30
  31 func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C) {
  32         s.setUpVocabulary(c, "")
  33         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
  34
  35         tests := []struct {
  36                 name    string
  37                 props   map[string]interface{}
  38                 success bool
  39         }{
  40                 {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
  41                 {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
  42                 {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
  43                 {"Empty properties", map[string]interface{}{}, true},
  44         }
  45         for _, tt := range tests {
  46                 c.Log(c.TestName()+" ", tt.name)
  47
  48                 // Create with properties
  49                 coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
  50                         Select: []string{"uuid", "properties"},
  51                         Attrs: map[string]interface{}{
  52                                 "properties": tt.props,
  53                         }})
  54                 if tt.success {
  55                         c.Assert(err, check.IsNil)
  56                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
  57                 } else {
  58                         c.Assert(err, check.NotNil)
  59                 }
  60
  61                 // Create, then update with properties
  62                 coll, err = s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
  63                 c.Assert(err, check.IsNil)
  64                 coll, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
  65                         UUID:   coll.UUID,
  66                         Select: []string{"uuid", "properties"},
  67                         Attrs: map[string]interface{}{
  68                                 "properties": tt.props,
  69                         }})
  70                 if tt.success {
  71                         c.Assert(err, check.IsNil)
  72                         c.Assert(coll.Properties, check.DeepEquals, tt.props)
  73                 } else {
  74                         c.Assert(err, check.NotNil)
  75                 }
  76         }
  77 }
  78
  79 func (s *CollectionSuite) TestCollectionReplaceFiles(c *check.C) {
  80         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AdminToken}})
  81         foo, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
  82                 Attrs: map[string]interface{}{
  83                         "owner_uuid":    arvadostest.ActiveUserUUID,
  84                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
  85                 }})
  86         c.Assert(err, check.IsNil)
  87         s.localdb.signCollection(ctx, &foo)
  88         foobarbaz, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
  89                 Attrs: map[string]interface{}{
  90                         "owner_uuid":    arvadostest.ActiveUserUUID,
  91                         "manifest_text": "./foo/bar 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
  92                 }})
  93         c.Assert(err, check.IsNil)
  94         s.localdb.signCollection(ctx, &foobarbaz)
  95         wazqux, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
  96                 Attrs: map[string]interface{}{
  97                         "owner_uuid":    arvadostest.ActiveUserUUID,
  98                         "manifest_text": "./waz d85b1213473c2fd7c2045020a6b9c62b+3 0:3:qux.txt\n",
  99                 }})
 100         c.Assert(err, check.IsNil)
 101         s.localdb.signCollection(ctx, &wazqux)
 102
 103         ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 104
 105         // Create using content from existing collections
 106         dst, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 107                 ReplaceFiles: map[string]string{
 108                         "/f": foo.PortableDataHash + "/foo.txt",
 109                         "/b": foobarbaz.PortableDataHash + "/foo/bar",
 110                         "/q": wazqux.PortableDataHash + "/",
 111                         "/w": wazqux.PortableDataHash + "/waz",
 112                 },
 113                 Attrs: map[string]interface{}{
 114                         "owner_uuid": arvadostest.ActiveUserUUID,
 115                 }})
 116         c.Assert(err, check.IsNil)
 117         s.expectFiles(c, dst, "f", "b/baz.txt", "q/waz/qux.txt", "w/qux.txt")
 118
 119         // Delete a file and a directory
 120         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 121                 UUID: dst.UUID,
 122                 ReplaceFiles: map[string]string{
 123                         "/f":     "",
 124                         "/q/waz": "",
 125                 }})
 126         c.Assert(err, check.IsNil)
 127         s.expectFiles(c, dst, "b/baz.txt", "q/", "w/qux.txt")
 128
 129         // Move and copy content within collection
 130         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 131                 UUID: dst.UUID,
 132                 ReplaceFiles: map[string]string{
 133                         // Note splicing content to /b/corge.txt but
 134                         // removing everything else from /b
 135                         "/b":              "",
 136                         "/b/corge.txt":    dst.PortableDataHash + "/b/baz.txt",
 137                         "/quux/corge.txt": dst.PortableDataHash + "/b/baz.txt",
 138                 }})
 139         c.Assert(err, check.IsNil)
 140         s.expectFiles(c, dst, "b/corge.txt", "q/", "w/qux.txt", "quux/corge.txt")
 141
 142         // Remove everything except one file
 143         dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 144                 UUID: dst.UUID,
 145                 ReplaceFiles: map[string]string{
 146                         "/":            "",
 147                         "/b/corge.txt": dst.PortableDataHash + "/b/corge.txt",
 148                 }})
 149         c.Assert(err, check.IsNil)
 150         s.expectFiles(c, dst, "b/corge.txt")
 151
 152         // Copy entire collection to root
 153         dstcopy, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 154                 ReplaceFiles: map[string]string{
 155                         "/": dst.PortableDataHash,
 156                 }})
 157         c.Check(err, check.IsNil)
 158         c.Check(dstcopy.PortableDataHash, check.Equals, dst.PortableDataHash)
 159         s.expectFiles(c, dstcopy, "b/corge.txt")
 160
 161         // Check invalid targets, sources, and combinations
 162         for _, badrepl := range []map[string]string{
 163                 {
 164                         "/foo/nope": dst.PortableDataHash + "/b",
 165                         "/foo":      dst.PortableDataHash + "/b",
 166                 },
 167                 {
 168                         "/foo":      dst.PortableDataHash + "/b",
 169                         "/foo/nope": "",
 170                 },
 171                 {
 172                         "/":     dst.PortableDataHash + "/",
 173                         "/nope": "",
 174                 },
 175                 {
 176                         "/":     dst.PortableDataHash + "/",
 177                         "/nope": dst.PortableDataHash + "/b",
 178                 },
 179                 {"/bad/": ""},
 180                 {"/./bad": ""},
 181                 {"/b/./ad": ""},
 182                 {"/b/../ad": ""},
 183                 {"/b/.": ""},
 184                 {".": ""},
 185                 {"bad": ""},
 186                 {"": ""},
 187                 {"/bad": "/b"},
 188                 {"/bad": "bad/b"},
 189                 {"/bad": dst.UUID + "/b"},
 190         } {
 191                 _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 192                         UUID:         dst.UUID,
 193                         ReplaceFiles: badrepl,
 194                 })
 195                 c.Logf("badrepl %#v\n... got err: %s", badrepl, err)
 196                 c.Check(err, check.NotNil)
 197         }
 198
 199         // Check conflicting replace_files and manifest_text
 200         _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 201                 UUID:         dst.UUID,
 202                 ReplaceFiles: map[string]string{"/": ""},
 203                 Attrs: map[string]interface{}{
 204                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:z\n",
 205                 }})
 206         c.Logf("replace_files+manifest_text\n... got err: %s", err)
 207         c.Check(err, check.ErrorMatches, "ambiguous request: both.*replace_files.*manifest_text.*")
 208 }
 209
 210 // expectFiles checks coll's directory structure against the given
 211 // list of expected files and empty directories. An expected path with
 212 // a trailing slash indicates an empty directory.
 213 func (s *CollectionSuite) expectFiles(c *check.C, coll arvados.Collection, expected ...string) {
 214         client := arvados.NewClientFromEnv()
 215         ac, err := arvadosclient.New(client)
 216         c.Assert(err, check.IsNil)
 217         kc, err := keepclient.MakeKeepClient(ac)
 218         c.Assert(err, check.IsNil)
 219         cfs, err := coll.FileSystem(arvados.NewClientFromEnv(), kc)
 220         c.Assert(err, check.IsNil)
 221         var found []string
 222         nonemptydirs := map[string]bool{}
 223         fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
 224                 dir, _ := filepath.Split(path)
 225                 nonemptydirs[dir] = true
 226                 if d.IsDir() {
 227                         if path != "/" {
 228                                 path += "/"
 229                         }
 230                         if !nonemptydirs[path] {
 231                                 nonemptydirs[path] = false
 232                         }
 233                 } else {
 234                         found = append(found, path)
 235                 }
 236                 return nil
 237         })
 238         for d, nonempty := range nonemptydirs {
 239                 if !nonempty {
 240                         found = append(found, d)
 241                 }
 242         }
 243         for i, path := range found {
 244                 if path != "/" {
 245                         found[i] = strings.TrimPrefix(path, "/")
 246                 }
 247         }
 248         sort.Strings(found)
 249         sort.Strings(expected)
 250         c.Check(found, check.DeepEquals, expected)
 251 }
 252
 253 func (s *CollectionSuite) TestSignatures(c *check.C) {
 254         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 255
 256         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 257         c.Check(err, check.IsNil)
 258         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 259         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 260
 261         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
 262         c.Check(err, check.IsNil)
 263         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 264
 265         lresp, err := s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}})
 266         c.Check(err, check.IsNil)
 267         if c.Check(lresp.Items, check.HasLen, 1) {
 268                 c.Check(lresp.Items[0].UUID, check.Equals, arvadostest.FooCollection)
 269                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 270                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 271         }
 272
 273         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"manifest_text"}})
 274         c.Check(err, check.IsNil)
 275         if c.Check(lresp.Items, check.HasLen, 1) {
 276                 c.Check(lresp.Items[0].ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
 277                 c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
 278         }
 279
 280         lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"unsigned_manifest_text"}})
 281         c.Check(err, check.IsNil)
 282         if c.Check(lresp.Items, check.HasLen, 1) {
 283                 c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 284                 c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
 285         }
 286
 287         // early trash date causes lower signature TTL (even if
 288         // trash_at and is_trashed fields are unselected)
 289         trashed, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 290                 Select: []string{"uuid", "manifest_text"},
 291                 Attrs: map[string]interface{}{
 292                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
 293                         "trash_at":      time.Now().UTC().Add(time.Hour),
 294                 }})
 295         c.Assert(err, check.IsNil)
 296         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour)
 297         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
 298         c.Assert(err, check.IsNil)
 299         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour)
 300
 301         // distant future trash date does not cause higher signature TTL
 302         trashed, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 303                 UUID: trashed.UUID,
 304                 Attrs: map[string]interface{}{
 305                         "trash_at": time.Now().UTC().Add(time.Hour * 24 * 365),
 306                 }})
 307         c.Assert(err, check.IsNil)
 308         s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour*24*7*2)
 309         resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
 310         c.Assert(err, check.IsNil)
 311         s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 312
 313         // Make sure groups/contents doesn't return manifest_text with
 314         // collections (if it did, we'd need to sign it).
 315         gresp, err := s.localdb.GroupContents(ctx, arvados.GroupContentsOptions{
 316                 Limit:   -1,
 317                 Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}},
 318                 Select:  []string{"uuid", "manifest_text"},
 319         })
 320         if err != nil {
 321                 c.Check(err, check.ErrorMatches, `.*Invalid attribute.*manifest_text.*`)
 322         } else if c.Check(gresp.Items, check.HasLen, 1) {
 323                 c.Check(gresp.Items[0].(map[string]interface{})["uuid"], check.Equals, arvadostest.FooCollection)
 324                 c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
 325         }
 326 }
 327
 328 func (s *CollectionSuite) checkSignatureExpiry(c *check.C, manifestText string, expectedTTL time.Duration) {
 329         m := regexp.MustCompile(`@([[:xdigit:]]+)`).FindStringSubmatch(manifestText)
 330         c.Assert(m, check.HasLen, 2)
 331         sigexp, err := strconv.ParseInt(m[1], 16, 64)
 332         c.Assert(err, check.IsNil)
 333         expectedExp := time.Now().Add(expectedTTL).Unix()
 334         c.Check(sigexp > expectedExp-60, check.Equals, true)
 335         c.Check(sigexp <= expectedExp, check.Equals, true)
 336 }
 337
 338 func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
 339         s.localdb.cluster.Collections.BlobSigning = false
 340         ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 341
 342         resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 343         c.Check(err, check.IsNil)
 344         c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ +]*\+3 0:.*`)
 345 }