sdk/go/arvados/fs_project_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "errors"
  11         "io"
  12         "os"
  13         "strings"
  14
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 type spiedRequest struct {
  19         method string
  20         path   string
  21         params map[string]interface{}
  22 }
  23
  24 type spyingClient struct {
  25         *Client
  26         calls []spiedRequest
  27 }
  28
  29 func (sc *spyingClient) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
  30         var paramsCopy map[string]interface{}
  31         var buf bytes.Buffer
  32         json.NewEncoder(&buf).Encode(params)
  33         json.NewDecoder(&buf).Decode(&paramsCopy)
  34         sc.calls = append(sc.calls, spiedRequest{
  35                 method: method,
  36                 path:   path,
  37                 params: paramsCopy,
  38         })
  39         return sc.Client.RequestAndDecode(dst, method, path, body, params)
  40 }
  41
  42 func (s *SiteFSSuite) TestFilterGroup(c *check.C) {
  43         // Make sure that a collection and group that match the filter are present,
  44         // and that a group that does not match the filter is not present.
  45
  46         checkOpen := func(path string, exists bool) {
  47                 f, err := s.fs.Open(path)
  48                 if exists {
  49                         if c.Check(err, check.IsNil) {
  50                                 c.Check(f.Close(), check.IsNil)
  51                         }
  52                 } else {
  53                         c.Check(err, check.Equals, os.ErrNotExist)
  54                 }
  55         }
  56
  57         checkDirContains := func(parent, child string, exists bool) {
  58                 f, err := s.fs.Open(parent)
  59                 if !c.Check(err, check.IsNil) {
  60                         return
  61                 }
  62                 ents, err := f.Readdir(-1)
  63                 if !c.Check(err, check.IsNil) {
  64                         return
  65                 }
  66                 for _, ent := range ents {
  67                         if !exists {
  68                                 c.Check(ent.Name(), check.Not(check.Equals), child)
  69                                 if child == "" {
  70                                         // no children are expected
  71                                         c.Errorf("child %q found in parent %q", child, parent)
  72                                 }
  73                         } else if ent.Name() == child {
  74                                 return
  75                         }
  76                 }
  77                 if exists {
  78                         c.Errorf("child %q not found in parent %q", child, parent)
  79                 }
  80         }
  81
  82         checkOpen("/users/active/This filter group/baz_file", true)
  83         checkOpen("/users/active/This filter group/A Subproject", true)
  84         checkOpen("/users/active/This filter group/A Project", false)
  85         s.fs.MountProject("fg", fixtureThisFilterGroupUUID)
  86         checkOpen("/fg/baz_file", true)
  87         checkOpen("/fg/A Subproject", true)
  88         checkOpen("/fg/A Project", false)
  89         s.fs.MountProject("home", "")
  90         checkOpen("/home/A filter group with an is_a collection filter/baz_file", true)
  91         checkOpen("/home/A filter group with an is_a collection filter/baz_file/baz", true)
  92         checkOpen("/home/A filter group with an is_a collection filter/A Subproject", false)
  93         checkOpen("/home/A filter group with an is_a collection filter/A Project", false)
  94
  95         // An empty filter means everything that is visible should be returned.
  96         checkOpen("/users/active/A filter group without filters/baz_file", true)
  97         checkOpen("/users/active/A filter group without filters/A Subproject", true)
  98         checkOpen("/users/active/A filter group without filters/A Project", true)
  99         s.fs.MountProject("fg2", fixtureAFilterGroupTwoUUID)
 100         checkOpen("/fg2/baz_file", true)
 101         checkOpen("/fg2/A Subproject", true)
 102         checkOpen("/fg2/A Project", true)
 103
 104         // If a filter group matches itself or one of its ancestors,
 105         // the matched item appears as an empty directory.
 106         checkDirContains("/users/active/A filter group without filters", "A filter group without filters", true)
 107         checkOpen("/users/active/A filter group without filters/A filter group without filters", true)
 108         checkOpen("/users/active/A filter group without filters/A filter group without filters/baz_file", false)
 109         checkDirContains("/users/active/A filter group without filters/A filter group without filters", "", false)
 110
 111         // An 'is_a' 'arvados#collection' filter means only collections should be returned.
 112         checkOpen("/users/active/A filter group with an is_a collection filter/baz_file", true)
 113         checkOpen("/users/active/A filter group with an is_a collection filter/baz_file/baz", true)
 114         checkOpen("/users/active/A filter group with an is_a collection filter/A Subproject", false)
 115         checkOpen("/users/active/A filter group with an is_a collection filter/A Project", false)
 116         s.fs.MountProject("fg3", fixtureAFilterGroupThreeUUID)
 117         checkOpen("/fg3/baz_file", true)
 118         checkOpen("/fg3/baz_file/baz", true)
 119         checkOpen("/fg3/A Subproject", false)
 120
 121         // An 'exists' 'arvados#collection' filter means only collections with certain properties should be returned.
 122         s.fs.MountProject("fg4", fixtureAFilterGroupFourUUID)
 123         checkOpen("/fg4/collection with list property with odd values", true)
 124         checkOpen("/fg4/collection with list property with even values", true)
 125         checkOpen("/fg4/baz_file", false)
 126
 127         // A 'contains' 'arvados#collection' filter means only collections with certain properties should be returned.
 128         s.fs.MountProject("fg5", fixtureAFilterGroupFiveUUID)
 129         checkOpen("/fg5/collection with list property with odd values", true)
 130         checkOpen("/fg5/collection with list property with string value", true)
 131         checkOpen("/fg5/collection with prop2 5", false)
 132         checkOpen("/fg5/collection with list property with even values", false)
 133 }
 134
 135 func (s *SiteFSSuite) TestCurrentUserHome(c *check.C) {
 136         s.fs.MountProject("home", "")
 137         s.testHomeProject(c, "/home", "home")
 138 }
 139
 140 func (s *SiteFSSuite) TestUsersDir(c *check.C) {
 141         // /users/active is a hardlink to a dir whose name is the UUID
 142         // of the active user
 143         s.testHomeProject(c, "/users/active", fixtureActiveUserUUID)
 144 }
 145
 146 func (s *SiteFSSuite) testHomeProject(c *check.C, path, expectRealName string) {
 147         f, err := s.fs.Open(path)
 148         c.Assert(err, check.IsNil)
 149         fis, err := f.Readdir(-1)
 150         c.Assert(err, check.IsNil)
 151         c.Check(len(fis), check.Not(check.Equals), 0)
 152
 153         ok := false
 154         for _, fi := range fis {
 155                 c.Check(fi.Name(), check.Not(check.Equals), "")
 156                 if fi.Name() == "A Project" {
 157                         ok = true
 158                 }
 159         }
 160         c.Check(ok, check.Equals, true)
 161
 162         f, err = s.fs.Open(path + "/A Project/..")
 163         c.Assert(err, check.IsNil)
 164         fi, err := f.Stat()
 165         c.Assert(err, check.IsNil)
 166         c.Check(fi.IsDir(), check.Equals, true)
 167         c.Check(fi.Name(), check.Equals, expectRealName)
 168
 169         f, err = s.fs.Open(path + "/A Project/A Subproject")
 170         c.Assert(err, check.IsNil)
 171         fi, err = f.Stat()
 172         c.Assert(err, check.IsNil)
 173         c.Check(fi.IsDir(), check.Equals, true)
 174
 175         for _, nx := range []string{
 176                 path + "/Unrestricted public data",
 177                 path + "/Unrestricted public data/does not exist",
 178                 path + "/A Project/does not exist",
 179         } {
 180                 c.Log(nx)
 181                 f, err = s.fs.Open(nx)
 182                 c.Check(err, check.NotNil)
 183                 c.Check(os.IsNotExist(err), check.Equals, true)
 184         }
 185 }
 186
 187 func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
 188         f, err := s.fs.Open("/users/active/A Project/A Subproject")
 189         c.Assert(err, check.IsNil)
 190         defer f.Close()
 191         f, err = s.fs.Open("/users/active/A Project/Project does not exist")
 192         c.Assert(err, check.NotNil)
 193         f, err = s.fs.Open("/users/active/A Project/A Subproject")
 194         c.Assert(err, check.IsNil)
 195         defer f.Close()
 196         f, err = s.fs.Open("/users/active/A Project")
 197         c.Assert(err, check.IsNil)
 198         defer f.Close()
 199         fis, err := f.Readdir(-1)
 200         c.Assert(err, check.IsNil)
 201         c.Logf("%#v", fis)
 202         var foundSubproject, foundCollection bool
 203         for _, fi := range fis {
 204                 switch fi.Name() {
 205                 case "A Subproject":
 206                         foundSubproject = true
 207                 case "collection_to_move_around":
 208                         foundCollection = true
 209                 }
 210         }
 211         c.Check(foundSubproject, check.Equals, true)
 212         c.Check(foundCollection, check.Equals, true)
 213 }
 214
 215 func (s *SiteFSSuite) TestSlashInName(c *check.C) {
 216         var badCollection Collection
 217         err := s.client.RequestAndDecode(&badCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 218                 "collection": map[string]string{
 219                         "name":       "bad/collection",
 220                         "owner_uuid": fixtureAProjectUUID,
 221                 },
 222         })
 223         c.Assert(err, check.IsNil)
 224         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+badCollection.UUID, nil, nil)
 225
 226         var badProject Group
 227         err = s.client.RequestAndDecode(&badProject, "POST", "arvados/v1/groups", nil, map[string]interface{}{
 228                 "group": map[string]string{
 229                         "name":        "bad/project",
 230                         "group_class": "project",
 231                         "owner_uuid":  fixtureAProjectUUID,
 232                 },
 233         })
 234         c.Assert(err, check.IsNil)
 235         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
 236
 237         dir, err := s.fs.Open("/users/active/A Project")
 238         c.Assert(err, check.IsNil)
 239         fis, err := dir.Readdir(-1)
 240         c.Check(err, check.IsNil)
 241         for _, fi := range fis {
 242                 c.Logf("fi.Name() == %q", fi.Name())
 243                 c.Check(strings.Contains(fi.Name(), "/"), check.Equals, false)
 244         }
 245
 246         // Make a new fs (otherwise content will still be cached from
 247         // above) and enable "/" replacement string.
 248         s.fs = s.client.SiteFileSystem(s.kc)
 249         s.fs.ForwardSlashNameSubstitution("___")
 250         dir, err = s.fs.Open("/users/active/A Project/bad___collection")
 251         if c.Check(err, check.IsNil) {
 252                 _, err = dir.Readdir(-1)
 253                 c.Check(err, check.IsNil)
 254         }
 255         dir, err = s.fs.Open("/users/active/A Project/bad___project")
 256         if c.Check(err, check.IsNil) {
 257                 _, err = dir.Readdir(-1)
 258                 c.Check(err, check.IsNil)
 259         }
 260 }
 261
 262 func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
 263         s.fs.MountProject("home", "")
 264
 265         project, err := s.fs.OpenFile("/home/A Project", 0, 0)
 266         c.Assert(err, check.IsNil)
 267
 268         _, err = s.fs.Open("/home/A Project/oob")
 269         c.Check(err, check.NotNil)
 270
 271         var oob Collection
 272         err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 273                 "collection": map[string]string{
 274                         "name":       "oob",
 275                         "owner_uuid": fixtureAProjectUUID,
 276                 },
 277         })
 278         c.Assert(err, check.IsNil)
 279         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
 280
 281         err = project.Sync()
 282         c.Check(err, check.IsNil)
 283         f, err := s.fs.Open("/home/A Project/oob")
 284         c.Assert(err, check.IsNil)
 285         fi, err := f.Stat()
 286         c.Assert(err, check.IsNil)
 287         c.Check(fi.IsDir(), check.Equals, true)
 288         f.Close()
 289
 290         wf, err := s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
 291         c.Assert(err, check.IsNil)
 292         _, err = wf.Write([]byte("hello oob\n"))
 293         c.Check(err, check.IsNil)
 294         err = wf.Close()
 295         c.Check(err, check.IsNil)
 296
 297         err = project.Sync()
 298         c.Check(err, check.IsNil)
 299         f, err = s.fs.Open("/home/A Project/oob/test.txt")
 300         if c.Check(err, check.IsNil) {
 301                 f.Close()
 302         }
 303
 304         // Ensure collection was flushed by Sync
 305         var latest Collection
 306         err = s.client.RequestAndDecode(&latest, "GET", "arvados/v1/collections/"+oob.UUID, nil, nil)
 307         c.Check(err, check.IsNil)
 308         c.Check(latest.ManifestText, check.Matches, `.*:test.txt.*\n`)
 309
 310         // Delete test.txt behind s.fs's back by updating the
 311         // collection record with an empty ManifestText.
 312         err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, nil, map[string]interface{}{
 313                 "collection": map[string]string{
 314                         "manifest_text":      "",
 315                         "portable_data_hash": "d41d8cd98f00b204e9800998ecf8427e+0",
 316                 },
 317         })
 318         c.Assert(err, check.IsNil)
 319
 320         // Sync again to reload collection.
 321         err = project.Sync()
 322         c.Check(err, check.IsNil)
 323
 324         // Check test.txt deletion is reflected in fs.
 325         _, err = s.fs.Open("/home/A Project/oob/test.txt")
 326         c.Check(err, check.NotNil)
 327         f, err = s.fs.Open("/home/A Project/oob")
 328         if c.Check(err, check.IsNil) {
 329                 f.Close()
 330         }
 331
 332         err = s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
 333         c.Assert(err, check.IsNil)
 334
 335         wf, err = s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
 336         c.Assert(err, check.IsNil)
 337         err = wf.Close()
 338         c.Check(err, check.IsNil)
 339
 340         err = project.Sync()
 341         c.Check(err, check.NotNil) // can't update the deleted collection
 342         _, err = s.fs.Open("/home/A Project/oob")
 343         c.Check(err, check.IsNil) // parent dir still has old collection -- didn't reload, because Sync failed
 344 }
 345
 346 func (s *SiteFSSuite) TestProjectUnsupportedOperations(c *check.C) {
 347         s.fs.MountByID("by_id")
 348         s.fs.MountProject("home", "")
 349
 350         _, err := s.fs.OpenFile("/home/A Project/newfilename", os.O_CREATE|os.O_RDWR, 0)
 351         c.Check(err, ErrorIs, ErrInvalidOperation)
 352
 353         err = s.fs.Mkdir("/home/A Project/newdirname", 0)
 354         c.Check(err, ErrorIs, ErrInvalidOperation)
 355
 356         err = s.fs.Mkdir("/by_id/newdirname", 0)
 357         c.Check(err, ErrorIs, ErrInvalidOperation)
 358
 359         err = s.fs.Mkdir("/by_id/"+fixtureAProjectUUID+"/newdirname", 0)
 360         c.Check(err, ErrorIs, ErrInvalidOperation)
 361
 362         _, err = s.fs.OpenFile("/home/A Project", 0, 0)
 363         c.Check(err, check.IsNil)
 364 }
 365
 366 type errorIsChecker struct {
 367         *check.CheckerInfo
 368 }
 369
 370 var ErrorIs check.Checker = errorIsChecker{
 371         &check.CheckerInfo{Name: "ErrorIs", Params: []string{"value", "target"}},
 372 }
 373
 374 func (checker errorIsChecker) Check(params []interface{}, names []string) (result bool, errStr string) {
 375         err, ok := params[0].(error)
 376         if !ok {
 377                 return false, ""
 378         }
 379         target, ok := params[1].(error)
 380         if !ok {
 381                 return false, ""
 382         }
 383         return errors.Is(err, target), ""
 384 }