sdk/go/arvados/fs_project_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "errors"
  11         "io"
  12         "os"
  13         "path/filepath"
  14         "strings"
  15
  16         check "gopkg.in/check.v1"
  17 )
  18
  19 type spiedRequest struct {
  20         method string
  21         path   string
  22         params map[string]interface{}
  23 }
  24
  25 type spyingClient struct {
  26         *Client
  27         calls []spiedRequest
  28 }
  29
  30 func (sc *spyingClient) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
  31         var paramsCopy map[string]interface{}
  32         var buf bytes.Buffer
  33         json.NewEncoder(&buf).Encode(params)
  34         json.NewDecoder(&buf).Decode(&paramsCopy)
  35         sc.calls = append(sc.calls, spiedRequest{
  36                 method: method,
  37                 path:   path,
  38                 params: paramsCopy,
  39         })
  40         return sc.Client.RequestAndDecode(dst, method, path, body, params)
  41 }
  42
  43 func (s *SiteFSSuite) TestFilterGroup(c *check.C) {
  44         // Make sure that a collection and group that match the filter are present,
  45         // and that a group that does not match the filter is not present.
  46         s.fs.MountProject("fg", fixtureThisFilterGroupUUID)
  47
  48         _, err := s.fs.OpenFile("/fg/baz_file", 0, 0)
  49         c.Assert(err, check.IsNil)
  50
  51         _, err = s.fs.OpenFile("/fg/A Subproject", 0, 0)
  52         c.Assert(err, check.IsNil)
  53
  54         _, err = s.fs.OpenFile("/fg/A Project", 0, 0)
  55         c.Assert(err, check.Not(check.IsNil))
  56
  57         // An empty filter means everything that is visible should be returned.
  58         s.fs.MountProject("fg2", fixtureAFilterGroupTwoUUID)
  59
  60         _, err = s.fs.OpenFile("/fg2/baz_file", 0, 0)
  61         c.Assert(err, check.IsNil)
  62
  63         _, err = s.fs.OpenFile("/fg2/A Subproject", 0, 0)
  64         c.Assert(err, check.IsNil)
  65
  66         _, err = s.fs.OpenFile("/fg2/A Project", 0, 0)
  67         c.Assert(err, check.IsNil)
  68
  69         // An 'is_a' 'arvados#collection' filter means only collections should be returned.
  70         s.fs.MountProject("fg3", fixtureAFilterGroupThreeUUID)
  71
  72         _, err = s.fs.OpenFile("/fg3/baz_file", 0, 0)
  73         c.Assert(err, check.IsNil)
  74
  75         _, err = s.fs.OpenFile("/fg3/A Subproject", 0, 0)
  76         c.Assert(err, check.Not(check.IsNil))
  77
  78         // An 'exists' 'arvados#collection' filter means only collections with certain properties should be returned.
  79         s.fs.MountProject("fg4", fixtureAFilterGroupFourUUID)
  80
  81         _, err = s.fs.Stat("/fg4/collection with list property with odd values")
  82         c.Assert(err, check.IsNil)
  83
  84         _, err = s.fs.Stat("/fg4/collection with list property with even values")
  85         c.Assert(err, check.IsNil)
  86
  87         // A 'contains' 'arvados#collection' filter means only collections with certain properties should be returned.
  88         s.fs.MountProject("fg5", fixtureAFilterGroupFiveUUID)
  89
  90         _, err = s.fs.Stat("/fg5/collection with list property with odd values")
  91         c.Assert(err, check.IsNil)
  92
  93         _, err = s.fs.Stat("/fg5/collection with list property with string value")
  94         c.Assert(err, check.IsNil)
  95
  96         _, err = s.fs.Stat("/fg5/collection with prop2 5")
  97         c.Assert(err, check.Not(check.IsNil))
  98
  99         _, err = s.fs.Stat("/fg5/collection with list property with even values")
 100         c.Assert(err, check.Not(check.IsNil))
 101 }
 102
 103 func (s *SiteFSSuite) TestCurrentUserHome(c *check.C) {
 104         s.fs.MountProject("home", "")
 105         s.testHomeProject(c, "/home")
 106 }
 107
 108 func (s *SiteFSSuite) TestUsersDir(c *check.C) {
 109         s.testHomeProject(c, "/users/active")
 110 }
 111
 112 func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
 113         f, err := s.fs.Open(path)
 114         c.Assert(err, check.IsNil)
 115         fis, err := f.Readdir(-1)
 116         c.Assert(err, check.IsNil)
 117         c.Check(len(fis), check.Not(check.Equals), 0)
 118
 119         ok := false
 120         for _, fi := range fis {
 121                 c.Check(fi.Name(), check.Not(check.Equals), "")
 122                 if fi.Name() == "A Project" {
 123                         ok = true
 124                 }
 125         }
 126         c.Check(ok, check.Equals, true)
 127
 128         f, err = s.fs.Open(path + "/A Project/..")
 129         c.Assert(err, check.IsNil)
 130         fi, err := f.Stat()
 131         c.Assert(err, check.IsNil)
 132         c.Check(fi.IsDir(), check.Equals, true)
 133         _, basename := filepath.Split(path)
 134         c.Check(fi.Name(), check.Equals, basename)
 135
 136         f, err = s.fs.Open(path + "/A Project/A Subproject")
 137         c.Assert(err, check.IsNil)
 138         fi, err = f.Stat()
 139         c.Assert(err, check.IsNil)
 140         c.Check(fi.IsDir(), check.Equals, true)
 141
 142         for _, nx := range []string{
 143                 path + "/Unrestricted public data",
 144                 path + "/Unrestricted public data/does not exist",
 145                 path + "/A Project/does not exist",
 146         } {
 147                 c.Log(nx)
 148                 f, err = s.fs.Open(nx)
 149                 c.Check(err, check.NotNil)
 150                 c.Check(os.IsNotExist(err), check.Equals, true)
 151         }
 152 }
 153
 154 func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
 155         f, err := s.fs.Open("/users/active/A Project/A Subproject")
 156         c.Assert(err, check.IsNil)
 157         defer f.Close()
 158         f, err = s.fs.Open("/users/active/A Project/Project does not exist")
 159         c.Assert(err, check.NotNil)
 160         f, err = s.fs.Open("/users/active/A Project/A Subproject")
 161         c.Assert(err, check.IsNil)
 162         defer f.Close()
 163         f, err = s.fs.Open("/users/active/A Project")
 164         c.Assert(err, check.IsNil)
 165         defer f.Close()
 166         fis, err := f.Readdir(-1)
 167         c.Assert(err, check.IsNil)
 168         c.Logf("%#v", fis)
 169         var foundSubproject, foundCollection bool
 170         for _, fi := range fis {
 171                 switch fi.Name() {
 172                 case "A Subproject":
 173                         foundSubproject = true
 174                 case "collection_to_move_around":
 175                         foundCollection = true
 176                 }
 177         }
 178         c.Check(foundSubproject, check.Equals, true)
 179         c.Check(foundCollection, check.Equals, true)
 180 }
 181
 182 func (s *SiteFSSuite) TestSlashInName(c *check.C) {
 183         var badCollection Collection
 184         err := s.client.RequestAndDecode(&badCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 185                 "collection": map[string]string{
 186                         "name":       "bad/collection",
 187                         "owner_uuid": fixtureAProjectUUID,
 188                 },
 189         })
 190         c.Assert(err, check.IsNil)
 191         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+badCollection.UUID, nil, nil)
 192
 193         var badProject Group
 194         err = s.client.RequestAndDecode(&badProject, "POST", "arvados/v1/groups", nil, map[string]interface{}{
 195                 "group": map[string]string{
 196                         "name":        "bad/project",
 197                         "group_class": "project",
 198                         "owner_uuid":  fixtureAProjectUUID,
 199                 },
 200         })
 201         c.Assert(err, check.IsNil)
 202         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
 203
 204         dir, err := s.fs.Open("/users/active/A Project")
 205         c.Assert(err, check.IsNil)
 206         fis, err := dir.Readdir(-1)
 207         c.Check(err, check.IsNil)
 208         for _, fi := range fis {
 209                 c.Logf("fi.Name() == %q", fi.Name())
 210                 c.Check(strings.Contains(fi.Name(), "/"), check.Equals, false)
 211         }
 212
 213         // Make a new fs (otherwise content will still be cached from
 214         // above) and enable "/" replacement string.
 215         s.fs = s.client.SiteFileSystem(s.kc)
 216         s.fs.ForwardSlashNameSubstitution("___")
 217         dir, err = s.fs.Open("/users/active/A Project/bad___collection")
 218         if c.Check(err, check.IsNil) {
 219                 _, err = dir.Readdir(-1)
 220                 c.Check(err, check.IsNil)
 221         }
 222         dir, err = s.fs.Open("/users/active/A Project/bad___project")
 223         if c.Check(err, check.IsNil) {
 224                 _, err = dir.Readdir(-1)
 225                 c.Check(err, check.IsNil)
 226         }
 227 }
 228
 229 func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
 230         s.fs.MountProject("home", "")
 231
 232         project, err := s.fs.OpenFile("/home/A Project", 0, 0)
 233         c.Assert(err, check.IsNil)
 234
 235         _, err = s.fs.Open("/home/A Project/oob")
 236         c.Check(err, check.NotNil)
 237
 238         var oob Collection
 239         err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 240                 "collection": map[string]string{
 241                         "name":       "oob",
 242                         "owner_uuid": fixtureAProjectUUID,
 243                 },
 244         })
 245         c.Assert(err, check.IsNil)
 246         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
 247
 248         err = project.Sync()
 249         c.Check(err, check.IsNil)
 250         f, err := s.fs.Open("/home/A Project/oob")
 251         c.Assert(err, check.IsNil)
 252         fi, err := f.Stat()
 253         c.Assert(err, check.IsNil)
 254         c.Check(fi.IsDir(), check.Equals, true)
 255         f.Close()
 256
 257         wf, err := s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
 258         c.Assert(err, check.IsNil)
 259         _, err = wf.Write([]byte("hello oob\n"))
 260         c.Check(err, check.IsNil)
 261         err = wf.Close()
 262         c.Check(err, check.IsNil)
 263
 264         err = project.Sync()
 265         c.Check(err, check.IsNil)
 266         _, err = s.fs.Open("/home/A Project/oob/test.txt")
 267         c.Check(err, check.IsNil)
 268
 269         // Sync again to mark the project dir as stale, so the
 270         // collection gets reloaded from the controller on next
 271         // lookup.
 272         err = project.Sync()
 273         c.Check(err, check.IsNil)
 274
 275         // Ensure collection was flushed by Sync
 276         var latest Collection
 277         err = s.client.RequestAndDecode(&latest, "GET", "arvados/v1/collections/"+oob.UUID, nil, nil)
 278         c.Check(err, check.IsNil)
 279         c.Check(latest.ManifestText, check.Matches, `.*:test.txt.*\n`)
 280
 281         // Delete test.txt behind s.fs's back by updating the
 282         // collection record with an empty ManifestText.
 283         err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, nil, map[string]interface{}{
 284                 "collection": map[string]string{
 285                         "manifest_text":      "",
 286                         "portable_data_hash": "d41d8cd98f00b204e9800998ecf8427e+0",
 287                 },
 288         })
 289         c.Assert(err, check.IsNil)
 290
 291         _, err = s.fs.Open("/home/A Project/oob/test.txt")
 292         c.Check(err, check.NotNil)
 293         _, err = s.fs.Open("/home/A Project/oob")
 294         c.Check(err, check.IsNil)
 295
 296         err = s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
 297         c.Assert(err, check.IsNil)
 298
 299         wf, err = s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
 300         c.Assert(err, check.IsNil)
 301         err = wf.Close()
 302         c.Check(err, check.IsNil)
 303
 304         err = project.Sync()
 305         c.Check(err, check.NotNil) // can't update the deleted collection
 306         _, err = s.fs.Open("/home/A Project/oob")
 307         c.Check(err, check.IsNil) // parent dir still has old collection -- didn't reload, because Sync failed
 308 }
 309
 310 func (s *SiteFSSuite) TestProjectUnsupportedOperations(c *check.C) {
 311         s.fs.MountByID("by_id")
 312         s.fs.MountProject("home", "")
 313
 314         _, err := s.fs.OpenFile("/home/A Project/newfilename", os.O_CREATE|os.O_RDWR, 0)
 315         c.Check(err, ErrorIs, ErrInvalidOperation)
 316
 317         err = s.fs.Mkdir("/home/A Project/newdirname", 0)
 318         c.Check(err, ErrorIs, ErrInvalidOperation)
 319
 320         err = s.fs.Mkdir("/by_id/newdirname", 0)
 321         c.Check(err, ErrorIs, ErrInvalidOperation)
 322
 323         err = s.fs.Mkdir("/by_id/"+fixtureAProjectUUID+"/newdirname", 0)
 324         c.Check(err, ErrorIs, ErrInvalidOperation)
 325
 326         _, err = s.fs.OpenFile("/home/A Project", 0, 0)
 327         c.Check(err, check.IsNil)
 328 }
 329
 330 type errorIsChecker struct {
 331         *check.CheckerInfo
 332 }
 333
 334 var ErrorIs check.Checker = errorIsChecker{
 335         &check.CheckerInfo{Name: "ErrorIs", Params: []string{"value", "target"}},
 336 }
 337
 338 func (checker errorIsChecker) Check(params []interface{}, names []string) (result bool, errStr string) {
 339         err, ok := params[0].(error)
 340         if !ok {
 341                 return false, ""
 342         }
 343         target, ok := params[1].(error)
 344         if !ok {
 345                 return false, ""
 346         }
 347         return errors.Is(err, target), ""
 348 }