sdk/go/arvados/fs_project_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "io"
  11         "os"
  12         "path/filepath"
  13         "strings"
  14
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 type spiedRequest struct {
  19         method string
  20         path   string
  21         params map[string]interface{}
  22 }
  23
  24 type spyingClient struct {
  25         *Client
  26         calls []spiedRequest
  27 }
  28
  29 func (sc *spyingClient) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
  30         var paramsCopy map[string]interface{}
  31         var buf bytes.Buffer
  32         json.NewEncoder(&buf).Encode(params)
  33         json.NewDecoder(&buf).Decode(&paramsCopy)
  34         sc.calls = append(sc.calls, spiedRequest{
  35                 method: method,
  36                 path:   path,
  37                 params: paramsCopy,
  38         })
  39         return sc.Client.RequestAndDecode(dst, method, path, body, params)
  40 }
  41
  42 func (s *SiteFSSuite) TestFilterGroup(c *check.C) {
  43         // Make sure that a collection and group that match the filter are present,
  44         // and that a group that does not match the filter is not present.
  45         s.fs.MountProject("fg", fixtureThisFilterGroupUUID)
  46
  47         _, err := s.fs.OpenFile("/fg/baz_file", 0, 0)
  48         c.Assert(err, check.IsNil)
  49
  50         _, err = s.fs.OpenFile("/fg/A Subproject", 0, 0)
  51         c.Assert(err, check.IsNil)
  52
  53         _, err = s.fs.OpenFile("/fg/A Project", 0, 0)
  54         c.Assert(err, check.Not(check.IsNil))
  55
  56         // An empty filter means everything that is visible should be returned.
  57         s.fs.MountProject("fg2", fixtureAFilterGroupTwoUUID)
  58
  59         _, err = s.fs.OpenFile("/fg2/baz_file", 0, 0)
  60         c.Assert(err, check.IsNil)
  61
  62         _, err = s.fs.OpenFile("/fg2/A Subproject", 0, 0)
  63         c.Assert(err, check.IsNil)
  64
  65         _, err = s.fs.OpenFile("/fg2/A Project", 0, 0)
  66         c.Assert(err, check.IsNil)
  67
  68         // An 'is_a' 'arvados#collection' filter means only collections should be returned.
  69         s.fs.MountProject("fg3", fixtureAFilterGroupThreeUUID)
  70
  71         _, err = s.fs.OpenFile("/fg3/baz_file", 0, 0)
  72         c.Assert(err, check.IsNil)
  73
  74         _, err = s.fs.OpenFile("/fg3/A Subproject", 0, 0)
  75         c.Assert(err, check.Not(check.IsNil))
  76
  77         // An 'exists' 'arvados#collection' filter means only collections with certain properties should be returned.
  78         s.fs.MountProject("fg4", fixtureAFilterGroupFourUUID)
  79
  80         _, err = s.fs.Stat("/fg4/collection with list property with odd values")
  81         c.Assert(err, check.IsNil)
  82
  83         _, err = s.fs.Stat("/fg4/collection with list property with even values")
  84         c.Assert(err, check.IsNil)
  85
  86         // A 'contains' 'arvados#collection' filter means only collections with certain properties should be returned.
  87         s.fs.MountProject("fg5", fixtureAFilterGroupFiveUUID)
  88
  89         _, err = s.fs.Stat("/fg5/collection with list property with odd values")
  90         c.Assert(err, check.IsNil)
  91
  92         _, err = s.fs.Stat("/fg5/collection with list property with string value")
  93         c.Assert(err, check.IsNil)
  94
  95         _, err = s.fs.Stat("/fg5/collection with prop2 5")
  96         c.Assert(err, check.Not(check.IsNil))
  97
  98         _, err = s.fs.Stat("/fg5/collection with list property with even values")
  99         c.Assert(err, check.Not(check.IsNil))
 100 }
 101
 102 func (s *SiteFSSuite) TestCurrentUserHome(c *check.C) {
 103         s.fs.MountProject("home", "")
 104         s.testHomeProject(c, "/home")
 105 }
 106
 107 func (s *SiteFSSuite) TestUsersDir(c *check.C) {
 108         s.testHomeProject(c, "/users/active")
 109 }
 110
 111 func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
 112         f, err := s.fs.Open(path)
 113         c.Assert(err, check.IsNil)
 114         fis, err := f.Readdir(-1)
 115         c.Assert(err, check.IsNil)
 116         c.Check(len(fis), check.Not(check.Equals), 0)
 117
 118         ok := false
 119         for _, fi := range fis {
 120                 c.Check(fi.Name(), check.Not(check.Equals), "")
 121                 if fi.Name() == "A Project" {
 122                         ok = true
 123                 }
 124         }
 125         c.Check(ok, check.Equals, true)
 126
 127         f, err = s.fs.Open(path + "/A Project/..")
 128         c.Assert(err, check.IsNil)
 129         fi, err := f.Stat()
 130         c.Assert(err, check.IsNil)
 131         c.Check(fi.IsDir(), check.Equals, true)
 132         _, basename := filepath.Split(path)
 133         c.Check(fi.Name(), check.Equals, basename)
 134
 135         f, err = s.fs.Open(path + "/A Project/A Subproject")
 136         c.Assert(err, check.IsNil)
 137         fi, err = f.Stat()
 138         c.Assert(err, check.IsNil)
 139         c.Check(fi.IsDir(), check.Equals, true)
 140
 141         for _, nx := range []string{
 142                 path + "/Unrestricted public data",
 143                 path + "/Unrestricted public data/does not exist",
 144                 path + "/A Project/does not exist",
 145         } {
 146                 c.Log(nx)
 147                 f, err = s.fs.Open(nx)
 148                 c.Check(err, check.NotNil)
 149                 c.Check(os.IsNotExist(err), check.Equals, true)
 150         }
 151 }
 152
 153 func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
 154         f, err := s.fs.Open("/users/active/A Project/A Subproject")
 155         c.Assert(err, check.IsNil)
 156         defer f.Close()
 157         f, err = s.fs.Open("/users/active/A Project/Project does not exist")
 158         c.Assert(err, check.NotNil)
 159         f, err = s.fs.Open("/users/active/A Project/A Subproject")
 160         c.Assert(err, check.IsNil)
 161         defer f.Close()
 162         f, err = s.fs.Open("/users/active/A Project")
 163         c.Assert(err, check.IsNil)
 164         defer f.Close()
 165         fis, err := f.Readdir(-1)
 166         c.Assert(err, check.IsNil)
 167         c.Logf("%#v", fis)
 168         var foundSubproject, foundCollection bool
 169         for _, fi := range fis {
 170                 switch fi.Name() {
 171                 case "A Subproject":
 172                         foundSubproject = true
 173                 case "collection_to_move_around":
 174                         foundCollection = true
 175                 }
 176         }
 177         c.Check(foundSubproject, check.Equals, true)
 178         c.Check(foundCollection, check.Equals, true)
 179 }
 180
 181 func (s *SiteFSSuite) TestSlashInName(c *check.C) {
 182         var badCollection Collection
 183         err := s.client.RequestAndDecode(&badCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 184                 "collection": map[string]string{
 185                         "name":       "bad/collection",
 186                         "owner_uuid": fixtureAProjectUUID,
 187                 },
 188         })
 189         c.Assert(err, check.IsNil)
 190         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+badCollection.UUID, nil, nil)
 191
 192         var badProject Group
 193         err = s.client.RequestAndDecode(&badProject, "POST", "arvados/v1/groups", nil, map[string]interface{}{
 194                 "group": map[string]string{
 195                         "name":        "bad/project",
 196                         "group_class": "project",
 197                         "owner_uuid":  fixtureAProjectUUID,
 198                 },
 199         })
 200         c.Assert(err, check.IsNil)
 201         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
 202
 203         dir, err := s.fs.Open("/users/active/A Project")
 204         c.Assert(err, check.IsNil)
 205         fis, err := dir.Readdir(-1)
 206         c.Check(err, check.IsNil)
 207         for _, fi := range fis {
 208                 c.Logf("fi.Name() == %q", fi.Name())
 209                 c.Check(strings.Contains(fi.Name(), "/"), check.Equals, false)
 210         }
 211
 212         // Make a new fs (otherwise content will still be cached from
 213         // above) and enable "/" replacement string.
 214         s.fs = s.client.SiteFileSystem(s.kc)
 215         s.fs.ForwardSlashNameSubstitution("___")
 216         dir, err = s.fs.Open("/users/active/A Project/bad___collection")
 217         if c.Check(err, check.IsNil) {
 218                 _, err = dir.Readdir(-1)
 219                 c.Check(err, check.IsNil)
 220         }
 221         dir, err = s.fs.Open("/users/active/A Project/bad___project")
 222         if c.Check(err, check.IsNil) {
 223                 _, err = dir.Readdir(-1)
 224                 c.Check(err, check.IsNil)
 225         }
 226 }
 227
 228 func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
 229         s.fs.MountProject("home", "")
 230
 231         project, err := s.fs.OpenFile("/home/A Project", 0, 0)
 232         c.Assert(err, check.IsNil)
 233
 234         _, err = s.fs.Open("/home/A Project/oob")
 235         c.Check(err, check.NotNil)
 236
 237         var oob Collection
 238         err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 239                 "collection": map[string]string{
 240                         "name":       "oob",
 241                         "owner_uuid": fixtureAProjectUUID,
 242                 },
 243         })
 244         c.Assert(err, check.IsNil)
 245         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
 246
 247         err = project.Sync()
 248         c.Check(err, check.IsNil)
 249         f, err := s.fs.Open("/home/A Project/oob")
 250         c.Assert(err, check.IsNil)
 251         fi, err := f.Stat()
 252         c.Assert(err, check.IsNil)
 253         c.Check(fi.IsDir(), check.Equals, true)
 254         f.Close()
 255
 256         wf, err := s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
 257         c.Assert(err, check.IsNil)
 258         _, err = wf.Write([]byte("hello oob\n"))
 259         c.Check(err, check.IsNil)
 260         err = wf.Close()
 261         c.Check(err, check.IsNil)
 262
 263         err = project.Sync()
 264         c.Check(err, check.IsNil)
 265         _, err = s.fs.Open("/home/A Project/oob/test.txt")
 266         c.Check(err, check.IsNil)
 267
 268         // Sync again to mark the project dir as stale, so the
 269         // collection gets reloaded from the controller on next
 270         // lookup.
 271         err = project.Sync()
 272         c.Check(err, check.IsNil)
 273
 274         // Ensure collection was flushed by Sync
 275         var latest Collection
 276         err = s.client.RequestAndDecode(&latest, "GET", "arvados/v1/collections/"+oob.UUID, nil, nil)
 277         c.Check(err, check.IsNil)
 278         c.Check(latest.ManifestText, check.Matches, `.*:test.txt.*\n`)
 279
 280         // Delete test.txt behind s.fs's back by updating the
 281         // collection record with an empty ManifestText.
 282         err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, nil, map[string]interface{}{
 283                 "collection": map[string]string{
 284                         "manifest_text":      "",
 285                         "portable_data_hash": "d41d8cd98f00b204e9800998ecf8427e+0",
 286                 },
 287         })
 288         c.Assert(err, check.IsNil)
 289
 290         _, err = s.fs.Open("/home/A Project/oob/test.txt")
 291         c.Check(err, check.NotNil)
 292         _, err = s.fs.Open("/home/A Project/oob")
 293         c.Check(err, check.IsNil)
 294
 295         err = s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
 296         c.Assert(err, check.IsNil)
 297
 298         err = project.Sync()
 299         c.Check(err, check.NotNil) // can't update the deleted collection
 300         _, err = s.fs.Open("/home/A Project/oob")
 301         c.Check(err, check.IsNil) // parent dir still has old collection -- didn't reload, because Sync failed
 302 }
 303
 304 func (s *SiteFSSuite) TestProjectUnsupportedOperations(c *check.C) {
 305         s.fs.MountByID("by_id")
 306         s.fs.MountProject("home", "")
 307
 308         _, err := s.fs.OpenFile("/home/A Project/newfilename", os.O_CREATE|os.O_RDWR, 0)
 309         c.Check(err, check.ErrorMatches, "invalid argument")
 310
 311         err = s.fs.Mkdir("/home/A Project/newdirname", 0)
 312         c.Check(err, check.ErrorMatches, "invalid argument")
 313
 314         err = s.fs.Mkdir("/by_id/newdirname", 0)
 315         c.Check(err, check.ErrorMatches, "invalid argument")
 316
 317         err = s.fs.Mkdir("/by_id/"+fixtureAProjectUUID+"/newdirname", 0)
 318         c.Check(err, check.ErrorMatches, "invalid argument")
 319
 320         _, err = s.fs.OpenFile("/home/A Project", 0, 0)
 321         c.Check(err, check.IsNil)
 322 }