sdk/go/arvados/fs_project_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "io"
  11         "os"
  12         "path/filepath"
  13         "strings"
  14
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 type spiedRequest struct {
  19         method string
  20         path   string
  21         params map[string]interface{}
  22 }
  23
  24 type spyingClient struct {
  25         *Client
  26         calls []spiedRequest
  27 }
  28
  29 func (sc *spyingClient) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
  30         var paramsCopy map[string]interface{}
  31         var buf bytes.Buffer
  32         json.NewEncoder(&buf).Encode(params)
  33         json.NewDecoder(&buf).Decode(&paramsCopy)
  34         sc.calls = append(sc.calls, spiedRequest{
  35                 method: method,
  36                 path:   path,
  37                 params: paramsCopy,
  38         })
  39         return sc.Client.RequestAndDecode(dst, method, path, body, params)
  40 }
  41
  42 func (s *SiteFSSuite) TestFilterGroup(c *check.C) {
  43         // Make sure that a collection and group that match the filter are present,
  44         // and that a group that does not match the filter is not present.
  45         s.fs.MountProject("fg", fixtureThisFilterGroupUUID)
  46
  47         _, err := s.fs.OpenFile("/fg/baz_file", 0, 0)
  48         c.Assert(err, check.IsNil)
  49
  50         _, err = s.fs.OpenFile("/fg/A Subproject", 0, 0)
  51         c.Assert(err, check.IsNil)
  52
  53         _, err = s.fs.OpenFile("/fg/A Project", 0, 0)
  54         c.Assert(err, check.Not(check.IsNil))
  55
  56         // An empty filter means everything that is visible should be returned.
  57         s.fs.MountProject("fg2", fixtureAFilterGroupTwoUUID)
  58
  59         _, err = s.fs.OpenFile("/fg2/baz_file", 0, 0)
  60         c.Assert(err, check.IsNil)
  61
  62         _, err = s.fs.OpenFile("/fg2/A Subproject", 0, 0)
  63         c.Assert(err, check.IsNil)
  64
  65         _, err = s.fs.OpenFile("/fg2/A Project", 0, 0)
  66         c.Assert(err, check.IsNil)
  67
  68         // An 'is_a' 'arvados#collection' filter means only collections should be returned.
  69         s.fs.MountProject("fg3", fixtureAFilterGroupThreeUUID)
  70
  71         _, err = s.fs.OpenFile("/fg3/baz_file", 0, 0)
  72         c.Assert(err, check.IsNil)
  73
  74         _, err = s.fs.OpenFile("/fg3/A Subproject", 0, 0)
  75         c.Assert(err, check.Not(check.IsNil))
  76 }
  77
  78 func (s *SiteFSSuite) TestCurrentUserHome(c *check.C) {
  79         s.fs.MountProject("home", "")
  80         s.testHomeProject(c, "/home")
  81 }
  82
  83 func (s *SiteFSSuite) TestUsersDir(c *check.C) {
  84         s.testHomeProject(c, "/users/active")
  85 }
  86
  87 func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
  88         f, err := s.fs.Open(path)
  89         c.Assert(err, check.IsNil)
  90         fis, err := f.Readdir(-1)
  91         c.Assert(err, check.IsNil)
  92         c.Check(len(fis), check.Not(check.Equals), 0)
  93
  94         ok := false
  95         for _, fi := range fis {
  96                 c.Check(fi.Name(), check.Not(check.Equals), "")
  97                 if fi.Name() == "A Project" {
  98                         ok = true
  99                 }
 100         }
 101         c.Check(ok, check.Equals, true)
 102
 103         f, err = s.fs.Open(path + "/A Project/..")
 104         c.Assert(err, check.IsNil)
 105         fi, err := f.Stat()
 106         c.Assert(err, check.IsNil)
 107         c.Check(fi.IsDir(), check.Equals, true)
 108         _, basename := filepath.Split(path)
 109         c.Check(fi.Name(), check.Equals, basename)
 110
 111         f, err = s.fs.Open(path + "/A Project/A Subproject")
 112         c.Assert(err, check.IsNil)
 113         fi, err = f.Stat()
 114         c.Assert(err, check.IsNil)
 115         c.Check(fi.IsDir(), check.Equals, true)
 116
 117         for _, nx := range []string{
 118                 path + "/Unrestricted public data",
 119                 path + "/Unrestricted public data/does not exist",
 120                 path + "/A Project/does not exist",
 121         } {
 122                 c.Log(nx)
 123                 f, err = s.fs.Open(nx)
 124                 c.Check(err, check.NotNil)
 125                 c.Check(os.IsNotExist(err), check.Equals, true)
 126         }
 127 }
 128
 129 func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
 130         f, err := s.fs.Open("/users/active/A Project/A Subproject")
 131         c.Assert(err, check.IsNil)
 132         defer f.Close()
 133         f, err = s.fs.Open("/users/active/A Project/Project does not exist")
 134         c.Assert(err, check.NotNil)
 135         f, err = s.fs.Open("/users/active/A Project/A Subproject")
 136         c.Assert(err, check.IsNil)
 137         defer f.Close()
 138         f, err = s.fs.Open("/users/active/A Project")
 139         c.Assert(err, check.IsNil)
 140         defer f.Close()
 141         fis, err := f.Readdir(-1)
 142         c.Assert(err, check.IsNil)
 143         c.Logf("%#v", fis)
 144         var foundSubproject, foundCollection bool
 145         for _, fi := range fis {
 146                 switch fi.Name() {
 147                 case "A Subproject":
 148                         foundSubproject = true
 149                 case "collection_to_move_around":
 150                         foundCollection = true
 151                 }
 152         }
 153         c.Check(foundSubproject, check.Equals, true)
 154         c.Check(foundCollection, check.Equals, true)
 155 }
 156
 157 func (s *SiteFSSuite) TestSlashInName(c *check.C) {
 158         var badCollection Collection
 159         err := s.client.RequestAndDecode(&badCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 160                 "collection": map[string]string{
 161                         "name":       "bad/collection",
 162                         "owner_uuid": fixtureAProjectUUID,
 163                 },
 164         })
 165         c.Assert(err, check.IsNil)
 166         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+badCollection.UUID, nil, nil)
 167
 168         var badProject Group
 169         err = s.client.RequestAndDecode(&badProject, "POST", "arvados/v1/groups", nil, map[string]interface{}{
 170                 "group": map[string]string{
 171                         "name":        "bad/project",
 172                         "group_class": "project",
 173                         "owner_uuid":  fixtureAProjectUUID,
 174                 },
 175         })
 176         c.Assert(err, check.IsNil)
 177         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
 178
 179         dir, err := s.fs.Open("/users/active/A Project")
 180         c.Assert(err, check.IsNil)
 181         fis, err := dir.Readdir(-1)
 182         c.Check(err, check.IsNil)
 183         for _, fi := range fis {
 184                 c.Logf("fi.Name() == %q", fi.Name())
 185                 c.Check(strings.Contains(fi.Name(), "/"), check.Equals, false)
 186         }
 187
 188         // Make a new fs (otherwise content will still be cached from
 189         // above) and enable "/" replacement string.
 190         s.fs = s.client.SiteFileSystem(s.kc)
 191         s.fs.ForwardSlashNameSubstitution("___")
 192         dir, err = s.fs.Open("/users/active/A Project/bad___collection")
 193         if c.Check(err, check.IsNil) {
 194                 _, err = dir.Readdir(-1)
 195                 c.Check(err, check.IsNil)
 196         }
 197         dir, err = s.fs.Open("/users/active/A Project/bad___project")
 198         if c.Check(err, check.IsNil) {
 199                 _, err = dir.Readdir(-1)
 200                 c.Check(err, check.IsNil)
 201         }
 202 }
 203
 204 func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
 205         s.fs.MountProject("home", "")
 206
 207         project, err := s.fs.OpenFile("/home/A Project", 0, 0)
 208         c.Assert(err, check.IsNil)
 209
 210         _, err = s.fs.Open("/home/A Project/oob")
 211         c.Check(err, check.NotNil)
 212
 213         var oob Collection
 214         err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 215                 "collection": map[string]string{
 216                         "name":       "oob",
 217                         "owner_uuid": fixtureAProjectUUID,
 218                 },
 219         })
 220         c.Assert(err, check.IsNil)
 221         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
 222
 223         err = project.Sync()
 224         c.Check(err, check.IsNil)
 225         f, err := s.fs.Open("/home/A Project/oob")
 226         c.Assert(err, check.IsNil)
 227         fi, err := f.Stat()
 228         c.Assert(err, check.IsNil)
 229         c.Check(fi.IsDir(), check.Equals, true)
 230         f.Close()
 231
 232         wf, err := s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
 233         c.Assert(err, check.IsNil)
 234         _, err = wf.Write([]byte("hello oob\n"))
 235         c.Check(err, check.IsNil)
 236         err = wf.Close()
 237         c.Check(err, check.IsNil)
 238
 239         err = project.Sync()
 240         c.Check(err, check.IsNil)
 241         _, err = s.fs.Open("/home/A Project/oob/test.txt")
 242         c.Check(err, check.IsNil)
 243
 244         // Sync again to mark the project dir as stale, so the
 245         // collection gets reloaded from the controller on next
 246         // lookup.
 247         err = project.Sync()
 248         c.Check(err, check.IsNil)
 249
 250         // Ensure collection was flushed by Sync
 251         var latest Collection
 252         err = s.client.RequestAndDecode(&latest, "GET", "arvados/v1/collections/"+oob.UUID, nil, nil)
 253         c.Check(err, check.IsNil)
 254         c.Check(latest.ManifestText, check.Matches, `.*:test.txt.*\n`)
 255
 256         // Delete test.txt behind s.fs's back by updating the
 257         // collection record with an empty ManifestText.
 258         err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, nil, map[string]interface{}{
 259                 "collection": map[string]string{
 260                         "manifest_text":      "",
 261                         "portable_data_hash": "d41d8cd98f00b204e9800998ecf8427e+0",
 262                 },
 263         })
 264         c.Assert(err, check.IsNil)
 265
 266         _, err = s.fs.Open("/home/A Project/oob/test.txt")
 267         c.Check(err, check.NotNil)
 268         _, err = s.fs.Open("/home/A Project/oob")
 269         c.Check(err, check.IsNil)
 270
 271         err = s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
 272         c.Assert(err, check.IsNil)
 273
 274         err = project.Sync()
 275         c.Check(err, check.NotNil) // can't update the deleted collection
 276         _, err = s.fs.Open("/home/A Project/oob")
 277         c.Check(err, check.IsNil) // parent dir still has old collection -- didn't reload, because Sync failed
 278 }
 279
 280 func (s *SiteFSSuite) TestProjectUnsupportedOperations(c *check.C) {
 281         s.fs.MountByID("by_id")
 282         s.fs.MountProject("home", "")
 283
 284         _, err := s.fs.OpenFile("/home/A Project/newfilename", os.O_CREATE|os.O_RDWR, 0)
 285         c.Check(err, check.ErrorMatches, "invalid argument")
 286
 287         err = s.fs.Mkdir("/home/A Project/newdirname", 0)
 288         c.Check(err, check.ErrorMatches, "invalid argument")
 289
 290         err = s.fs.Mkdir("/by_id/newdirname", 0)
 291         c.Check(err, check.ErrorMatches, "invalid argument")
 292
 293         err = s.fs.Mkdir("/by_id/"+fixtureAProjectUUID+"/newdirname", 0)
 294         c.Check(err, check.ErrorMatches, "invalid argument")
 295
 296         _, err = s.fs.OpenFile("/home/A Project", 0, 0)
 297         c.Check(err, check.IsNil)
 298 }