18874: Merge commit '6f8dcb2b13f3058db656908fb26b09e23b527f08' into 18874-merge-wb2
[arvados.git] / sdk / go / arvados / fs_project_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "bytes"
9         "encoding/json"
10         "errors"
11         "io"
12         "os"
13         "strings"
14
15         check "gopkg.in/check.v1"
16 )
17
18 type spiedRequest struct {
19         method string
20         path   string
21         params map[string]interface{}
22 }
23
24 type spyingClient struct {
25         *Client
26         calls []spiedRequest
27 }
28
29 func (sc *spyingClient) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
30         var paramsCopy map[string]interface{}
31         var buf bytes.Buffer
32         json.NewEncoder(&buf).Encode(params)
33         json.NewDecoder(&buf).Decode(&paramsCopy)
34         sc.calls = append(sc.calls, spiedRequest{
35                 method: method,
36                 path:   path,
37                 params: paramsCopy,
38         })
39         return sc.Client.RequestAndDecode(dst, method, path, body, params)
40 }
41
42 func (s *SiteFSSuite) TestFilterGroup(c *check.C) {
43         // Make sure that a collection and group that match the filter are present,
44         // and that a group that does not match the filter is not present.
45         s.fs.MountProject("fg", fixtureThisFilterGroupUUID)
46
47         _, err := s.fs.OpenFile("/fg/baz_file", 0, 0)
48         c.Assert(err, check.IsNil)
49
50         _, err = s.fs.OpenFile("/fg/A Subproject", 0, 0)
51         c.Assert(err, check.IsNil)
52
53         _, err = s.fs.OpenFile("/fg/A Project", 0, 0)
54         c.Assert(err, check.Not(check.IsNil))
55
56         // An empty filter means everything that is visible should be returned.
57         s.fs.MountProject("fg2", fixtureAFilterGroupTwoUUID)
58
59         _, err = s.fs.OpenFile("/fg2/baz_file", 0, 0)
60         c.Assert(err, check.IsNil)
61
62         _, err = s.fs.OpenFile("/fg2/A Subproject", 0, 0)
63         c.Assert(err, check.IsNil)
64
65         _, err = s.fs.OpenFile("/fg2/A Project", 0, 0)
66         c.Assert(err, check.IsNil)
67
68         // An 'is_a' 'arvados#collection' filter means only collections should be returned.
69         s.fs.MountProject("fg3", fixtureAFilterGroupThreeUUID)
70
71         _, err = s.fs.OpenFile("/fg3/baz_file", 0, 0)
72         c.Assert(err, check.IsNil)
73
74         _, err = s.fs.OpenFile("/fg3/A Subproject", 0, 0)
75         c.Assert(err, check.Not(check.IsNil))
76
77         // An 'exists' 'arvados#collection' filter means only collections with certain properties should be returned.
78         s.fs.MountProject("fg4", fixtureAFilterGroupFourUUID)
79
80         _, err = s.fs.Stat("/fg4/collection with list property with odd values")
81         c.Assert(err, check.IsNil)
82
83         _, err = s.fs.Stat("/fg4/collection with list property with even values")
84         c.Assert(err, check.IsNil)
85
86         // A 'contains' 'arvados#collection' filter means only collections with certain properties should be returned.
87         s.fs.MountProject("fg5", fixtureAFilterGroupFiveUUID)
88
89         _, err = s.fs.Stat("/fg5/collection with list property with odd values")
90         c.Assert(err, check.IsNil)
91
92         _, err = s.fs.Stat("/fg5/collection with list property with string value")
93         c.Assert(err, check.IsNil)
94
95         _, err = s.fs.Stat("/fg5/collection with prop2 5")
96         c.Assert(err, check.Not(check.IsNil))
97
98         _, err = s.fs.Stat("/fg5/collection with list property with even values")
99         c.Assert(err, check.Not(check.IsNil))
100 }
101
102 func (s *SiteFSSuite) TestCurrentUserHome(c *check.C) {
103         s.fs.MountProject("home", "")
104         s.testHomeProject(c, "/home", "home")
105 }
106
107 func (s *SiteFSSuite) TestUsersDir(c *check.C) {
108         // /users/active is a hardlink to a dir whose name is the UUID
109         // of the active user
110         s.testHomeProject(c, "/users/active", fixtureActiveUserUUID)
111 }
112
113 func (s *SiteFSSuite) testHomeProject(c *check.C, path, expectRealName string) {
114         f, err := s.fs.Open(path)
115         c.Assert(err, check.IsNil)
116         fis, err := f.Readdir(-1)
117         c.Assert(err, check.IsNil)
118         c.Check(len(fis), check.Not(check.Equals), 0)
119
120         ok := false
121         for _, fi := range fis {
122                 c.Check(fi.Name(), check.Not(check.Equals), "")
123                 if fi.Name() == "A Project" {
124                         ok = true
125                 }
126         }
127         c.Check(ok, check.Equals, true)
128
129         f, err = s.fs.Open(path + "/A Project/..")
130         c.Assert(err, check.IsNil)
131         fi, err := f.Stat()
132         c.Assert(err, check.IsNil)
133         c.Check(fi.IsDir(), check.Equals, true)
134         c.Check(fi.Name(), check.Equals, expectRealName)
135
136         f, err = s.fs.Open(path + "/A Project/A Subproject")
137         c.Assert(err, check.IsNil)
138         fi, err = f.Stat()
139         c.Assert(err, check.IsNil)
140         c.Check(fi.IsDir(), check.Equals, true)
141
142         for _, nx := range []string{
143                 path + "/Unrestricted public data",
144                 path + "/Unrestricted public data/does not exist",
145                 path + "/A Project/does not exist",
146         } {
147                 c.Log(nx)
148                 f, err = s.fs.Open(nx)
149                 c.Check(err, check.NotNil)
150                 c.Check(os.IsNotExist(err), check.Equals, true)
151         }
152 }
153
154 func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
155         f, err := s.fs.Open("/users/active/A Project/A Subproject")
156         c.Assert(err, check.IsNil)
157         defer f.Close()
158         f, err = s.fs.Open("/users/active/A Project/Project does not exist")
159         c.Assert(err, check.NotNil)
160         f, err = s.fs.Open("/users/active/A Project/A Subproject")
161         c.Assert(err, check.IsNil)
162         defer f.Close()
163         f, err = s.fs.Open("/users/active/A Project")
164         c.Assert(err, check.IsNil)
165         defer f.Close()
166         fis, err := f.Readdir(-1)
167         c.Assert(err, check.IsNil)
168         c.Logf("%#v", fis)
169         var foundSubproject, foundCollection bool
170         for _, fi := range fis {
171                 switch fi.Name() {
172                 case "A Subproject":
173                         foundSubproject = true
174                 case "collection_to_move_around":
175                         foundCollection = true
176                 }
177         }
178         c.Check(foundSubproject, check.Equals, true)
179         c.Check(foundCollection, check.Equals, true)
180 }
181
182 func (s *SiteFSSuite) TestSlashInName(c *check.C) {
183         var badCollection Collection
184         err := s.client.RequestAndDecode(&badCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
185                 "collection": map[string]string{
186                         "name":       "bad/collection",
187                         "owner_uuid": fixtureAProjectUUID,
188                 },
189         })
190         c.Assert(err, check.IsNil)
191         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+badCollection.UUID, nil, nil)
192
193         var badProject Group
194         err = s.client.RequestAndDecode(&badProject, "POST", "arvados/v1/groups", nil, map[string]interface{}{
195                 "group": map[string]string{
196                         "name":        "bad/project",
197                         "group_class": "project",
198                         "owner_uuid":  fixtureAProjectUUID,
199                 },
200         })
201         c.Assert(err, check.IsNil)
202         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
203
204         dir, err := s.fs.Open("/users/active/A Project")
205         c.Assert(err, check.IsNil)
206         fis, err := dir.Readdir(-1)
207         c.Check(err, check.IsNil)
208         for _, fi := range fis {
209                 c.Logf("fi.Name() == %q", fi.Name())
210                 c.Check(strings.Contains(fi.Name(), "/"), check.Equals, false)
211         }
212
213         // Make a new fs (otherwise content will still be cached from
214         // above) and enable "/" replacement string.
215         s.fs = s.client.SiteFileSystem(s.kc)
216         s.fs.ForwardSlashNameSubstitution("___")
217         dir, err = s.fs.Open("/users/active/A Project/bad___collection")
218         if c.Check(err, check.IsNil) {
219                 _, err = dir.Readdir(-1)
220                 c.Check(err, check.IsNil)
221         }
222         dir, err = s.fs.Open("/users/active/A Project/bad___project")
223         if c.Check(err, check.IsNil) {
224                 _, err = dir.Readdir(-1)
225                 c.Check(err, check.IsNil)
226         }
227 }
228
229 func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
230         s.fs.MountProject("home", "")
231
232         project, err := s.fs.OpenFile("/home/A Project", 0, 0)
233         c.Assert(err, check.IsNil)
234
235         _, err = s.fs.Open("/home/A Project/oob")
236         c.Check(err, check.NotNil)
237
238         var oob Collection
239         err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", nil, map[string]interface{}{
240                 "collection": map[string]string{
241                         "name":       "oob",
242                         "owner_uuid": fixtureAProjectUUID,
243                 },
244         })
245         c.Assert(err, check.IsNil)
246         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
247
248         err = project.Sync()
249         c.Check(err, check.IsNil)
250         f, err := s.fs.Open("/home/A Project/oob")
251         c.Assert(err, check.IsNil)
252         fi, err := f.Stat()
253         c.Assert(err, check.IsNil)
254         c.Check(fi.IsDir(), check.Equals, true)
255         f.Close()
256
257         wf, err := s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
258         c.Assert(err, check.IsNil)
259         _, err = wf.Write([]byte("hello oob\n"))
260         c.Check(err, check.IsNil)
261         err = wf.Close()
262         c.Check(err, check.IsNil)
263
264         err = project.Sync()
265         c.Check(err, check.IsNil)
266         f, err = s.fs.Open("/home/A Project/oob/test.txt")
267         if c.Check(err, check.IsNil) {
268                 f.Close()
269         }
270
271         // Ensure collection was flushed by Sync
272         var latest Collection
273         err = s.client.RequestAndDecode(&latest, "GET", "arvados/v1/collections/"+oob.UUID, nil, nil)
274         c.Check(err, check.IsNil)
275         c.Check(latest.ManifestText, check.Matches, `.*:test.txt.*\n`)
276
277         // Delete test.txt behind s.fs's back by updating the
278         // collection record with an empty ManifestText.
279         err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, nil, map[string]interface{}{
280                 "collection": map[string]string{
281                         "manifest_text":      "",
282                         "portable_data_hash": "d41d8cd98f00b204e9800998ecf8427e+0",
283                 },
284         })
285         c.Assert(err, check.IsNil)
286
287         // Sync again to reload collection.
288         err = project.Sync()
289         c.Check(err, check.IsNil)
290
291         // Check test.txt deletion is reflected in fs.
292         _, err = s.fs.Open("/home/A Project/oob/test.txt")
293         c.Check(err, check.NotNil)
294         f, err = s.fs.Open("/home/A Project/oob")
295         if c.Check(err, check.IsNil) {
296                 f.Close()
297         }
298
299         err = s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
300         c.Assert(err, check.IsNil)
301
302         wf, err = s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
303         c.Assert(err, check.IsNil)
304         err = wf.Close()
305         c.Check(err, check.IsNil)
306
307         err = project.Sync()
308         c.Check(err, check.NotNil) // can't update the deleted collection
309         _, err = s.fs.Open("/home/A Project/oob")
310         c.Check(err, check.IsNil) // parent dir still has old collection -- didn't reload, because Sync failed
311 }
312
313 func (s *SiteFSSuite) TestProjectUnsupportedOperations(c *check.C) {
314         s.fs.MountByID("by_id")
315         s.fs.MountProject("home", "")
316
317         _, err := s.fs.OpenFile("/home/A Project/newfilename", os.O_CREATE|os.O_RDWR, 0)
318         c.Check(err, ErrorIs, ErrInvalidOperation)
319
320         err = s.fs.Mkdir("/home/A Project/newdirname", 0)
321         c.Check(err, ErrorIs, ErrInvalidOperation)
322
323         err = s.fs.Mkdir("/by_id/newdirname", 0)
324         c.Check(err, ErrorIs, ErrInvalidOperation)
325
326         err = s.fs.Mkdir("/by_id/"+fixtureAProjectUUID+"/newdirname", 0)
327         c.Check(err, ErrorIs, ErrInvalidOperation)
328
329         _, err = s.fs.OpenFile("/home/A Project", 0, 0)
330         c.Check(err, check.IsNil)
331 }
332
333 type errorIsChecker struct {
334         *check.CheckerInfo
335 }
336
337 var ErrorIs check.Checker = errorIsChecker{
338         &check.CheckerInfo{Name: "ErrorIs", Params: []string{"value", "target"}},
339 }
340
341 func (checker errorIsChecker) Check(params []interface{}, names []string) (result bool, errStr string) {
342         err, ok := params[0].(error)
343         if !ok {
344                 return false, ""
345         }
346         target, ok := params[1].(error)
347         if !ok {
348                 return false, ""
349         }
350         return errors.Is(err, target), ""
351 }