From 29a6622585581b5e4f519968bbd291939bf49392 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Tue, 13 Feb 2018 17:29:38 -0500 Subject: [PATCH] 13111: Reload project dir if fsync(2) was called since last load. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- sdk/go/arvados/collection.go | 1 + sdk/go/arvados/fs_project.go | 26 +++++++++---- sdk/go/arvados/fs_project_test.go | 62 ++++++++++++++++++++++++++++--- sdk/go/arvados/fs_site.go | 19 ++++++++++ sdk/go/arvados/fs_site_test.go | 6 ++- sdk/go/arvadostest/fixtures.go | 3 ++ 6 files changed, 103 insertions(+), 14 deletions(-) diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go index 999b4e9d48..aea0cc043f 100644 --- a/sdk/go/arvados/collection.go +++ b/sdk/go/arvados/collection.go @@ -16,6 +16,7 @@ import ( // Collection is an arvados#collection resource. type Collection struct { UUID string `json:"uuid,omitempty"` + OwnerUUID string `json:"owner_uuid,omitempty"` TrashAt *time.Time `json:"trash_at,omitempty"` ManifestText string `json:"manifest_text,omitempty"` UnsignedManifestText string `json:"unsigned_manifest_text,omitempty"` diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go index a5e4710633..776c8a3365 100644 --- a/sdk/go/arvados/fs_project.go +++ b/sdk/go/arvados/fs_project.go @@ -7,18 +7,29 @@ package arvados import ( "os" "sync" + "time" ) // projectnode exposes an Arvados project as a filesystem directory. type projectnode struct { inode - uuid string - setupOnce sync.Once - err error + uuid string + err error + + loadLock sync.Mutex + loadStart time.Time } -func (pn *projectnode) setup() { +func (pn *projectnode) load() { fs := pn.FS().(*siteFileSystem) + + pn.loadLock.Lock() + defer pn.loadLock.Unlock() + if !fs.Stale(pn.loadStart) { + return + } + pn.loadStart = time.Now() + if pn.uuid == "" { var resp User pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil) @@ -36,7 +47,6 @@ func (pn *projectnode) setup() { var resp CollectionList pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params) if pn.err != nil { - // TODO: retry on next access, instead of returning the same error forever return } if len(resp.Items) == 0 { @@ -60,7 +70,6 @@ func (pn *projectnode) setup() { var resp GroupList pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params) if pn.err != nil { - // TODO: retry on next access, instead of returning the same error forever return } if len(resp.Items) == 0 { @@ -76,10 +85,11 @@ func (pn *projectnode) setup() { } params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}) } + pn.err = nil } func (pn *projectnode) Readdir() ([]os.FileInfo, error) { - pn.setupOnce.Do(pn.setup) + pn.load() if pn.err != nil { return nil, pn.err } @@ -87,7 +97,7 @@ func (pn *projectnode) Readdir() ([]os.FileInfo, error) { } func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) { - pn.setupOnce.Do(pn.setup) + pn.load() if pn.err != nil { return nil, pn.err } diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go index 1de07142fd..b7dc08e3c2 100644 --- a/sdk/go/arvados/fs_project_test.go +++ b/sdk/go/arvados/fs_project_test.go @@ -10,6 +10,7 @@ import ( "io" "os" + "git.curoverse.com/arvados.git/sdk/go/arvadostest" check "gopkg.in/check.v1" ) @@ -46,29 +47,29 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) { ok := false for _, fi := range fis { c.Check(fi.Name(), check.Not(check.Equals), "") - if fi.Name() == "Unrestricted public data" { + if fi.Name() == "A Project" { ok = true } } c.Check(ok, check.Equals, true) - f, err = s.fs.Open("/home/Unrestricted public data/..") + f, err = s.fs.Open("/home/A Project/..") c.Assert(err, check.IsNil) fi, err := f.Stat() c.Check(err, check.IsNil) c.Check(fi.IsDir(), check.Equals, true) c.Check(fi.Name(), check.Equals, "home") - f, err = s.fs.Open("/home/Unrestricted public data/Subproject in anonymous accessible project") + f, err = s.fs.Open("/home/A Project/A Subproject") c.Check(err, check.IsNil) fi, err = f.Stat() c.Check(err, check.IsNil) c.Check(fi.IsDir(), check.Equals, true) for _, nx := range []string{ - "/home/A Project", - "/home/A Project/does not exist", + "/home/Unrestricted public data", "/home/Unrestricted public data/does not exist", + "/home/A Project/does not exist", } { c.Log(nx) f, err = s.fs.Open(nx) @@ -76,3 +77,54 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) { c.Check(os.IsNotExist(err), check.Equals, true) } } + +func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) { + project, err := s.fs.OpenFile("/home/A Project", 0, 0) + c.Check(err, check.IsNil) + + _, err = s.fs.Open("/home/A Project/oob") + c.Check(err, check.NotNil) + + oob := Collection{ + Name: "oob", + OwnerUUID: arvadostest.AProjectUUID, + } + err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", s.client.UpdateBody(&oob), nil) + c.Assert(err, check.IsNil) + defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil) + + err = project.Sync() + c.Check(err, check.IsNil) + f, err := s.fs.Open("/home/A Project/oob") + c.Assert(err, check.IsNil) + fi, err := f.Stat() + c.Check(fi.IsDir(), check.Equals, true) + f.Close() + + wf, err := s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700) + c.Assert(err, check.IsNil) + _, err = wf.Write([]byte("hello oob\n")) + c.Check(err, check.IsNil) + err = wf.Close() + c.Check(err, check.IsNil) + + // Delete test.txt behind s.fs's back by updating the + // collection record with the old (empty) ManifestText. + err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, s.client.UpdateBody(&oob), nil) + c.Assert(err, check.IsNil) + + err = project.Sync() + c.Check(err, check.IsNil) + _, err = s.fs.Open("/home/A Project/oob/test.txt") + c.Check(err, check.NotNil) + _, err = s.fs.Open("/home/A Project/oob") + c.Check(err, check.IsNil) + + err = s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil) + c.Assert(err, check.IsNil) + + err = project.Sync() + c.Check(err, check.IsNil) + _, err = s.fs.Open("/home/A Project/oob") + c.Check(err, check.NotNil) +} diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go index 701711e217..cdcf40e114 100644 --- a/sdk/go/arvados/fs_site.go +++ b/sdk/go/arvados/fs_site.go @@ -6,11 +6,15 @@ package arvados import ( "os" + "sync" "time" ) type siteFileSystem struct { fileSystem + + staleThreshold time.Time + staleLock sync.Mutex } // SiteFileSystem returns a FileSystem that maps collections and other @@ -58,6 +62,21 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem { return fs } +func (fs *siteFileSystem) Sync() error { + fs.staleLock.Lock() + defer fs.staleLock.Unlock() + fs.staleThreshold = time.Now() + return nil +} + +// Stale returns true if information obtained at time t should be +// considered stale. +func (fs *siteFileSystem) Stale(t time.Time) bool { + fs.staleLock.Lock() + defer fs.staleLock.Unlock() + return !fs.staleThreshold.Before(t) +} + func (fs *siteFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) { return nil, ErrInvalidOperation } diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go index 26a2212a07..a3a9712ae8 100644 --- a/sdk/go/arvados/fs_site_test.go +++ b/sdk/go/arvados/fs_site_test.go @@ -21,7 +21,11 @@ type SiteFSSuite struct { } func (s *SiteFSSuite) SetUpTest(c *check.C) { - s.client = NewClientFromEnv() + s.client = &Client{ + APIHost: os.Getenv("ARVADOS_API_HOST"), + AuthToken: arvadostest.ActiveToken, + Insecure: true, + } s.kc = &keepClientStub{ blocks: map[string][]byte{ "3858f62230ac3c915f300c664312c63f": []byte("foobar"), diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go index d057c09b22..5fccfb3aa2 100644 --- a/sdk/go/arvadostest/fixtures.go +++ b/sdk/go/arvadostest/fixtures.go @@ -25,6 +25,9 @@ const ( FooPdh = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45" HelloWorldPdh = "55713e6a34081eb03609e7ad5fcad129+62" + AProjectUUID = "zzzzz-j7d0g-v955i6s2oi1cbso" + ASubprojectUUID = "zzzzz-j7d0g-axqo7eu9pwvna1x" + FooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir" FooAndBarFilesInDirPDH = "6bbac24198d09a93975f60098caf0bdf+62" -- 2.30.2