13111: Reload project dir if fsync(2) was called since last load.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index 999b4e9d483454ace177cad829e90f85ddccc44c..aea0cc043f40f6460feaa6ecc3f26408cdc54e18 100644 (file)
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -16,6 +16,7 @@ import (
 // Collection is an arvados#collection resource.
 type Collection struct {
        UUID                   string     `json:"uuid,omitempty"`
+       OwnerUUID              string     `json:"owner_uuid,omitempty"`
        TrashAt                *time.Time `json:"trash_at,omitempty"`
        ManifestText           string     `json:"manifest_text,omitempty"`
        UnsignedManifestText   string     `json:"unsigned_manifest_text,omitempty"`

diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index a5e4710633b9952b748d3535b8feb36a7594c1da..776c8a3365178e62874c24d1ac237691e9f60a9a 100644 (file)
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -7,18 +7,29 @@ package arvados
 import (
        "os"
        "sync"
+       "time"
 )
 
 // projectnode exposes an Arvados project as a filesystem directory.
 type projectnode struct {
        inode
-       uuid      string
-       setupOnce sync.Once
-       err       error
+       uuid string
+       err  error
+
+       loadLock  sync.Mutex
+       loadStart time.Time
 }
 
-func (pn *projectnode) setup() {
+func (pn *projectnode) load() {
        fs := pn.FS().(*siteFileSystem)
+
+       pn.loadLock.Lock()
+       defer pn.loadLock.Unlock()
+       if !fs.Stale(pn.loadStart) {
+               return
+       }
+       pn.loadStart = time.Now()
+
        if pn.uuid == "" {
                var resp User
                pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
@@ -36,7 +47,6 @@ func (pn *projectnode) setup() {
                var resp CollectionList
                pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
                if pn.err != nil {
-                       // TODO: retry on next access, instead of returning the same error forever
                        return
                }
                if len(resp.Items) == 0 {
@@ -60,7 +70,6 @@ func (pn *projectnode) setup() {
                var resp GroupList
                pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
                if pn.err != nil {
-                       // TODO: retry on next access, instead of returning the same error forever
                        return
                }
                if len(resp.Items) == 0 {
@@ -76,10 +85,11 @@ func (pn *projectnode) setup() {
                }
                params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
        }
+       pn.err = nil
 }
 
 func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
-       pn.setupOnce.Do(pn.setup)
+       pn.load()
        if pn.err != nil {
                return nil, pn.err
        }
@@ -87,7 +97,7 @@ func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
 }
 
 func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
-       pn.setupOnce.Do(pn.setup)
+       pn.load()
        if pn.err != nil {
                return nil, pn.err
        }

diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 1de07142fd5b723dbe2aaca4196230e1675fd1b5..b7dc08e3c2b67fd0834042fc0541353a33a24972 100644 (file)
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -10,6 +10,7 @@ import (
        "io"
        "os"
 
+       "git.curoverse.com/arvados.git/sdk/go/arvadostest"
        check "gopkg.in/check.v1"
 )
 
@@ -46,29 +47,29 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
        ok := false
        for _, fi := range fis {
                c.Check(fi.Name(), check.Not(check.Equals), "")
-               if fi.Name() == "Unrestricted public data" {
+               if fi.Name() == "A Project" {
                        ok = true
                }
        }
        c.Check(ok, check.Equals, true)
 
-       f, err = s.fs.Open("/home/Unrestricted public data/..")
+       f, err = s.fs.Open("/home/A Project/..")
        c.Assert(err, check.IsNil)
        fi, err := f.Stat()
        c.Check(err, check.IsNil)
        c.Check(fi.IsDir(), check.Equals, true)
        c.Check(fi.Name(), check.Equals, "home")
 
-       f, err = s.fs.Open("/home/Unrestricted public data/Subproject in anonymous accessible project")
+       f, err = s.fs.Open("/home/A Project/A Subproject")
        c.Check(err, check.IsNil)
        fi, err = f.Stat()
        c.Check(err, check.IsNil)
        c.Check(fi.IsDir(), check.Equals, true)
 
        for _, nx := range []string{
-               "/home/A Project",
-               "/home/A Project/does not exist",
+               "/home/Unrestricted public data",
                "/home/Unrestricted public data/does not exist",
+               "/home/A Project/does not exist",
        } {
                c.Log(nx)
                f, err = s.fs.Open(nx)
@@ -76,3 +77,54 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
                c.Check(os.IsNotExist(err), check.Equals, true)
        }
 }
+
+func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
+       project, err := s.fs.OpenFile("/home/A Project", 0, 0)
+       c.Check(err, check.IsNil)
+
+       _, err = s.fs.Open("/home/A Project/oob")
+       c.Check(err, check.NotNil)
+
+       oob := Collection{
+               Name:      "oob",
+               OwnerUUID: arvadostest.AProjectUUID,
+       }
+       err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", s.client.UpdateBody(&oob), nil)
+       c.Assert(err, check.IsNil)
+       defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
+
+       err = project.Sync()
+       c.Check(err, check.IsNil)
+       f, err := s.fs.Open("/home/A Project/oob")
+       c.Assert(err, check.IsNil)
+       fi, err := f.Stat()
+       c.Check(fi.IsDir(), check.Equals, true)
+       f.Close()
+
+       wf, err := s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
+       c.Assert(err, check.IsNil)
+       _, err = wf.Write([]byte("hello oob\n"))
+       c.Check(err, check.IsNil)
+       err = wf.Close()
+       c.Check(err, check.IsNil)
+
+       // Delete test.txt behind s.fs's back by updating the
+       // collection record with the old (empty) ManifestText.
+       err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, s.client.UpdateBody(&oob), nil)
+       c.Assert(err, check.IsNil)
+
+       err = project.Sync()
+       c.Check(err, check.IsNil)
+       _, err = s.fs.Open("/home/A Project/oob/test.txt")
+       c.Check(err, check.NotNil)
+       _, err = s.fs.Open("/home/A Project/oob")
+       c.Check(err, check.IsNil)
+
+       err = s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
+       c.Assert(err, check.IsNil)
+
+       err = project.Sync()
+       c.Check(err, check.IsNil)
+       _, err = s.fs.Open("/home/A Project/oob")
+       c.Check(err, check.NotNil)
+}

diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 701711e21777b5645f923e4258b2e4b2cdf1dce4..cdcf40e114418176293bb83d141c9c8b71f72eb6 100644 (file)
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -6,11 +6,15 @@ package arvados
 
 import (
        "os"
+       "sync"
        "time"
 )
 
 type siteFileSystem struct {
        fileSystem
+
+       staleThreshold time.Time
+       staleLock      sync.Mutex
 }
 
 // SiteFileSystem returns a FileSystem that maps collections and other
@@ -58,6 +62,21 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
        return fs
 }
 
+func (fs *siteFileSystem) Sync() error {
+       fs.staleLock.Lock()
+       defer fs.staleLock.Unlock()
+       fs.staleThreshold = time.Now()
+       return nil
+}
+
+// Stale returns true if information obtained at time t should be
+// considered stale.
+func (fs *siteFileSystem) Stale(t time.Time) bool {
+       fs.staleLock.Lock()
+       defer fs.staleLock.Unlock()
+       return !fs.staleThreshold.Before(t)
+}
+
 func (fs *siteFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
        return nil, ErrInvalidOperation
 }

diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index 26a2212a0771b1c12e5cb7c2825e59b48dd6cfcd..a3a9712ae8c8bda87160897ab5aeb2ee7addf4b4 100644 (file)
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -21,7 +21,11 @@ type SiteFSSuite struct {
 }
 
 func (s *SiteFSSuite) SetUpTest(c *check.C) {
-       s.client = NewClientFromEnv()
+       s.client = &Client{
+               APIHost:   os.Getenv("ARVADOS_API_HOST"),
+               AuthToken: arvadostest.ActiveToken,
+               Insecure:  true,
+       }
        s.kc = &keepClientStub{
                blocks: map[string][]byte{
                        "3858f62230ac3c915f300c664312c63f": []byte("foobar"),

diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index d057c09b227e9f375d2b3d04e95d9327044c4f33..5fccfb3aa21d5cd348b2ccf309639cd9789ac881 100644 (file)
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -25,6 +25,9 @@ const (
        FooPdh                  = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
        HelloWorldPdh           = "55713e6a34081eb03609e7ad5fcad129+62"
 
+       AProjectUUID    = "zzzzz-j7d0g-v955i6s2oi1cbso"
+       ASubprojectUUID = "zzzzz-j7d0g-axqo7eu9pwvna1x"
+
        FooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
        FooAndBarFilesInDirPDH  = "6bbac24198d09a93975f60098caf0bdf+62"