X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/18c8fa2337a2db04ee6060184885731f4b5c7d7b..15f6e58f3b7c158aa735ce78597cf22c6ef3543b:/sdk/go/arvados/fs_site.go

diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 53261317b1..5225df59ee 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -6,23 +6,43 @@ package arvados
 
 import (
 	"os"
+	"strings"
+	"sync"
 	"time"
 )
 
-// SiteFileSystem returns a FileSystem that maps collections and other
-// Arvados objects onto a filesystem layout.
-//
-// This is experimental: the filesystem layout is not stable, and
-// there are significant known bugs and shortcomings. For example,
-// although the FileSystem allows files to be added and modified in
-// collections, these changes are not persistent or visible to other
-// Arvados clients.
-func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
-	fs := &fileSystem{
-		fsBackend: keepBackend{apiClient: c, keepClient: kc},
+type CustomFileSystem interface {
+	FileSystem
+	MountByID(mount string)
+	MountProject(mount, uuid string)
+	MountUsers(mount string)
+	ForwardSlashNameSubstitution(string)
+}
+
+type customFileSystem struct {
+	fileSystem
+	root *vdirnode
+	thr  *throttle
+
+	staleThreshold time.Time
+	staleLock      sync.Mutex
+
+	forwardSlashNameSubstitution string
+}
+
+func (c *Client) CustomFileSystem(kc keepClient) CustomFileSystem {
+	root := &vdirnode{}
+	fs := &customFileSystem{
+		root: root,
+		fileSystem: fileSystem{
+			fsBackend: keepBackend{apiClient: c, keepClient: kc},
+			root:      root,
+			thr:       newThrottle(concurrentWriters),
+		},
 	}
-	root := &treenode{
-		fs: fs,
+	root.treenode = treenode{
+		fs:     fs,
+		parent: root,
 		fileinfo: fileinfo{
 			name:    "/",
 			mode:    os.ModeDir | 0755,
@@ -30,57 +50,163 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
 		},
 		inodes: make(map[string]inode),
 	}
-	root.parent = root
-	root.Child("by_id", func(inode) inode {
-		var vn inode
-		vn = &vdirnode{
+	return fs
+}
+
+func (fs *customFileSystem) MountByID(mount string) {
+	fs.root.treenode.Lock()
+	defer fs.root.treenode.Unlock()
+	fs.root.treenode.Child(mount, func(inode) (inode, error) {
+		return &vdirnode{
 			treenode: treenode{
 				fs:     fs,
-				parent: root,
+				parent: fs.root,
 				inodes: make(map[string]inode),
 				fileinfo: fileinfo{
-					name:    "by_id",
+					name:    mount,
 					modTime: time.Now(),
 					mode:    0755 | os.ModeDir,
 				},
 			},
-			create: func(name string) inode {
-				return newEntByID(vn, name)
+			create: fs.mountByID,
+		}, nil
+	})
+}
+
+func (fs *customFileSystem) MountProject(mount, uuid string) {
+	fs.root.treenode.Lock()
+	defer fs.root.treenode.Unlock()
+	fs.root.treenode.Child(mount, func(inode) (inode, error) {
+		return fs.newProjectNode(fs.root, mount, uuid), nil
+	})
+}
+
+func (fs *customFileSystem) MountUsers(mount string) {
+	fs.root.treenode.Lock()
+	defer fs.root.treenode.Unlock()
+	fs.root.treenode.Child(mount, func(inode) (inode, error) {
+		return &lookupnode{
+			stale:   fs.Stale,
+			loadOne: fs.usersLoadOne,
+			loadAll: fs.usersLoadAll,
+			treenode: treenode{
+				fs:     fs,
+				parent: fs.root,
+				inodes: make(map[string]inode),
+				fileinfo: fileinfo{
+					name:    mount,
+					modTime: time.Now(),
+					mode:    0755 | os.ModeDir,
+				},
 			},
-		}
-		return vn
+		}, nil
 	})
-	fs.root = root
+}
+
+func (fs *customFileSystem) ForwardSlashNameSubstitution(repl string) {
+	fs.forwardSlashNameSubstitution = repl
+}
+
+// SiteFileSystem returns a FileSystem that maps collections and other
+// Arvados objects onto a filesystem layout.
+//
+// This is experimental: the filesystem layout is not stable, and
+// there are significant known bugs and shortcomings. For example,
+// writes are not persisted until Sync() is called.
+func (c *Client) SiteFileSystem(kc keepClient) CustomFileSystem {
+	fs := c.CustomFileSystem(kc)
+	fs.MountByID("by_id")
+	fs.MountUsers("users")
 	return fs
 }
 
-func newEntByID(parent inode, id string) inode {
+func (fs *customFileSystem) Sync() error {
+	return fs.root.Sync()
+}
+
+// Stale returns true if information obtained at time t should be
+// considered stale.
+func (fs *customFileSystem) Stale(t time.Time) bool {
+	fs.staleLock.Lock()
+	defer fs.staleLock.Unlock()
+	return !fs.staleThreshold.Before(t)
+}
+
+func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+	return nil, ErrInvalidArgument
+}
+
+func (fs *customFileSystem) mountByID(parent inode, id string) inode {
+	if strings.Contains(id, "-4zz18-") || pdhRegexp.MatchString(id) {
+		return fs.mountCollection(parent, id)
+	} else if strings.Contains(id, "-j7d0g-") {
+		return fs.newProjectNode(fs.root, id, id)
+	} else {
+		return nil
+	}
+}
+
+func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
 	var coll Collection
-	err := parent.FS().RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+	err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
 	if err != nil {
 		return nil
 	}
-	fs, err := coll.FileSystem(parent.FS(), parent.FS())
+	newfs, err := coll.FileSystem(fs, fs)
 	if err != nil {
 		return nil
 	}
-	root := fs.(*collectionFileSystem).root.(*dirnode)
-	root.fileinfo.name = id
-	root.parent = parent
-	return root
+	cfs := newfs.(*collectionFileSystem)
+	cfs.SetParent(parent, id)
+	return cfs
 }
 
+func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode {
+	return &lookupnode{
+		stale:   fs.Stale,
+		loadOne: func(parent inode, name string) (inode, error) { return fs.projectsLoadOne(parent, uuid, name) },
+		loadAll: func(parent inode) ([]inode, error) { return fs.projectsLoadAll(parent, uuid) },
+		treenode: treenode{
+			fs:     fs,
+			parent: root,
+			inodes: make(map[string]inode),
+			fileinfo: fileinfo{
+				name:    name,
+				modTime: time.Now(),
+				mode:    0755 | os.ModeDir,
+			},
+		},
+	}
+}
+
+// vdirnode wraps an inode by rejecting (with ErrInvalidArgument)
+// calls that add/replace children directly, instead calling a
+// create() func when a non-existing child is looked up.
+//
+// create() can return either a new node, which will be added to the
+// treenode, or nil for ENOENT.
 type vdirnode struct {
 	treenode
-	create func(string) inode
+	create func(parent inode, name string) inode
 }
 
-func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
-	return vn.treenode.Child(name, func(existing inode) inode {
-		if existing != nil {
-			return existing
+func (vn *vdirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+	return vn.treenode.Child(name, func(existing inode) (inode, error) {
+		if existing == nil && vn.create != nil {
+			existing = vn.create(vn, name)
+			if existing != nil {
+				existing.SetParent(vn, name)
+				vn.treenode.fileinfo.modTime = time.Now()
+			}
+		}
+		if replace == nil {
+			return existing, nil
+		} else if tryRepl, err := replace(existing); err != nil {
+			return existing, err
+		} else if tryRepl != existing {
+			return existing, ErrInvalidArgument
 		} else {
-			return vn.create(name)
+			return existing, nil
 		}
 	})
 }