1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
15 type CustomFileSystem interface {
17 MountByID(mount string)
18 MountProject(mount, uuid string)
19 MountUsers(mount string)
20 ForwardSlashNameSubstitution(string)
23 type customFileSystem struct {
28 staleThreshold time.Time
31 forwardSlashNameSubstitution string
34 func (c *Client) CustomFileSystem(kc keepClient) CustomFileSystem {
36 fs := &customFileSystem{
38 fileSystem: fileSystem{
39 fsBackend: keepBackend{apiClient: c, keepClient: kc},
41 thr: newThrottle(concurrentWriters),
44 root.inode = &treenode{
49 mode: os.ModeDir | 0755,
52 inodes: make(map[string]inode),
57 func (fs *customFileSystem) MountByID(mount string) {
58 fs.root.inode.Child(mount, func(inode) (inode, error) {
63 inodes: make(map[string]inode),
67 mode: 0755 | os.ModeDir,
75 func (fs *customFileSystem) MountProject(mount, uuid string) {
76 fs.root.inode.Child(mount, func(inode) (inode, error) {
77 return fs.newProjectNode(fs.root, mount, uuid), nil
81 func (fs *customFileSystem) MountUsers(mount string) {
82 fs.root.inode.Child(mount, func(inode) (inode, error) {
85 loadOne: fs.usersLoadOne,
86 loadAll: fs.usersLoadAll,
90 inodes: make(map[string]inode),
94 mode: 0755 | os.ModeDir,
101 func (fs *customFileSystem) ForwardSlashNameSubstitution(repl string) {
102 fs.forwardSlashNameSubstitution = repl
105 // SiteFileSystem returns a FileSystem that maps collections and other
106 // Arvados objects onto a filesystem layout.
108 // This is experimental: the filesystem layout is not stable, and
109 // there are significant known bugs and shortcomings. For example,
110 // writes are not persisted until Sync() is called.
111 func (c *Client) SiteFileSystem(kc keepClient) CustomFileSystem {
112 fs := c.CustomFileSystem(kc)
113 fs.MountByID("by_id")
114 fs.MountUsers("users")
118 func (fs *customFileSystem) Sync() error {
120 fs.staleThreshold = time.Now()
121 fs.staleLock.Unlock()
122 return fs.syncTree("/", fs.root.inode)
125 // syncTree calls node.Sync() if it has its own Sync method, otherwise
126 // it calls syncTree() on all of node's children.
128 // Returns ErrInvalidArgument if node does not implement Sync() and
129 // isn't a directory (or if Readdir() fails for any other reason).
130 func (fs *customFileSystem) syncTree(path string, node inode) error {
131 if vn, ok := node.(*vdirnode); ok {
134 if syncer, ok := node.(interface{ Sync() error }); ok {
137 return fmt.Errorf("%s: %T: %w", path, syncer, err)
141 fis, err := node.Readdir()
143 return fmt.Errorf("%s: %T: %w", path, node, ErrInvalidArgument)
145 for _, fi := range fis {
146 child, err := node.Child(fi.Name(), nil)
150 err = fs.syncTree(path+"/"+fi.Name(), child)
158 // Stale returns true if information obtained at time t should be
160 func (fs *customFileSystem) Stale(t time.Time) bool {
162 defer fs.staleLock.Unlock()
163 return !fs.staleThreshold.Before(t)
166 func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
167 return nil, ErrInvalidOperation
170 func (fs *customFileSystem) mountByID(parent inode, id string) inode {
171 if strings.Contains(id, "-4zz18-") || pdhRegexp.MatchString(id) {
172 return fs.mountCollection(parent, id)
173 } else if strings.Contains(id, "-j7d0g-") {
174 return fs.newProjectNode(fs.root, id, id)
180 func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
182 err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
186 newfs, err := coll.FileSystem(fs, fs)
190 cfs := newfs.(*collectionFileSystem)
191 cfs.SetParent(parent, id)
195 func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode {
198 loadOne: func(parent inode, name string) (inode, error) { return fs.projectsLoadOne(parent, uuid, name) },
199 loadAll: func(parent inode) ([]inode, error) { return fs.projectsLoadAll(parent, uuid) },
203 inodes: make(map[string]inode),
207 mode: 0755 | os.ModeDir,
213 // vdirnode wraps an inode by ignoring any requests to add/replace
214 // children, and calling a create() func when a non-existing child is
217 // create() can return either a new node, which will be added to the
218 // treenode, or nil for ENOENT.
219 type vdirnode struct {
221 create func(parent inode, name string) inode
224 func (vn *vdirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
225 return vn.inode.Child(name, func(existing inode) (inode, error) {
226 if existing == nil && vn.create != nil {
227 existing = vn.create(vn, name)
229 existing.SetParent(vn, name)
230 vn.inode.(*treenode).fileinfo.modTime = time.Now()
235 } else if tryRepl, err := replace(existing); err != nil {
237 } else if tryRepl != existing {
238 return existing, ErrInvalidArgument