sdk/go/arvados/fs_site.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "os"
   9         "strings"
  10         "sync"
  11         "time"
  12 )
  13
  14 type CustomFileSystem interface {
  15         FileSystem
  16         MountByID(mount string)
  17         MountProject(mount, uuid string)
  18         MountUsers(mount string)
  19         ForwardSlashNameSubstitution(string)
  20 }
  21
  22 type customFileSystem struct {
  23         fileSystem
  24         root *vdirnode
  25         thr  *throttle
  26
  27         staleThreshold time.Time
  28         staleLock      sync.Mutex
  29
  30         forwardSlashNameSubstitution string
  31 }
  32
  33 func (c *Client) CustomFileSystem(kc keepClient) CustomFileSystem {
  34         root := &vdirnode{}
  35         fs := &customFileSystem{
  36                 root: root,
  37                 fileSystem: fileSystem{
  38                         fsBackend: keepBackend{apiClient: c, keepClient: kc},
  39                         root:      root,
  40                         thr:       newThrottle(concurrentWriters),
  41                 },
  42         }
  43         root.treenode = treenode{
  44                 fs:     fs,
  45                 parent: root,
  46                 fileinfo: fileinfo{
  47                         name:    "/",
  48                         mode:    os.ModeDir | 0755,
  49                         modTime: time.Now(),
  50                 },
  51                 inodes: make(map[string]inode),
  52         }
  53         return fs
  54 }
  55
  56 func (fs *customFileSystem) MountByID(mount string) {
  57         fs.root.treenode.Lock()
  58         defer fs.root.treenode.Unlock()
  59         fs.root.treenode.Child(mount, func(inode) (inode, error) {
  60                 return &vdirnode{
  61                         treenode: treenode{
  62                                 fs:     fs,
  63                                 parent: fs.root,
  64                                 inodes: make(map[string]inode),
  65                                 fileinfo: fileinfo{
  66                                         name:    mount,
  67                                         modTime: time.Now(),
  68                                         mode:    0755 | os.ModeDir,
  69                                 },
  70                         },
  71                         create: fs.mountByID,
  72                 }, nil
  73         })
  74 }
  75
  76 func (fs *customFileSystem) MountProject(mount, uuid string) {
  77         fs.root.treenode.Lock()
  78         defer fs.root.treenode.Unlock()
  79         fs.root.treenode.Child(mount, func(inode) (inode, error) {
  80                 return fs.newProjectNode(fs.root, mount, uuid, nil), nil
  81         })
  82 }
  83
  84 func (fs *customFileSystem) MountUsers(mount string) {
  85         fs.root.treenode.Lock()
  86         defer fs.root.treenode.Unlock()
  87         fs.root.treenode.Child(mount, func(inode) (inode, error) {
  88                 return &lookupnode{
  89                         stale:   fs.Stale,
  90                         loadOne: fs.usersLoadOne,
  91                         loadAll: fs.usersLoadAll,
  92                         treenode: treenode{
  93                                 fs:     fs,
  94                                 parent: fs.root,
  95                                 inodes: make(map[string]inode),
  96                                 fileinfo: fileinfo{
  97                                         name:    mount,
  98                                         modTime: time.Now(),
  99                                         mode:    0755 | os.ModeDir,
 100                                 },
 101                         },
 102                 }, nil
 103         })
 104 }
 105
 106 func (fs *customFileSystem) ForwardSlashNameSubstitution(repl string) {
 107         fs.forwardSlashNameSubstitution = repl
 108 }
 109
 110 // SiteFileSystem returns a FileSystem that maps collections and other
 111 // Arvados objects onto a filesystem layout.
 112 //
 113 // This is experimental: the filesystem layout is not stable, and
 114 // there are significant known bugs and shortcomings. For example,
 115 // writes are not persisted until Sync() is called.
 116 func (c *Client) SiteFileSystem(kc keepClient) CustomFileSystem {
 117         fs := c.CustomFileSystem(kc)
 118         fs.MountByID("by_id")
 119         fs.MountUsers("users")
 120         return fs
 121 }
 122
 123 func (fs *customFileSystem) Sync() error {
 124         return fs.root.Sync()
 125 }
 126
 127 // Stale returns true if information obtained at time t should be
 128 // considered stale.
 129 func (fs *customFileSystem) Stale(t time.Time) bool {
 130         fs.staleLock.Lock()
 131         defer fs.staleLock.Unlock()
 132         return !fs.staleThreshold.Before(t)
 133 }
 134
 135 func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
 136         return nil, ErrInvalidOperation
 137 }
 138
 139 func (fs *customFileSystem) mountByID(parent inode, id string) inode {
 140         if strings.Contains(id, "-4zz18-") || pdhRegexp.MatchString(id) {
 141                 return fs.mountCollection(parent, id)
 142         } else if strings.Contains(id, "-j7d0g-") {
 143                 return fs.newProjectNode(fs.root, id, id, nil)
 144         } else {
 145                 return nil
 146         }
 147 }
 148
 149 func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
 150         var coll Collection
 151         err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
 152         if err != nil {
 153                 return nil
 154         }
 155         newfs, err := coll.FileSystem(fs, fs)
 156         if err != nil {
 157                 return nil
 158         }
 159         cfs := newfs.(*collectionFileSystem)
 160         cfs.SetParent(parent, id)
 161         return cfs
 162 }
 163
 164 func (fs *customFileSystem) newProjectNode(root inode, name, uuid string, proj *Group) inode {
 165         var projLoading sync.Mutex
 166         return &lookupnode{
 167                 stale:   fs.Stale,
 168                 loadOne: func(parent inode, name string) (inode, error) { return fs.projectsLoadOne(parent, uuid, name) },
 169                 loadAll: func(parent inode) ([]inode, error) { return fs.projectsLoadAll(parent, uuid) },
 170                 treenode: treenode{
 171                         fs:     fs,
 172                         parent: root,
 173                         inodes: make(map[string]inode),
 174                         fileinfo: fileinfo{
 175                                 name:    name,
 176                                 modTime: time.Now(),
 177                                 mode:    0755 | os.ModeDir,
 178                                 sys: func() interface{} {
 179                                         projLoading.Lock()
 180                                         defer projLoading.Unlock()
 181                                         if proj != nil {
 182                                                 return proj
 183                                         }
 184                                         var g Group
 185                                         err := fs.RequestAndDecode(&g, "GET", "arvados/v1/groups/"+uuid, nil, nil)
 186                                         if err != nil {
 187                                                 return err
 188                                         }
 189                                         proj = &g
 190                                         return proj
 191                                 },
 192                         },
 193                 },
 194         }
 195 }
 196
 197 // vdirnode wraps an inode by rejecting (with ErrInvalidOperation)
 198 // calls that add/replace children directly, instead calling a
 199 // create() func when a non-existing child is looked up.
 200 //
 201 // create() can return either a new node, which will be added to the
 202 // treenode, or nil for ENOENT.
 203 type vdirnode struct {
 204         treenode
 205         create func(parent inode, name string) inode
 206 }
 207
 208 func (vn *vdirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
 209         return vn.treenode.Child(name, func(existing inode) (inode, error) {
 210                 if existing == nil && vn.create != nil {
 211                         existing = vn.create(vn, name)
 212                         if existing != nil {
 213                                 existing.SetParent(vn, name)
 214                                 vn.treenode.fileinfo.modTime = time.Now()
 215                         }
 216                 }
 217                 if replace == nil {
 218                         return existing, nil
 219                 } else if tryRepl, err := replace(existing); err != nil {
 220                         return existing, err
 221                 } else if tryRepl != existing {
 222                         return existing, ErrInvalidOperation
 223                 } else {
 224                         return existing, nil
 225                 }
 226         })
 227 }