sdk/go/arvados/fs_site.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "net/http"
   9         "os"
  10         "strings"
  11         "sync"
  12         "time"
  13 )
  14
  15 type CustomFileSystem interface {
  16         FileSystem
  17         MountByID(mount string)
  18         MountProject(mount, uuid string)
  19         MountUsers(mount string)
  20         ForwardSlashNameSubstitution(string)
  21 }
  22
  23 type customFileSystem struct {
  24         fileSystem
  25         root *vdirnode
  26         thr  *throttle
  27
  28         staleThreshold time.Time
  29         staleLock      sync.Mutex
  30
  31         forwardSlashNameSubstitution string
  32 }
  33
  34 func (c *Client) CustomFileSystem(kc keepClient) CustomFileSystem {
  35         root := &vdirnode{}
  36         fs := &customFileSystem{
  37                 root: root,
  38                 fileSystem: fileSystem{
  39                         fsBackend: keepBackend{apiClient: c, keepClient: kc},
  40                         root:      root,
  41                         thr:       newThrottle(concurrentWriters),
  42                 },
  43         }
  44         root.treenode = treenode{
  45                 fs:     fs,
  46                 parent: root,
  47                 fileinfo: fileinfo{
  48                         name:    "/",
  49                         mode:    os.ModeDir | 0755,
  50                         modTime: time.Now(),
  51                 },
  52                 inodes: make(map[string]inode),
  53         }
  54         return fs
  55 }
  56
  57 func (fs *customFileSystem) MountByID(mount string) {
  58         fs.root.treenode.Lock()
  59         defer fs.root.treenode.Unlock()
  60         fs.root.treenode.Child(mount, func(inode) (inode, error) {
  61                 return &vdirnode{
  62                         treenode: treenode{
  63                                 fs:     fs,
  64                                 parent: fs.root,
  65                                 inodes: make(map[string]inode),
  66                                 fileinfo: fileinfo{
  67                                         name:    mount,
  68                                         modTime: time.Now(),
  69                                         mode:    0755 | os.ModeDir,
  70                                 },
  71                         },
  72                         create: fs.mountByID,
  73                 }, nil
  74         })
  75 }
  76
  77 func (fs *customFileSystem) MountProject(mount, uuid string) {
  78         fs.root.treenode.Lock()
  79         defer fs.root.treenode.Unlock()
  80         fs.root.treenode.Child(mount, func(inode) (inode, error) {
  81                 return fs.newProjectNode(fs.root, mount, uuid, nil), nil
  82         })
  83 }
  84
  85 func (fs *customFileSystem) MountUsers(mount string) {
  86         fs.root.treenode.Lock()
  87         defer fs.root.treenode.Unlock()
  88         fs.root.treenode.Child(mount, func(inode) (inode, error) {
  89                 return &lookupnode{
  90                         stale:   fs.Stale,
  91                         loadOne: fs.usersLoadOne,
  92                         loadAll: fs.usersLoadAll,
  93                         treenode: treenode{
  94                                 fs:     fs,
  95                                 parent: fs.root,
  96                                 inodes: make(map[string]inode),
  97                                 fileinfo: fileinfo{
  98                                         name:    mount,
  99                                         modTime: time.Now(),
 100                                         mode:    0755 | os.ModeDir,
 101                                 },
 102                         },
 103                 }, nil
 104         })
 105 }
 106
 107 func (fs *customFileSystem) ForwardSlashNameSubstitution(repl string) {
 108         fs.forwardSlashNameSubstitution = repl
 109 }
 110
 111 // SiteFileSystem returns a FileSystem that maps collections and other
 112 // Arvados objects onto a filesystem layout.
 113 //
 114 // This is experimental: the filesystem layout is not stable, and
 115 // there are significant known bugs and shortcomings. For example,
 116 // writes are not persisted until Sync() is called.
 117 func (c *Client) SiteFileSystem(kc keepClient) CustomFileSystem {
 118         fs := c.CustomFileSystem(kc)
 119         fs.MountByID("by_id")
 120         fs.MountUsers("users")
 121         return fs
 122 }
 123
 124 func (fs *customFileSystem) Sync() error {
 125         return fs.root.Sync()
 126 }
 127
 128 // Stale returns true if information obtained at time t should be
 129 // considered stale.
 130 func (fs *customFileSystem) Stale(t time.Time) bool {
 131         fs.staleLock.Lock()
 132         defer fs.staleLock.Unlock()
 133         return !fs.staleThreshold.Before(t)
 134 }
 135
 136 func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
 137         return nil, ErrInvalidOperation
 138 }
 139
 140 func (fs *customFileSystem) mountByID(parent inode, id string) (inode, error) {
 141         if strings.Contains(id, "-4zz18-") || pdhRegexp.MatchString(id) {
 142                 return fs.mountCollection(parent, id)
 143         } else if strings.Contains(id, "-j7d0g-") {
 144                 return fs.newProjectNode(fs.root, id, id, nil), nil
 145         } else {
 146                 return nil, nil
 147         }
 148 }
 149
 150 func (fs *customFileSystem) mountCollection(parent inode, id string) (inode, error) {
 151         var coll Collection
 152         err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
 153         if statusErr, ok := err.(interface{ HTTPStatus() int }); ok && statusErr.HTTPStatus() == http.StatusNotFound {
 154                 return nil, nil
 155         } else if err != nil {
 156                 return nil, err
 157         }
 158         if len(id) != 27 {
 159                 // This means id is a PDH, and controller/railsapi
 160                 // returned one of (possibly) many collections with
 161                 // that PDH. Even if controller returns more fields
 162                 // besides PDH and manifest text (which are equal for
 163                 // all matching collections), we don't want to expose
 164                 // them (e.g., through Sys()).
 165                 coll = Collection{
 166                         PortableDataHash: coll.PortableDataHash,
 167                         ManifestText:     coll.ManifestText,
 168                 }
 169         }
 170         newfs, err := coll.FileSystem(fs, fs)
 171         if err != nil {
 172                 return nil, err
 173         }
 174         cfs := newfs.(*collectionFileSystem)
 175         cfs.SetParent(parent, id)
 176         return cfs, nil
 177 }
 178
 179 func (fs *customFileSystem) newProjectNode(root inode, name, uuid string, proj *Group) inode {
 180         var projLoading sync.Mutex
 181         return &lookupnode{
 182                 stale:   fs.Stale,
 183                 loadOne: func(parent inode, name string) (inode, error) { return fs.projectsLoadOne(parent, uuid, name) },
 184                 loadAll: func(parent inode) ([]inode, error) { return fs.projectsLoadAll(parent, uuid) },
 185                 treenode: treenode{
 186                         fs:     fs,
 187                         parent: root,
 188                         inodes: make(map[string]inode),
 189                         fileinfo: fileinfo{
 190                                 name:    name,
 191                                 modTime: time.Now(),
 192                                 mode:    0755 | os.ModeDir,
 193                                 sys: func() interface{} {
 194                                         projLoading.Lock()
 195                                         defer projLoading.Unlock()
 196                                         if proj != nil {
 197                                                 return proj
 198                                         }
 199                                         var g Group
 200                                         err := fs.RequestAndDecode(&g, "GET", "arvados/v1/groups/"+uuid, nil, nil)
 201                                         if err != nil {
 202                                                 return err
 203                                         }
 204                                         proj = &g
 205                                         return proj
 206                                 },
 207                         },
 208                 },
 209         }
 210 }
 211
 212 // vdirnode wraps an inode by rejecting (with ErrInvalidOperation)
 213 // calls that add/replace children directly, instead calling a
 214 // create() func when a non-existing child is looked up.
 215 //
 216 // create() can return either a new node, which will be added to the
 217 // treenode, or nil for ENOENT.
 218 type vdirnode struct {
 219         treenode
 220         create func(parent inode, name string) (inode, error)
 221 }
 222
 223 func (vn *vdirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
 224         return vn.treenode.Child(name, func(existing inode) (inode, error) {
 225                 if existing == nil && vn.create != nil {
 226                         newnode, err := vn.create(vn, name)
 227                         if err != nil {
 228                                 return nil, err
 229                         }
 230                         if newnode != nil {
 231                                 newnode.SetParent(vn, name)
 232                                 existing = newnode
 233                                 vn.treenode.fileinfo.modTime = time.Now()
 234                         }
 235                 }
 236                 if replace == nil {
 237                         return existing, nil
 238                 } else if tryRepl, err := replace(existing); err != nil {
 239                         return existing, err
 240                 } else if tryRepl != existing {
 241                         return existing, ErrInvalidOperation
 242                 } else {
 243                         return existing, nil
 244                 }
 245         })
 246 }