sdk/python/bin/arv-mount2

   1 #!/usr/bin/env python
   2
   3 import os
   4 import sys
   5
   6 import llfuse
   7 import errno
   8 import stat
   9 import threading
  10 import arvados
  11 import argparse
  12 import pprint
  13
  14 from time import time
  15 from llfuse import FUSEError
  16
  17 class Directory(object):
  18     def __init__(self, inode, parent):
  19         self.inode = inode
  20         self.parent = parent
  21         self.entries = {}
  22
  23     def __getitem__(self, item):
  24         return self.entries[item]
  25
  26     def __setitem__(self, key, item):
  27         self.entries[key] = item
  28
  29     def __iter__(self):
  30         return self.entries.iterkeys()
  31
  32     def __contains__(self, k):
  33         return k in self.entries
  34
  35     def size(self):
  36         return 0
  37
  38 class File(object):
  39     def __init__(self, inode, parent, reader):
  40         self.inode = inode
  41         self.parent = parent
  42         self.reader = reader
  43
  44     def size(self):
  45         return self.reader.size()
  46
  47 class FileHandle(object):
  48     def __init__(self, fh, entry):
  49         self.fh = fh
  50         self.entry = entry
  51
  52 class Operations(llfuse.Operations):
  53
  54     def __init__(self, collection):
  55         super(Operations, self).__init__()
  56         #self.api = arvados.api('v1')
  57
  58         # dict of inodes to collection entry
  59         self._inodes = {}
  60
  61         i = 1
  62         self.root = Directory(i, i)
  63         self._inodes[i] = self.root
  64
  65         for s in collection.all_streams():
  66             cwd = self.root
  67             for part in s.name().split('/'):
  68                 if part != '' and part != '.':
  69                     if part not in cwd:
  70                         i += 1
  71                         cwd[part] = Directory(i, cwd.inode)
  72                         self._inodes[i] = cwd[part]
  73                     cwd = cwd[part]
  74             for k, v in s.files().items():
  75                 i += 1
  76                 cwd[k] = File(i, cwd.inode, v)
  77                 self._inodes[i] = cwd[k]
  78
  79         # dict of inode to filehandle
  80         self._filehandles = {}
  81         self._filehandles_lock = threading.Lock()
  82         self._filehandles_counter = 1
  83
  84     def access(self, inode, mode, ctx):
  85         return True
  86
  87     def getattr(self, inode):
  88         e = self._inodes[inode]
  89
  90         entry = llfuse.EntryAttributes()
  91         entry.st_ino = inode
  92         entry.generation = 0
  93         entry.entry_timeout = 300
  94         entry.attr_timeout = 300
  95
  96         entry.st_mode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
  97         if isinstance(e, Directory):
  98             entry.st_mode |= stat.S_IFDIR
  99         else:
 100             entry.st_mode |= stat.S_IFREG
 101
 102         entry.st_nlink = 1
 103         entry.st_uid = 0
 104         entry.st_gid = 0
 105         entry.st_rdev = 0
 106
 107         entry.st_size = e.size()
 108
 109         entry.st_blksize = 512
 110         entry.st_blocks = 1
 111         entry.st_atime = 0
 112         entry.st_mtime = 0
 113         entry.st_ctime = 0
 114
 115         return entry
 116
 117     def lookup(self, parent_inode, name):
 118         #print "lookup: parent_inode", parent_inode, "name", name
 119         inode = None
 120
 121         if name == '.':
 122             inode = parent_inode
 123         else:
 124             if parent_inode in self._inodes:
 125                 p = self._inodes[parent_inode]
 126                 if name == '..':
 127                     inode = p.parent
 128                 elif name in p:
 129                     inode = p[name].inode
 130
 131         if inode != None:
 132             return self.getattr(inode)
 133         else:
 134             raise llfuse.FUSEError(errno.ENOENT)
 135
 136     def open(self, inode, flags):
 137         if inode in self._inodes:
 138             p = self._inodes[inode]
 139         else:
 140             raise llfuse.FUSEError(errno.ENOENT)
 141
 142         self._filehandles_lock.acquire()
 143         try:
 144             fh = self._filehandles_counter
 145             self._filehandles_counter += 1
 146             self._filehandles[fh] = FileHandle(fh, p)
 147             return fh
 148         finally:
 149             self._filehandles_lock.release()
 150
 151     def read(self, fh, off, size):
 152         self._filehandles_lock.acquire()
 153         try:
 154             if fh in self._filehandles:
 155                 handle = self._filehandles[fh]
 156             else:
 157                 raise llfuse.FUSEError(errno.EBADF)
 158         finally:
 159             self._filehandles_lock.release()
 160
 161         try:
 162             return handle.entry.reader.readfrom(off, size)
 163         except:
 164             raise llfuse.FUSEError(errno.EPIPE)
 165
 166     def release(self, fh):
 167         self._filehandles_lock.acquire()
 168         try:
 169             del self._filehandles[fh]
 170         finally:
 171             self._filehandles_lock.release()
 172
 173     def opendir(self, inode):
 174         #print "opendir: inode", inode
 175
 176         if inode in self._inodes:
 177             p = self._inodes[inode]
 178         else:
 179             raise llfuse.FUSEError(errno.ENOENT)
 180
 181         self._filehandles_lock.acquire()
 182         try:
 183             fh = self._filehandles_counter
 184             self._filehandles_counter += 1
 185             self._filehandles[fh] = FileHandle(fh, list(p.entries.items()))
 186             return fh
 187         finally:
 188             self._filehandles_lock.release()
 189
 190     def readdir(self, fh, off):
 191         #print "readdir: fh", fh, "off", off
 192
 193         self._filehandles_lock.acquire()
 194         try:
 195             if fh in self._filehandles:
 196                 handle = self._filehandles[fh]
 197             else:
 198                 raise llfuse.FUSEError(errno.EBADF)
 199         finally:
 200             self._filehandles_lock.release()
 201
 202         #print "handle.entry", handle.entry
 203
 204         e = off
 205         while e < len(handle.entry):
 206             yield (handle.entry[e][0], self.getattr(handle.entry[e][1].inode), e+1)
 207             e += 1
 208
 209     def releasedir(self, fh):
 210         self._filehandles_lock.acquire()
 211         try:
 212             del self._filehandles[fh]
 213         finally:
 214             self._filehandles_lock.release()
 215
 216 if __name__ == '__main__':
 217
 218     parser = argparse.ArgumentParser(
 219         description='Mount Keep data under the local filesystem.')
 220     parser.add_argument('mountpoint', type=str,
 221                         help="""Mount point.""")
 222
 223     parser.add_argument('--collection', type=str, action='append',
 224                         help="""Collection locator""")
 225
 226     args = parser.parse_args()
 227
 228     # for testing only!
 229     manifest = open('/home/tetron/work/arvados/sdk/python/testdata/jlake_manifest').read()
 230
 231     operations = Operations(arvados.CollectionReader(manifest))
 232
 233     #operations = Operations(arvados.CollectionReader(arvados.Keep.get(args.collection)))
 234
 235     llfuse.init(operations, args.mountpoint, [ b'fsname=keepfuse' ])
 236
 237     try:
 238         llfuse.main()
 239     except:
 240         llfuse.close(unmount=True)
 241         raise
 242
 243     llfuse.close()