services/fuse/arvados_fuse/fusedir.py

   1 import logging
   2 import re
   3 import time
   4 import llfuse
   5 import arvados
   6 import apiclient
   7 import functools
   8 import threading
   9 from apiclient import errors as apiclient_errors
  10 import errno
  11
  12 from fusefile import StringFile, ObjectFile, FuseArvadosFile
  13 from fresh import FreshBase, convertTime, use_counter
  14
  15 import arvados.collection
  16 from arvados.util import portable_data_hash_pattern, uuid_pattern, collection_uuid_pattern, group_uuid_pattern, user_uuid_pattern, link_uuid_pattern
  17
  18 _logger = logging.getLogger('arvados.arvados_fuse')
  19
  20
  21 # Match any character which FUSE or Linux cannot accommodate as part
  22 # of a filename. (If present in a collection filename, they will
  23 # appear as underscores in the fuse mount.)
  24 _disallowed_filename_characters = re.compile('[\x00/]')
  25
  26 def sanitize_filename(dirty):
  27     """Replace disallowed filename characters with harmless "_"."""
  28     if dirty is None:
  29         return None
  30     elif dirty == '':
  31         return '_'
  32     elif dirty == '.':
  33         return '_'
  34     elif dirty == '..':
  35         return '__'
  36     else:
  37         return _disallowed_filename_characters.sub('_', dirty)
  38
  39
  40 class Directory(FreshBase):
  41     """Generic directory object, backed by a dict.
  42
  43     Consists of a set of entries with the key representing the filename
  44     and the value referencing a File or Directory object.
  45     """
  46
  47     def __init__(self, parent_inode, inodes):
  48         super(Directory, self).__init__()
  49
  50         """parent_inode is the integer inode number"""
  51         self.inode = None
  52         if not isinstance(parent_inode, int):
  53             raise Exception("parent_inode should be an int")
  54         self.parent_inode = parent_inode
  55         self.inodes = inodes
  56         self._entries = {}
  57         self._mtime = time.time()
  58
  59     #  Overriden by subclasses to implement logic to update the entries dict
  60     #  when the directory is stale
  61     @use_counter
  62     def update(self):
  63         pass
  64
  65     # Only used when computing the size of the disk footprint of the directory
  66     # (stub)
  67     def size(self):
  68         return 0
  69
  70     def persisted(self):
  71         return False
  72
  73     def checkupdate(self):
  74         if self.stale():
  75             try:
  76                 self.update()
  77             except apiclient.errors.HttpError as e:
  78                 _logger.warn(e)
  79
  80     @use_counter
  81     def __getitem__(self, item):
  82         self.checkupdate()
  83         return self._entries[item]
  84
  85     @use_counter
  86     def items(self):
  87         self.checkupdate()
  88         return list(self._entries.items())
  89
  90     @use_counter
  91     def __contains__(self, k):
  92         self.checkupdate()
  93         return k in self._entries
  94
  95     @use_counter
  96     def __len__(self):
  97         self.checkupdate()
  98         return len(self._entries)
  99
 100     def fresh(self):
 101         self.inodes.touch(self)
 102         super(Directory, self).fresh()
 103
 104     def merge(self, items, fn, same, new_entry):
 105         """Helper method for updating the contents of the directory.
 106
 107         Takes a list describing the new contents of the directory, reuse
 108         entries that are the same in both the old and new lists, create new
 109         entries, and delete old entries missing from the new list.
 110
 111         :items: iterable with new directory contents
 112
 113         :fn: function to take an entry in 'items' and return the desired file or
 114         directory name, or None if this entry should be skipped
 115
 116         :same: function to compare an existing entry (a File or Directory
 117         object) with an entry in the items list to determine whether to keep
 118         the existing entry.
 119
 120         :new_entry: function to create a new directory entry (File or Directory
 121         object) from an entry in the items list.
 122
 123         """
 124
 125         oldentries = self._entries
 126         self._entries = {}
 127         changed = False
 128         for i in items:
 129             name = sanitize_filename(fn(i))
 130             if name:
 131                 if name in oldentries and same(oldentries[name], i):
 132                     # move existing directory entry over
 133                     self._entries[name] = oldentries[name]
 134                     del oldentries[name]
 135                 else:
 136                     _logger.debug("Adding entry '%s' to inode %i", name, self.inode)
 137                     # create new directory entry
 138                     ent = new_entry(i)
 139                     if ent is not None:
 140                         self._entries[name] = self.inodes.add_entry(ent)
 141                         changed = True
 142
 143         # delete any other directory entries that were not in found in 'items'
 144         for i in oldentries:
 145             _logger.debug("Forgetting about entry '%s' on inode %i", str(i), self.inode)
 146             llfuse.invalidate_entry(self.inode, str(i))
 147             self.inodes.del_entry(oldentries[i])
 148             changed = True
 149
 150         if changed:
 151             llfuse.invalidate_inode(self.inode)
 152             self._mtime = time.time()
 153
 154         self.fresh()
 155
 156     def clear(self, force=False):
 157         """Delete all entries"""
 158
 159         if not self.in_use() or force:
 160             oldentries = self._entries
 161             self._entries = {}
 162             for n in oldentries:
 163                 if not oldentries[n].clear(force):
 164                     self._entries = oldentries
 165                     return False
 166             for n in oldentries:
 167                 llfuse.invalidate_entry(self.inode, str(n))
 168                 self.inodes.del_entry(oldentries[n])
 169             llfuse.invalidate_inode(self.inode)
 170             self.invalidate()
 171             return True
 172         else:
 173             return False
 174
 175     def mtime(self):
 176         return self._mtime
 177
 178     def writable(self):
 179         return False
 180
 181     def flush(self):
 182         pass
 183
 184     def create(self, name):
 185         raise NotImplementedError()
 186
 187     def mkdir(self, name):
 188         raise NotImplementedError()
 189
 190     def unlink(self, name):
 191         raise NotImplementedError()
 192
 193     def rmdir(self, name):
 194         raise NotImplementedError()
 195
 196     def rename(self, name_old, name_new, src):
 197         raise NotImplementedError()
 198
 199 class CollectionDirectoryBase(Directory):
 200     def __init__(self, parent_inode, inodes, collection):
 201         super(CollectionDirectoryBase, self).__init__(parent_inode, inodes)
 202         self.collection = collection
 203
 204     def new_entry(self, name, item, mtime):
 205         name = sanitize_filename(name)
 206         if hasattr(item, "fuse_entry") and item.fuse_entry is not None:
 207             if item.fuse_entry.dead is not True:
 208                 raise Exception("Can only reparent dead inode entry")
 209             if item.fuse_entry.inode is None:
 210                 raise Exception("Reparented entry must still have valid inode")
 211             item.fuse_entry.dead = False
 212             self._entries[name] = item.fuse_entry
 213         elif isinstance(item, arvados.collection.RichCollectionBase):
 214             self._entries[name] = self.inodes.add_entry(CollectionDirectoryBase(self.inode, self.inodes, item))
 215             self._entries[name].populate(mtime)
 216         else:
 217             self._entries[name] = self.inodes.add_entry(FuseArvadosFile(self.inode, item, mtime))
 218         item.fuse_entry = self._entries[name]
 219
 220     def on_event(self, event, collection, name, item):
 221         if collection == self.collection:
 222             _logger.debug("%s %s %s %s", event, collection, name, item)
 223             with llfuse.lock:
 224                 if event == arvados.collection.ADD:
 225                     self.new_entry(name, item, self.mtime())
 226                 elif event == arvados.collection.DEL:
 227                     ent = self._entries[name]
 228                     del self._entries[name]
 229                     llfuse.invalidate_entry(self.inode, name)
 230                     self.inodes.del_entry(ent)
 231                 elif event == arvados.collection.MOD:
 232                     if hasattr(item, "fuse_entry") and item.fuse_entry is not None:
 233                         llfuse.invalidate_inode(item.fuse_entry.inode)
 234                     elif name in self._entries:
 235                         llfuse.invalidate_inode(self._entries[name].inode)
 236
 237     def populate(self, mtime):
 238         self._mtime = mtime
 239         self.collection.subscribe(self.on_event)
 240         for entry, item in self.collection.items():
 241             self.new_entry(entry, item, self.mtime())
 242
 243     def writable(self):
 244         return self.collection.writable()
 245
 246     def flush(self):
 247         with llfuse.lock_released:
 248             self.collection.root_collection().save()
 249
 250     def create(self, name):
 251         with llfuse.lock_released:
 252             self.collection.open(name, "w").close()
 253
 254     def mkdir(self, name):
 255         with llfuse.lock_released:
 256             self.collection.mkdirs(name)
 257
 258     def unlink(self, name):
 259         with llfuse.lock_released:
 260             self.collection.remove(name)
 261         self.flush()
 262
 263     def rmdir(self, name):
 264         with llfuse.lock_released:
 265             self.collection.remove(name)
 266         self.flush()
 267
 268     def rename(self, name_old, name_new, src):
 269         if not isinstance(src, CollectionDirectoryBase):
 270             raise llfuse.FUSEError(errno.EPERM)
 271
 272         if name_new in self:
 273             ent = src[name_old]
 274             tgt = self[name_new]
 275             if isinstance(ent, FuseArvadosFile) and isinstance(tgt, FuseArvadosFile):
 276                 pass
 277             elif isinstance(ent, CollectionDirectoryBase) and isinstance(tgt, CollectionDirectoryBase):
 278                 if len(tgt) > 0:
 279                     raise llfuse.FUSEError(errno.ENOTEMPTY)
 280             elif isinstance(ent, CollectionDirectoryBase) and isinstance(tgt, FuseArvadosFile):
 281                 raise llfuse.FUSEError(errno.ENOTDIR)
 282             elif isinstance(ent, FuseArvadosFile) and isinstance(tgt, CollectionDirectoryBase):
 283                 raise llfuse.FUSEError(errno.EISDIR)
 284
 285         with llfuse.lock_released:
 286             self.collection.rename(name_old, name_new, source_collection=src.collection, overwrite=True)
 287         self.flush()
 288         src.flush()
 289
 290
 291 class CollectionDirectory(CollectionDirectoryBase):
 292     """Represents the root of a directory tree holding a collection."""
 293
 294     def __init__(self, parent_inode, inodes, api, num_retries, collection_record=None, explicit_collection=None):
 295         super(CollectionDirectory, self).__init__(parent_inode, inodes, None)
 296         self.api = api
 297         self.num_retries = num_retries
 298         self.collection_record_file = None
 299         self.collection_record = None
 300         if isinstance(collection_record, dict):
 301             self.collection_locator = collection_record['uuid']
 302             self._mtime = convertTime(collection_record.get('modified_at'))
 303         else:
 304             self.collection_locator = collection_record
 305             self._mtime = 0
 306         self._manifest_size = 0
 307         if self.collection_locator:
 308             self._writable = (uuid_pattern.match(self.collection_locator) is not None)
 309         self._updating_lock = threading.Lock()
 310
 311     def same(self, i):
 312         return i['uuid'] == self.collection_locator or i['portable_data_hash'] == self.collection_locator
 313
 314     def writable(self):
 315         return self.collection.writable() if self.collection is not None else self._writable
 316
 317     # Used by arv-web.py to switch the contents of the CollectionDirectory
 318     def change_collection(self, new_locator):
 319         """Switch the contents of the CollectionDirectory.
 320
 321         Must be called with llfuse.lock held.
 322         """
 323
 324         self.collection_locator = new_locator
 325         self.collection_record = None
 326         self.update()
 327
 328     def new_collection(self, new_collection_record, coll_reader):
 329         if self.inode:
 330             self.clear(force=True)
 331
 332         self.collection_record = new_collection_record
 333
 334         if self.collection_record:
 335             self._mtime = convertTime(self.collection_record.get('modified_at'))
 336             self.collection_locator = self.collection_record["uuid"]
 337             if self.collection_record_file is not None:
 338                 self.collection_record_file.update(self.collection_record)
 339
 340         self.collection = coll_reader
 341         self.populate(self.mtime())
 342
 343     def uuid(self):
 344         return self.collection_locator
 345
 346     def update(self):
 347         try:
 348             if self.collection_record is not None and portable_data_hash_pattern.match(self.collection_locator):
 349                 return True
 350
 351             if self.collection_locator is None:
 352                 self.fresh()
 353                 return True
 354
 355             try:
 356                 with llfuse.lock_released:
 357                     self._updating_lock.acquire()
 358                     if not self.stale():
 359                         return
 360
 361                     _logger.debug("Updating %s", self.collection_locator)
 362                     if self.collection:
 363                         self.collection.update()
 364                     else:
 365                         if uuid_pattern.match(self.collection_locator):
 366                             coll_reader = arvados.collection.Collection(
 367                                 self.collection_locator, self.api, self.api.keep,
 368                                 num_retries=self.num_retries)
 369                         else:
 370                             coll_reader = arvados.collection.CollectionReader(
 371                                 self.collection_locator, self.api, self.api.keep,
 372                                 num_retries=self.num_retries)
 373                         new_collection_record = coll_reader.api_response() or {}
 374                         # If the Collection only exists in Keep, there will be no API
 375                         # response.  Fill in the fields we need.
 376                         if 'uuid' not in new_collection_record:
 377                             new_collection_record['uuid'] = self.collection_locator
 378                         if "portable_data_hash" not in new_collection_record:
 379                             new_collection_record["portable_data_hash"] = new_collection_record["uuid"]
 380                         if 'manifest_text' not in new_collection_record:
 381                             new_collection_record['manifest_text'] = coll_reader.manifest_text()
 382
 383                         if self.collection_record is None or self.collection_record["portable_data_hash"] != new_collection_record.get("portable_data_hash"):
 384                             self.new_collection(new_collection_record, coll_reader)
 385
 386                         self._manifest_size = len(coll_reader.manifest_text())
 387                         _logger.debug("%s manifest_size %i", self, self._manifest_size)
 388                 # end with llfuse.lock_released, re-acquire lock
 389
 390                 self.fresh()
 391                 return True
 392             finally:
 393                 self._updating_lock.release()
 394         except arvados.errors.NotFoundError:
 395             _logger.exception("arv-mount %s: error", self.collection_locator)
 396         except arvados.errors.ArgumentError as detail:
 397             _logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
 398             if self.collection_record is not None and "manifest_text" in self.collection_record:
 399                 _logger.warning("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
 400         except Exception:
 401             _logger.exception("arv-mount %s: error", self.collection_locator)
 402             if self.collection_record is not None and "manifest_text" in self.collection_record:
 403                 _logger.error("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
 404         return False
 405
 406     def __getitem__(self, item):
 407         self.checkupdate()
 408         if item == '.arvados#collection':
 409             if self.collection_record_file is None:
 410                 self.collection_record_file = ObjectFile(self.inode, self.collection_record)
 411                 self.inodes.add_entry(self.collection_record_file)
 412             return self.collection_record_file
 413         else:
 414             return super(CollectionDirectory, self).__getitem__(item)
 415
 416     def __contains__(self, k):
 417         if k == '.arvados#collection':
 418             return True
 419         else:
 420             return super(CollectionDirectory, self).__contains__(k)
 421
 422     def invalidate(self):
 423         self.collection_record = None
 424         self.collection_record_file = None
 425         super(CollectionDirectory, self).invalidate()
 426
 427     def persisted(self):
 428         return (self.collection_locator is not None)
 429
 430     def objsize(self):
 431         # This is an empirically-derived heuristic to estimate the memory used
 432         # to store this collection's metadata.  Calculating the memory
 433         # footprint directly would be more accurate, but also more complicated.
 434         return self._manifest_size * 128
 435
 436 class MagicDirectory(Directory):
 437     """A special directory that logically contains the set of all extant keep locators.
 438
 439     When a file is referenced by lookup(), it is tested to see if it is a valid
 440     keep locator to a manifest, and if so, loads the manifest contents as a
 441     subdirectory of this directory with the locator as the directory name.
 442     Since querying a list of all extant keep locators is impractical, only
 443     collections that have already been accessed are visible to readdir().
 444
 445     """
 446
 447     README_TEXT = """
 448 This directory provides access to Arvados collections as subdirectories listed
 449 by uuid (in the form 'zzzzz-4zz18-1234567890abcde') or portable data hash (in
 450 the form '1234567890abcdefghijklmnopqrstuv+123').
 451
 452 Note that this directory will appear empty until you attempt to access a
 453 specific collection subdirectory (such as trying to 'cd' into it), at which
 454 point the collection will actually be looked up on the server and the directory
 455 will appear if it exists.
 456 """.lstrip()
 457
 458     def __init__(self, parent_inode, inodes, api, num_retries):
 459         super(MagicDirectory, self).__init__(parent_inode, inodes)
 460         self.api = api
 461         self.num_retries = num_retries
 462
 463     def __setattr__(self, name, value):
 464         super(MagicDirectory, self).__setattr__(name, value)
 465         # When we're assigned an inode, add a README.
 466         if ((name == 'inode') and (self.inode is not None) and
 467               (not self._entries)):
 468             self._entries['README'] = self.inodes.add_entry(
 469                 StringFile(self.inode, self.README_TEXT, time.time()))
 470             # If we're the root directory, add an identical by_id subdirectory.
 471             if self.inode == llfuse.ROOT_INODE:
 472                 self._entries['by_id'] = self.inodes.add_entry(MagicDirectory(
 473                         self.inode, self.inodes, self.api, self.num_retries))
 474
 475     def __contains__(self, k):
 476         if k in self._entries:
 477             return True
 478
 479         if not portable_data_hash_pattern.match(k) and not uuid_pattern.match(k):
 480             return False
 481
 482         try:
 483             e = self.inodes.add_entry(CollectionDirectory(
 484                     self.inode, self.inodes, self.api, self.num_retries, k))
 485
 486             if e.update():
 487                 self._entries[k] = e
 488                 return True
 489             else:
 490                 return False
 491         except Exception as e:
 492             _logger.debug('arv-mount exception keep %s', e)
 493             return False
 494
 495     def __getitem__(self, item):
 496         if item in self:
 497             return self._entries[item]
 498         else:
 499             raise KeyError("No collection with id " + item)
 500
 501     def clear(self, force=False):
 502         pass
 503
 504
 505 class RecursiveInvalidateDirectory(Directory):
 506     def invalidate(self):
 507         try:
 508             super(RecursiveInvalidateDirectory, self).invalidate()
 509             for a in self._entries:
 510                 self._entries[a].invalidate()
 511         except Exception:
 512             _logger.exception()
 513
 514
 515 class TagsDirectory(RecursiveInvalidateDirectory):
 516     """A special directory that contains as subdirectories all tags visible to the user."""
 517
 518     def __init__(self, parent_inode, inodes, api, num_retries, poll_time=60):
 519         super(TagsDirectory, self).__init__(parent_inode, inodes)
 520         self.api = api
 521         self.num_retries = num_retries
 522         self._poll = True
 523         self._poll_time = poll_time
 524
 525     def update(self):
 526         with llfuse.lock_released:
 527             tags = self.api.links().list(
 528                 filters=[['link_class', '=', 'tag']],
 529                 select=['name'], distinct=True
 530                 ).execute(num_retries=self.num_retries)
 531         if "items" in tags:
 532             self.merge(tags['items'],
 533                        lambda i: i['name'],
 534                        lambda a, i: a.tag == i['name'],
 535                        lambda i: TagDirectory(self.inode, self.inodes, self.api, self.num_retries, i['name'], poll=self._poll, poll_time=self._poll_time))
 536
 537
 538 class TagDirectory(Directory):
 539     """A special directory that contains as subdirectories all collections visible
 540     to the user that are tagged with a particular tag.
 541     """
 542
 543     def __init__(self, parent_inode, inodes, api, num_retries, tag,
 544                  poll=False, poll_time=60):
 545         super(TagDirectory, self).__init__(parent_inode, inodes)
 546         self.api = api
 547         self.num_retries = num_retries
 548         self.tag = tag
 549         self._poll = poll
 550         self._poll_time = poll_time
 551
 552     def update(self):
 553         with llfuse.lock_released:
 554             taggedcollections = self.api.links().list(
 555                 filters=[['link_class', '=', 'tag'],
 556                          ['name', '=', self.tag],
 557                          ['head_uuid', 'is_a', 'arvados#collection']],
 558                 select=['head_uuid']
 559                 ).execute(num_retries=self.num_retries)
 560         self.merge(taggedcollections['items'],
 561                    lambda i: i['head_uuid'],
 562                    lambda a, i: a.collection_locator == i['head_uuid'],
 563                    lambda i: CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid']))
 564
 565
 566 class ProjectDirectory(Directory):
 567     """A special directory that contains the contents of a project."""
 568
 569     def __init__(self, parent_inode, inodes, api, num_retries, project_object,
 570                  poll=False, poll_time=60):
 571         super(ProjectDirectory, self).__init__(parent_inode, inodes)
 572         self.api = api
 573         self.num_retries = num_retries
 574         self.project_object = project_object
 575         self.project_object_file = None
 576         self.project_uuid = project_object['uuid']
 577         self._poll = poll
 578         self._poll_time = poll_time
 579         self._updating_lock = threading.Lock()
 580         self._current_user = None
 581
 582     def createDirectory(self, i):
 583         if collection_uuid_pattern.match(i['uuid']):
 584             return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i)
 585         elif group_uuid_pattern.match(i['uuid']):
 586             return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time)
 587         elif link_uuid_pattern.match(i['uuid']):
 588             if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']):
 589                 return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid'])
 590             else:
 591                 return None
 592         elif uuid_pattern.match(i['uuid']):
 593             return ObjectFile(self.parent_inode, i)
 594         else:
 595             return None
 596
 597     def uuid(self):
 598         return self.project_uuid
 599
 600     def update(self):
 601         if self.project_object_file == None:
 602             self.project_object_file = ObjectFile(self.inode, self.project_object)
 603             self.inodes.add_entry(self.project_object_file)
 604
 605         def namefn(i):
 606             if 'name' in i:
 607                 if i['name'] is None or len(i['name']) == 0:
 608                     return None
 609                 elif collection_uuid_pattern.match(i['uuid']) or group_uuid_pattern.match(i['uuid']):
 610                     # collection or subproject
 611                     return i['name']
 612                 elif link_uuid_pattern.match(i['uuid']) and i['head_kind'] == 'arvados#collection':
 613                     # name link
 614                     return i['name']
 615                 elif 'kind' in i and i['kind'].startswith('arvados#'):
 616                     # something else
 617                     return "{}.{}".format(i['name'], i['kind'][8:])
 618             else:
 619                 return None
 620
 621         def samefn(a, i):
 622             if isinstance(a, CollectionDirectory) or isinstance(a, ProjectDirectory):
 623                 return a.uuid() == i['uuid']
 624             elif isinstance(a, ObjectFile):
 625                 return a.uuid() == i['uuid'] and not a.stale()
 626             return False
 627
 628         try:
 629             with llfuse.lock_released:
 630                 self._updating_lock.acquire()
 631                 if not self.stale():
 632                     return
 633
 634                 if group_uuid_pattern.match(self.project_uuid):
 635                     self.project_object = self.api.groups().get(
 636                         uuid=self.project_uuid).execute(num_retries=self.num_retries)
 637                 elif user_uuid_pattern.match(self.project_uuid):
 638                     self.project_object = self.api.users().get(
 639                         uuid=self.project_uuid).execute(num_retries=self.num_retries)
 640
 641                 contents = arvados.util.list_all(self.api.groups().contents,
 642                                                  self.num_retries, uuid=self.project_uuid)
 643
 644             # end with llfuse.lock_released, re-acquire lock
 645
 646             self.merge(contents,
 647                        namefn,
 648                        samefn,
 649                        self.createDirectory)
 650         finally:
 651             self._updating_lock.release()
 652
 653     def __getitem__(self, item):
 654         self.checkupdate()
 655         if item == '.arvados#project':
 656             return self.project_object_file
 657         else:
 658             return super(ProjectDirectory, self).__getitem__(item)
 659
 660     def __contains__(self, k):
 661         if k == '.arvados#project':
 662             return True
 663         else:
 664             return super(ProjectDirectory, self).__contains__(k)
 665
 666     def writable(self):
 667         with llfuse.lock_released:
 668             if not self._current_user:
 669                 self._current_user = self.api.users().current().execute(num_retries=self.num_retries)
 670             return self._current_user["uuid"] in self.project_object["writable_by"]
 671
 672     def persisted(self):
 673         return True
 674
 675     def mkdir(self, name):
 676         try:
 677             with llfuse.lock_released:
 678                 self.api.collections().create(body={"owner_uuid": self.project_uuid,
 679                                                     "name": name,
 680                                                     "manifest_text": ""}).execute(num_retries=self.num_retries)
 681             self.invalidate()
 682         except apiclient_errors.Error as error:
 683             _logger.error(error)
 684             raise llfuse.FUSEError(errno.EEXIST)
 685
 686     def rmdir(self, name):
 687         if name not in self:
 688             raise llfuse.FUSEError(errno.ENOENT)
 689         if not isinstance(self[name], CollectionDirectory):
 690             raise llfuse.FUSEError(errno.EPERM)
 691         if len(self[name]) > 0:
 692             raise llfuse.FUSEError(errno.ENOTEMPTY)
 693         with llfuse.lock_released:
 694             self.api.collections().delete(uuid=self[name].uuid()).execute(num_retries=self.num_retries)
 695         self.invalidate()
 696
 697     def rename(self, name_old, name_new, src):
 698         if not isinstance(src, ProjectDirectory):
 699             raise llfuse.FUSEError(errno.EPERM)
 700
 701         ent = src[name_old]
 702
 703         if not isinstance(ent, CollectionDirectory):
 704             raise llfuse.FUSEError(errno.EPERM)
 705
 706         if name_new in self:
 707             # POSIX semantics for replacing one directory with another is
 708             # tricky (the target directory must be empty, the operation must be
 709             # atomic which isn't possible with the Arvados API as of this
 710             # writing) so don't support that.
 711             raise llfuse.FUSEError(errno.EPERM)
 712
 713         self.api.collections().update(uuid=ent.uuid(),
 714                                       body={"owner_uuid": self.uuid(),
 715                                             "name": name_new}).execute(num_retries=self.num_retries)
 716
 717         # Acually move the entry from source directory to this directory.
 718         del src._entries[name_old]
 719         self._entries[name_new] = ent
 720         llfuse.invalidate_entry(src.inode, name_old)
 721
 722 class SharedDirectory(Directory):
 723     """A special directory that represents users or groups who have shared projects with me."""
 724
 725     def __init__(self, parent_inode, inodes, api, num_retries, exclude,
 726                  poll=False, poll_time=60):
 727         super(SharedDirectory, self).__init__(parent_inode, inodes)
 728         self.api = api
 729         self.num_retries = num_retries
 730         self.current_user = api.users().current().execute(num_retries=num_retries)
 731         self._poll = True
 732         self._poll_time = poll_time
 733
 734     def update(self):
 735         with llfuse.lock_released:
 736             all_projects = arvados.util.list_all(
 737                 self.api.groups().list, self.num_retries,
 738                 filters=[['group_class','=','project']])
 739             objects = {}
 740             for ob in all_projects:
 741                 objects[ob['uuid']] = ob
 742
 743             roots = []
 744             root_owners = {}
 745             for ob in all_projects:
 746                 if ob['owner_uuid'] != self.current_user['uuid'] and ob['owner_uuid'] not in objects:
 747                     roots.append(ob)
 748                     root_owners[ob['owner_uuid']] = True
 749
 750             lusers = arvados.util.list_all(
 751                 self.api.users().list, self.num_retries,
 752                 filters=[['uuid','in', list(root_owners)]])
 753             lgroups = arvados.util.list_all(
 754                 self.api.groups().list, self.num_retries,
 755                 filters=[['uuid','in', list(root_owners)]])
 756
 757             users = {}
 758             groups = {}
 759
 760             for l in lusers:
 761                 objects[l["uuid"]] = l
 762             for l in lgroups:
 763                 objects[l["uuid"]] = l
 764
 765             contents = {}
 766             for r in root_owners:
 767                 if r in objects:
 768                     obr = objects[r]
 769                     if obr.get("name"):
 770                         contents[obr["name"]] = obr
 771                     #elif obr.get("username"):
 772                     #    contents[obr["username"]] = obr
 773                     elif "first_name" in obr:
 774                         contents[u"{} {}".format(obr["first_name"], obr["last_name"])] = obr
 775
 776
 777             for r in roots:
 778                 if r['owner_uuid'] not in objects:
 779                     contents[r['name']] = r
 780
 781         # end with llfuse.lock_released, re-acquire lock
 782
 783         try:
 784             self.merge(contents.items(),
 785                        lambda i: i[0],
 786                        lambda a, i: a.uuid() == i[1]['uuid'],
 787                        lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time))
 788         except Exception:
 789             _logger.exception()