X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/57647945b08a74bab02fed6adf947eb9adb8321f..99240b3ebe6affa2087b7c320970bdb46c4594fd:/services/fuse/arvados_fuse/fusedir.py diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py index 0178fe5544..2d58012fa8 100644 --- a/services/fuse/arvados_fuse/fusedir.py +++ b/services/fuse/arvados_fuse/fusedir.py @@ -150,12 +150,12 @@ class Directory(FreshBase): # delete any other directory entries that were not in found in 'items' for i in oldentries: _logger.debug("Forgetting about entry '%s' on inode %i", i, self.inode) - self.inodes.invalidate_entry(self.inode, i.encode(self.inodes.encoding)) + self.inodes.invalidate_entry(self, i) self.inodes.del_entry(oldentries[i]) changed = True if changed: - self.inodes.invalidate_inode(self.inode) + self.inodes.invalidate_inode(self) self._mtime = time.time() self.fresh() @@ -182,16 +182,21 @@ class Directory(FreshBase): self._entries = {} for n in oldentries: oldentries[n].clear() - self.inodes.invalidate_entry(self.inode, n.encode(self.inodes.encoding)) self.inodes.del_entry(oldentries[n]) - self.inodes.invalidate_inode(self.inode) self.invalidate() def kernel_invalidate(self): - for n, e in self._entries.iteritems(): - self.inodes.invalidate_entry(self.inode, n.encode(self.inodes.encoding)) - e.kernel_invalidate() - self.inodes.invalidate_inode(self.inode) + # Invalidating the dentry on the parent implies invalidating all paths + # below it as well. + parent = self.inodes[self.parent_inode] + + # Find self on the parent in order to invalidate this path. + # Calling the public items() method might trigger a refresh, + # which we definitely don't want, so read the internal dict directly. + for k,v in parent._entries.items(): + if v is self: + self.inodes.invalidate_entry(parent, k) + break def mtime(self): return self._mtime @@ -266,13 +271,13 @@ class CollectionDirectoryBase(Directory): elif event == arvados.collection.DEL: ent = self._entries[name] del self._entries[name] - self.inodes.invalidate_entry(self.inode, name.encode(self.inodes.encoding)) + self.inodes.invalidate_entry(self, name) self.inodes.del_entry(ent) elif event == arvados.collection.MOD: if hasattr(item, "fuse_entry") and item.fuse_entry is not None: - self.inodes.invalidate_inode(item.fuse_entry.inode) + self.inodes.invalidate_inode(item.fuse_entry) elif name in self._entries: - self.inodes.invalidate_inode(self._entries[name].inode) + self.inodes.invalidate_inode(self._entries[name]) def populate(self, mtime): self._mtime = mtime @@ -547,7 +552,7 @@ class TmpCollectionDirectory(CollectionDirectoryBase): if self.collection_record_file: with llfuse.lock: self.collection_record_file.invalidate() - self.inodes.invalidate_inode(self.collection_record_file.inode) + self.inodes.invalidate_inode(self.collection_record_file) _logger.debug("%s invalidated collection record", self) def collection_record(self): @@ -604,12 +609,13 @@ class MagicDirectory(Directory): README_TEXT = """ This directory provides access to Arvados collections as subdirectories listed by uuid (in the form 'zzzzz-4zz18-1234567890abcde') or portable data hash (in -the form '1234567890abcdef0123456789abcdef+123'). +the form '1234567890abcdef0123456789abcdef+123'), and Arvados projects by uuid +(in the form 'zzzzz-j7d0g-1234567890abcde'). Note that this directory will appear empty until you attempt to access a -specific collection subdirectory (such as trying to 'cd' into it), at which -point the collection will actually be looked up on the server and the directory -will appear if it exists. +specific collection or project subdirectory (such as trying to 'cd' into it), +at which point the collection or project will actually be looked up on the server +and the directory will appear if it exists. """.lstrip() @@ -639,8 +645,18 @@ will appear if it exists. return False try: - e = self.inodes.add_entry(CollectionDirectory( - self.inode, self.inodes, self.api, self.num_retries, k)) + e = None + + if group_uuid_pattern.match(k): + project = self.api.groups().list( + filters=[['group_class', '=', 'project'], ["uuid", "=", k]]).execute(num_retries=self.num_retries) + if project[u'items_available'] == 0: + return False + e = self.inodes.add_entry(ProjectDirectory( + self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0])) + else: + e = self.inodes.add_entry(CollectionDirectory( + self.inode, self.inodes, self.api, self.num_retries, k)) if e.update(): if k not in self._entries: @@ -649,12 +665,13 @@ will appear if it exists. self.inodes.del_entry(e) return True else: - self.inodes.invalidate_entry(self.inode, k) + self.inodes.invalidate_entry(self, k) self.inodes.del_entry(e) return False except Exception as ex: - _logger.debug('arv-mount exception keep %s', ex) - self.inodes.del_entry(e) + _logger.exception("arv-mount lookup '%s':", k) + if e is not None: + self.inodes.del_entry(e) return False def __getitem__(self, item): @@ -822,7 +839,7 @@ class ProjectDirectory(Directory): self.inodes.add_entry(self.project_object_file) if not self._full_listing: - return + return True def samefn(a, i): if isinstance(a, CollectionDirectory) or isinstance(a, ProjectDirectory): @@ -858,6 +875,7 @@ class ProjectDirectory(Directory): self.namefn, samefn, self.createDirectory) + return True finally: self._updating_lock.release() @@ -907,7 +925,7 @@ class ProjectDirectory(Directory): with llfuse.lock_released: if not self._current_user: self._current_user = self.api.users().current().execute(num_retries=self.num_retries) - return self._current_user["uuid"] in self.project_object["writable_by"] + return self._current_user["uuid"] in self.project_object.get("writable_by", []) def persisted(self): return True @@ -963,7 +981,7 @@ class ProjectDirectory(Directory): # Acually move the entry from source directory to this directory. del src._entries[name_old] self._entries[name_new] = ent - self.inodes.invalidate_entry(src.inode, name_old.encode(self.inodes.encoding)) + self.inodes.invalidate_entry(src, name_old) @use_counter def child_event(self, ev): @@ -987,20 +1005,19 @@ class ProjectDirectory(Directory): # Was moved to somewhere else, so don't try to add entry new_name = None - if ev.get("object_kind") == "arvados#collection": - if old_attrs.get("is_trashed"): - # Was previously deleted - old_name = None - if new_attrs.get("is_trashed"): - # Has been deleted - new_name = None + if old_attrs.get("is_trashed"): + # Was previously deleted + old_name = None + if new_attrs.get("is_trashed"): + # Has been deleted + new_name = None if new_name != old_name: ent = None if old_name in self._entries: ent = self._entries[old_name] del self._entries[old_name] - self.inodes.invalidate_entry(self.inode, old_name.encode(self.inodes.encoding)) + self.inodes.invalidate_entry(self, old_name) if new_name: if ent is not None: @@ -1022,64 +1039,91 @@ class SharedDirectory(Directory): self.current_user = api.users().current().execute(num_retries=num_retries) self._poll = True self._poll_time = poll_time + self._updating_lock = threading.Lock() @use_counter def update(self): - with llfuse.lock_released: - all_projects = arvados.util.list_all( - self.api.groups().list, self.num_retries, - filters=[['group_class','=','project']]) - objects = {} - for ob in all_projects: - objects[ob['uuid']] = ob - - roots = [] - root_owners = {} - for ob in all_projects: - if ob['owner_uuid'] != self.current_user['uuid'] and ob['owner_uuid'] not in objects: - roots.append(ob) - root_owners[ob['owner_uuid']] = True - - lusers = arvados.util.list_all( - self.api.users().list, self.num_retries, - filters=[['uuid','in', list(root_owners)]]) - lgroups = arvados.util.list_all( - self.api.groups().list, self.num_retries, - filters=[['uuid','in', list(root_owners)]]) - - users = {} - groups = {} - - for l in lusers: - objects[l["uuid"]] = l - for l in lgroups: - objects[l["uuid"]] = l - - contents = {} - for r in root_owners: - if r in objects: - obr = objects[r] - if obr.get("name"): - contents[obr["name"]] = obr - #elif obr.get("username"): - # contents[obr["username"]] = obr - elif "first_name" in obr: - contents[u"{} {}".format(obr["first_name"], obr["last_name"])] = obr - - - for r in roots: - if r['owner_uuid'] not in objects: - contents[r['name']] = r - - # end with llfuse.lock_released, re-acquire lock - try: + with llfuse.lock_released: + self._updating_lock.acquire() + if not self.stale(): + return + + contents = {} + roots = [] + root_owners = set() + objects = {} + + methods = self.api._rootDesc.get('resources')["groups"]['methods'] + if 'httpMethod' in methods.get('shared', {}): + page = [] + while True: + resp = self.api.groups().shared(filters=[['group_class', '=', 'project']]+page, + order="uuid", + limit=10000, + count="none", + include="owner_uuid").execute() + if not resp["items"]: + break + page = [["uuid", ">", resp["items"][len(resp["items"])-1]["uuid"]]] + for r in resp["items"]: + objects[r["uuid"]] = r + roots.append(r["uuid"]) + for r in resp["included"]: + objects[r["uuid"]] = r + root_owners.add(r["uuid"]) + else: + all_projects = arvados.util.list_all( + self.api.groups().list, self.num_retries, + filters=[['group_class','=','project']], + select=["uuid", "owner_uuid"]) + for ob in all_projects: + objects[ob['uuid']] = ob + + current_uuid = self.current_user['uuid'] + for ob in all_projects: + if ob['owner_uuid'] != current_uuid and ob['owner_uuid'] not in objects: + roots.append(ob['uuid']) + root_owners.add(ob['owner_uuid']) + + lusers = arvados.util.list_all( + self.api.users().list, self.num_retries, + filters=[['uuid','in', list(root_owners)]]) + lgroups = arvados.util.list_all( + self.api.groups().list, self.num_retries, + filters=[['uuid','in', list(root_owners)+roots]]) + + for l in lusers: + objects[l["uuid"]] = l + for l in lgroups: + objects[l["uuid"]] = l + + for r in root_owners: + if r in objects: + obr = objects[r] + if obr.get("name"): + contents[obr["name"]] = obr + #elif obr.get("username"): + # contents[obr["username"]] = obr + elif "first_name" in obr: + contents[u"{} {}".format(obr["first_name"], obr["last_name"])] = obr + + for r in roots: + if r in objects: + obr = objects[r] + if obr['owner_uuid'] not in objects: + contents[obr["name"]] = obr + + # end with llfuse.lock_released, re-acquire lock + self.merge(contents.items(), lambda i: i[0], lambda a, i: a.uuid() == i[1]['uuid'], lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time)) except Exception: - _logger.exception() + _logger.exception("arv-mount shared dir error") + finally: + self._updating_lock.release() def want_event_subscribe(self): return True