X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1b3578c44b8d005f70863cf332b487c915af9f28..8b873a9b3b8865a4d451263e48b49122b9c32759:/services/fuse/arvados_fuse/fusedir.py diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py index 7bd00d5862..3287657441 100644 --- a/services/fuse/arvados_fuse/fusedir.py +++ b/services/fuse/arvados_fuse/fusedir.py @@ -2,6 +2,11 @@ # # SPDX-License-Identifier: AGPL-3.0 +from __future__ import absolute_import +from __future__ import division +from future.utils import viewitems +from future.utils import itervalues +from builtins import dict import logging import re import time @@ -14,8 +19,8 @@ from apiclient import errors as apiclient_errors import errno import time -from fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile -from fresh import FreshBase, convertTime, use_counter, check_update +from .fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile +from .fresh import FreshBase, convertTime, use_counter, check_update import arvados.collection from arvados.util import portable_data_hash_pattern, uuid_pattern, collection_uuid_pattern, group_uuid_pattern, user_uuid_pattern, link_uuid_pattern @@ -163,7 +168,7 @@ class Directory(FreshBase): def in_use(self): if super(Directory, self).in_use(): return True - for v in self._entries.itervalues(): + for v in itervalues(self._entries): if v.in_use(): return True return False @@ -171,7 +176,7 @@ class Directory(FreshBase): def has_ref(self, only_children): if super(Directory, self).has_ref(only_children): return True - for v in self._entries.itervalues(): + for v in itervalues(self._entries): if v.has_ref(False): return True return False @@ -193,7 +198,7 @@ class Directory(FreshBase): # Find self on the parent in order to invalidate this path. # Calling the public items() method might trigger a refresh, # which we definitely don't want, so read the internal dict directly. - for k,v in parent._entries.items(): + for k,v in viewitems(parent._entries): if v is self: self.inodes.invalidate_entry(parent, k) break @@ -282,7 +287,7 @@ class CollectionDirectoryBase(Directory): def populate(self, mtime): self._mtime = mtime self.collection.subscribe(self.on_event) - for entry, item in self.collection.items(): + for entry, item in viewitems(self.collection): self.new_entry(entry, item, self.mtime()) def writable(self): @@ -359,7 +364,7 @@ class CollectionDirectory(CollectionDirectoryBase): self.collection_record = None self._poll = True try: - self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2)/2) + self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2) // 2) except: _logger.debug("Error getting blobSignatureTtl from discovery document: %s", sys.exc_info()[0]) self._poll_time = 60*60 @@ -609,12 +614,13 @@ class MagicDirectory(Directory): README_TEXT = """ This directory provides access to Arvados collections as subdirectories listed by uuid (in the form 'zzzzz-4zz18-1234567890abcde') or portable data hash (in -the form '1234567890abcdef0123456789abcdef+123'). +the form '1234567890abcdef0123456789abcdef+123'), and Arvados projects by uuid +(in the form 'zzzzz-j7d0g-1234567890abcde'). Note that this directory will appear empty until you attempt to access a -specific collection subdirectory (such as trying to 'cd' into it), at which -point the collection will actually be looked up on the server and the directory -will appear if it exists. +specific collection or project subdirectory (such as trying to 'cd' into it), +at which point the collection or project will actually be looked up on the server +and the directory will appear if it exists. """.lstrip() @@ -645,8 +651,17 @@ will appear if it exists. try: e = None - e = self.inodes.add_entry(CollectionDirectory( - self.inode, self.inodes, self.api, self.num_retries, k)) + + if group_uuid_pattern.match(k): + project = self.api.groups().list( + filters=[['group_class', '=', 'project'], ["uuid", "=", k]]).execute(num_retries=self.num_retries) + if project[u'items_available'] == 0: + return False + e = self.inodes.add_entry(ProjectDirectory( + self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0])) + else: + e = self.inodes.add_entry(CollectionDirectory( + self.inode, self.inodes, self.api, self.num_retries, k)) if e.update(): if k not in self._entries: @@ -829,7 +844,7 @@ class ProjectDirectory(Directory): self.inodes.add_entry(self.project_object_file) if not self._full_listing: - return + return True def samefn(a, i): if isinstance(a, CollectionDirectory) or isinstance(a, ProjectDirectory): @@ -865,6 +880,7 @@ class ProjectDirectory(Directory): self.namefn, samefn, self.createDirectory) + return True finally: self._updating_lock.release() @@ -914,7 +930,7 @@ class ProjectDirectory(Directory): with llfuse.lock_released: if not self._current_user: self._current_user = self.api.users().current().execute(num_retries=self.num_retries) - return self._current_user["uuid"] in self.project_object["writable_by"] + return self._current_user["uuid"] in self.project_object.get("writable_by", []) def persisted(self): return True @@ -994,13 +1010,12 @@ class ProjectDirectory(Directory): # Was moved to somewhere else, so don't try to add entry new_name = None - if ev.get("object_kind") == "arvados#collection": - if old_attrs.get("is_trashed"): - # Was previously deleted - old_name = None - if new_attrs.get("is_trashed"): - # Has been deleted - new_name = None + if old_attrs.get("is_trashed"): + # Was previously deleted + old_name = None + if new_attrs.get("is_trashed"): + # Has been deleted + new_name = None if new_name != old_name: ent = None @@ -1029,64 +1044,91 @@ class SharedDirectory(Directory): self.current_user = api.users().current().execute(num_retries=num_retries) self._poll = True self._poll_time = poll_time + self._updating_lock = threading.Lock() @use_counter def update(self): - with llfuse.lock_released: - all_projects = arvados.util.list_all( - self.api.groups().list, self.num_retries, - filters=[['group_class','=','project']]) - objects = {} - for ob in all_projects: - objects[ob['uuid']] = ob - - roots = [] - root_owners = {} - for ob in all_projects: - if ob['owner_uuid'] != self.current_user['uuid'] and ob['owner_uuid'] not in objects: - roots.append(ob) - root_owners[ob['owner_uuid']] = True - - lusers = arvados.util.list_all( - self.api.users().list, self.num_retries, - filters=[['uuid','in', list(root_owners)]]) - lgroups = arvados.util.list_all( - self.api.groups().list, self.num_retries, - filters=[['uuid','in', list(root_owners)]]) - - users = {} - groups = {} - - for l in lusers: - objects[l["uuid"]] = l - for l in lgroups: - objects[l["uuid"]] = l - - contents = {} - for r in root_owners: - if r in objects: - obr = objects[r] - if obr.get("name"): - contents[obr["name"]] = obr - #elif obr.get("username"): - # contents[obr["username"]] = obr - elif "first_name" in obr: - contents[u"{} {}".format(obr["first_name"], obr["last_name"])] = obr - - - for r in roots: - if r['owner_uuid'] not in objects: - contents[r['name']] = r - - # end with llfuse.lock_released, re-acquire lock - try: - self.merge(contents.items(), + with llfuse.lock_released: + self._updating_lock.acquire() + if not self.stale(): + return + + contents = {} + roots = [] + root_owners = set() + objects = {} + + methods = self.api._rootDesc.get('resources')["groups"]['methods'] + if 'httpMethod' in methods.get('shared', {}): + page = [] + while True: + resp = self.api.groups().shared(filters=[['group_class', '=', 'project']]+page, + order="uuid", + limit=10000, + count="none", + include="owner_uuid").execute() + if not resp["items"]: + break + page = [["uuid", ">", resp["items"][len(resp["items"])-1]["uuid"]]] + for r in resp["items"]: + objects[r["uuid"]] = r + roots.append(r["uuid"]) + for r in resp["included"]: + objects[r["uuid"]] = r + root_owners.add(r["uuid"]) + else: + all_projects = arvados.util.list_all( + self.api.groups().list, self.num_retries, + filters=[['group_class','=','project']], + select=["uuid", "owner_uuid"]) + for ob in all_projects: + objects[ob['uuid']] = ob + + current_uuid = self.current_user['uuid'] + for ob in all_projects: + if ob['owner_uuid'] != current_uuid and ob['owner_uuid'] not in objects: + roots.append(ob['uuid']) + root_owners.add(ob['owner_uuid']) + + lusers = arvados.util.list_all( + self.api.users().list, self.num_retries, + filters=[['uuid','in', list(root_owners)]]) + lgroups = arvados.util.list_all( + self.api.groups().list, self.num_retries, + filters=[['uuid','in', list(root_owners)+roots]]) + + for l in lusers: + objects[l["uuid"]] = l + for l in lgroups: + objects[l["uuid"]] = l + + for r in root_owners: + if r in objects: + obr = objects[r] + if obr.get("name"): + contents[obr["name"]] = obr + #elif obr.get("username"): + # contents[obr["username"]] = obr + elif "first_name" in obr: + contents[u"{} {}".format(obr["first_name"], obr["last_name"])] = obr + + for r in roots: + if r in objects: + obr = objects[r] + if obr['owner_uuid'] not in objects: + contents[obr["name"]] = obr + + # end with llfuse.lock_released, re-acquire lock + + self.merge(viewitems(contents), lambda i: i[0], lambda a, i: a.uuid() == i[1]['uuid'], lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time)) except Exception: - _logger.exception() + _logger.exception("arv-mount shared dir error") + finally: + self._updating_lock.release() def want_event_subscribe(self): return True