- def __init__(self, parent_inode, inodes, api, num_retries, collection):
- super(CollectionDirectory, self).__init__(parent_inode)
- self.inodes = inodes
- self.api = api
- self.num_retries = num_retries
- self.collection_object_file = None
- self.collection_object = None
- if isinstance(collection, dict):
- self.collection_locator = collection['uuid']
- else:
- self.collection_locator = collection
-
- def same(self, i):
- return i['uuid'] == self.collection_locator or i['portable_data_hash'] == self.collection_locator
-
- def new_collection(self, new_collection_object, coll_reader):
- self.collection_object = new_collection_object
-
- if self.collection_object_file is not None:
- self.collection_object_file.update(self.collection_object)
-
- self.clear()
- for s in coll_reader.all_streams():
- cwd = self
- for part in s.name().split('/'):
- if part != '' and part != '.':
- partname = sanitize_filename(part)
- if partname not in cwd._entries:
- cwd._entries[partname] = self.inodes.add_entry(Directory(cwd.inode))
- cwd = cwd._entries[partname]
- for k, v in s.files().items():
- cwd._entries[sanitize_filename(k)] = self.inodes.add_entry(StreamReaderFile(cwd.inode, v, self.mtime()))
-
- def update(self):
- try:
- if self.collection_object is not None and portable_data_hash_pattern.match(self.collection_locator):
- return True
-
- with llfuse.lock_released:
- coll_reader = arvados.CollectionReader(
- self.collection_locator, self.api, self.api.localkeep(),
- num_retries=self.num_retries)
- new_collection_object = coll_reader.api_response() or {}
- # If the Collection only exists in Keep, there will be no API
- # response. Fill in the fields we need.
- if 'uuid' not in new_collection_object:
- new_collection_object['uuid'] = self.collection_locator
- if "portable_data_hash" not in new_collection_object:
- new_collection_object["portable_data_hash"] = new_collection_object["uuid"]
- if 'manifest_text' not in new_collection_object:
- new_collection_object['manifest_text'] = coll_reader.manifest_text()
- coll_reader.normalize()
- # end with llfuse.lock_released, re-acquire lock
-
- if self.collection_object is None or self.collection_object["portable_data_hash"] != new_collection_object["portable_data_hash"]:
- self.new_collection(new_collection_object, coll_reader)
-
- self.fresh()
- return True
- except apiclient.errors.NotFoundError:
- _logger.exception("arv-mount %s: error", self.collection_locator)
- except arvados.errors.ArgumentError as detail:
- _logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
- if self.collection_object is not None and "manifest_text" in self.collection_object:
- _logger.warning("arv-mount manifest_text is: %s", self.collection_object["manifest_text"])
- except Exception:
- _logger.exception("arv-mount %s: error", self.collection_locator)
- if self.collection_object is not None and "manifest_text" in self.collection_object:
- _logger.error("arv-mount manifest_text is: %s", self.collection_object["manifest_text"])
- return False
-
- def __getitem__(self, item):
- self.checkupdate()
- if item == '.arvados#collection':
- if self.collection_object_file is None:
- self.collection_object_file = ObjectFile(self.inode, self.collection_object)
- self.inodes.add_entry(self.collection_object_file)
- return self.collection_object_file
- else:
- return super(CollectionDirectory, self).__getitem__(item)
-
- def __contains__(self, k):
- if k == '.arvados#collection':
- return True
- else:
- return super(CollectionDirectory, self).__contains__(k)
-
- def mtime(self):
- self.checkupdate()
- return convertTime(self.collection_object["modified_at"]) if self.collection_object is not None and 'modified_at' in self.collection_object else 0
-
-
-class MagicDirectory(Directory):
- '''A special directory that logically contains the set of all extant keep
- locators. When a file is referenced by lookup(), it is tested to see if it
- is a valid keep locator to a manifest, and if so, loads the manifest
- contents as a subdirectory of this directory with the locator as the
- directory name. Since querying a list of all extant keep locators is
- impractical, only collections that have already been accessed are visible
- to readdir().
- '''
-
- README_TEXT = '''
-This directory provides access to Arvados collections as subdirectories listed
-by uuid (in the form 'zzzzz-4zz18-1234567890abcde') or portable data hash (in
-the form '1234567890abcdefghijklmnopqrstuv+123').
-
-Note that this directory will appear empty until you attempt to access a
-specific collection subdirectory (such as trying to 'cd' into it), at which
-point the collection will actually be looked up on the server and the directory
-will appear if it exists.
-'''.lstrip()
-
- def __init__(self, parent_inode, inodes, api, num_retries):
- super(MagicDirectory, self).__init__(parent_inode)
- self.inodes = inodes
- self.api = api
- self.num_retries = num_retries
-
- def __setattr__(self, name, value):
- super(MagicDirectory, self).__setattr__(name, value)
- # When we're assigned an inode, add a README.
- if ((name == 'inode') and (self.inode is not None) and
- (not self._entries)):
- self._entries['README'] = self.inodes.add_entry(
- StringFile(self.inode, self.README_TEXT, time.time()))
- # If we're the root directory, add an identical by_id subdirectory.
- if self.inode == llfuse.ROOT_INODE:
- self._entries['by_id'] = self.inodes.add_entry(MagicDirectory(
- self.inode, self.inodes, self.api, self.num_retries))
-
- def __contains__(self, k):
- if k in self._entries:
- return True
-
- if not portable_data_hash_pattern.match(k) and not uuid_pattern.match(k):
- return False
-
- try:
- e = self.inodes.add_entry(CollectionDirectory(
- self.inode, self.inodes, self.api, self.num_retries, k))
- if e.update():
- self._entries[k] = e
- return True
- else:
- return False
- except Exception as e:
- _logger.debug('arv-mount exception keep %s', e)
- return False
-
- def __getitem__(self, item):
- if item in self:
- return self._entries[item]
- else:
- raise KeyError("No collection with id " + item)
-
-
-class RecursiveInvalidateDirectory(Directory):
- def invalidate(self):
- if self.inode == llfuse.ROOT_INODE:
- llfuse.lock.acquire()
- try:
- super(RecursiveInvalidateDirectory, self).invalidate()
- for a in self._entries:
- self._entries[a].invalidate()
- except Exception:
- _logger.exception()
- finally:
- if self.inode == llfuse.ROOT_INODE:
- llfuse.lock.release()
-
-
-class TagsDirectory(RecursiveInvalidateDirectory):
- '''A special directory that contains as subdirectories all tags visible to the user.'''
-
- def __init__(self, parent_inode, inodes, api, num_retries, poll_time=60):
- super(TagsDirectory, self).__init__(parent_inode)
- self.inodes = inodes
- self.api = api
- self.num_retries = num_retries
- self._poll = True
- self._poll_time = poll_time
-
- def update(self):
- with llfuse.lock_released:
- tags = self.api.links().list(
- filters=[['link_class', '=', 'tag']],
- select=['name'], distinct=True
- ).execute(num_retries=self.num_retries)
- if "items" in tags:
- self.merge(tags['items'],
- lambda i: i['name'] if 'name' in i else i['uuid'],
- lambda a, i: a.tag == i,
- lambda i: TagDirectory(self.inode, self.inodes, self.api, self.num_retries, i['name'], poll=self._poll, poll_time=self._poll_time))
-
-
-class TagDirectory(Directory):
- '''A special directory that contains as subdirectories all collections visible
- to the user that are tagged with a particular tag.
- '''
-
- def __init__(self, parent_inode, inodes, api, num_retries, tag,
- poll=False, poll_time=60):
- super(TagDirectory, self).__init__(parent_inode)
- self.inodes = inodes
- self.api = api
- self.num_retries = num_retries
- self.tag = tag
- self._poll = poll
- self._poll_time = poll_time
-
- def update(self):
- with llfuse.lock_released:
- taggedcollections = self.api.links().list(
- filters=[['link_class', '=', 'tag'],
- ['name', '=', self.tag],
- ['head_uuid', 'is_a', 'arvados#collection']],
- select=['head_uuid']
- ).execute(num_retries=self.num_retries)
- self.merge(taggedcollections['items'],
- lambda i: i['head_uuid'],
- lambda a, i: a.collection_locator == i['head_uuid'],
- lambda i: CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid']))
-
-
-class ProjectDirectory(Directory):
- '''A special directory that contains the contents of a project.'''
-
- def __init__(self, parent_inode, inodes, api, num_retries, project_object,
- poll=False, poll_time=60):
- super(ProjectDirectory, self).__init__(parent_inode)
- self.inodes = inodes
- self.api = api
- self.num_retries = num_retries
- self.project_object = project_object
- self.project_object_file = None
- self.uuid = project_object['uuid']
- self._poll = poll
- self._poll_time = poll_time
-
- def createDirectory(self, i):
- if collection_uuid_pattern.match(i['uuid']):
- return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i)
- elif group_uuid_pattern.match(i['uuid']):
- return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time)
- elif link_uuid_pattern.match(i['uuid']):
- if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']):
- return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid'])
- else:
- return None
- elif uuid_pattern.match(i['uuid']):
- return ObjectFile(self.parent_inode, i)
- else:
- return None
-
- def update(self):
- if self.project_object_file == None:
- self.project_object_file = ObjectFile(self.inode, self.project_object)
- self.inodes.add_entry(self.project_object_file)
-
- def namefn(i):
- if 'name' in i:
- if i['name'] is None or len(i['name']) == 0:
- return None
- elif collection_uuid_pattern.match(i['uuid']) or group_uuid_pattern.match(i['uuid']):
- # collection or subproject
- return i['name']
- elif link_uuid_pattern.match(i['uuid']) and i['head_kind'] == 'arvados#collection':
- # name link
- return i['name']
- elif 'kind' in i and i['kind'].startswith('arvados#'):
- # something else
- return "{}.{}".format(i['name'], i['kind'][8:])
- else:
- return None
-
- def samefn(a, i):
- if isinstance(a, CollectionDirectory):
- return a.collection_locator == i['uuid']
- elif isinstance(a, ProjectDirectory):
- return a.uuid == i['uuid']
- elif isinstance(a, ObjectFile):
- return a.uuid == i['uuid'] and not a.stale()