From 98fd0639a6703084ad9877d9713b8fa4a8dfb03d Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Tue, 25 Nov 2014 15:24:08 -0500 Subject: [PATCH] 4501: FUSE exposes by_id subdirectory when run with --by-id. This makes it possible for people to develop code against their own normal Keep mount, and then have it work as expected in Crunch (which uses --by-id for backward compatibility). I considered adding a second inode for by_id that pointed to the same underlying MagicDirectory instance. Unfortunately, the current implementation would make it difficult to avoid exposing infinitely recursing by_id subdirectories, and that seemed like a bigger problem than the relatively small overhead of having two MagicDirectory instances. (The Keep client will use comparatively more RAM, and it will use the same block cache for both directories, which should keep it relatively under control.) --- services/fuse/arvados_fuse/__init__.py | 50 +++++++++++++------------- services/fuse/tests/test_mount.py | 27 +++++++------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py index 9154c827ca..b2a3b2e7a1 100644 --- a/services/fuse/arvados_fuse/__init__.py +++ b/services/fuse/arvados_fuse/__init__.py @@ -5,7 +5,6 @@ import os import sys import llfuse -from llfuse import FUSEError import errno import stat import threading @@ -20,6 +19,8 @@ import time import _strptime import calendar import threading +import itertools + from arvados.util import portable_data_hash_pattern, uuid_pattern, collection_uuid_pattern, group_uuid_pattern, user_uuid_pattern, link_uuid_pattern _logger = logging.getLogger('arvados.arvados_fuse') @@ -391,18 +392,8 @@ class MagicDirectory(Directory): to readdir(). ''' - def __init__(self, parent_inode, inodes, api, num_retries): - super(MagicDirectory, self).__init__(parent_inode) - self.inodes = inodes - self.api = api - self.num_retries = num_retries - # Have to defer creating readme_file because at this point we don't - # yet have an inode assigned. - self.readme_file = None - - def create_readme(self): - if self.readme_file is None: - text = '''This directory provides access to Arvados collections as subdirectories listed + README_TEXT = ''' +This directory provides access to Arvados collections as subdirectories listed by uuid (in the form 'zzzzz-4zz18-1234567890abcde') or portable data hash (in the form '1234567890abcdefghijklmnopqrstuv+123'). @@ -410,13 +401,27 @@ Note that this directory will appear empty until you attempt to access a specific collection subdirectory (such as trying to 'cd' into it), at which point the collection will actually be looked up on the server and the directory will appear if it exists. -''' - self.readme_file = self.inodes.add_entry(StringFile(self.inode, text, time.time())) - self._entries["README"] = self.readme_file +'''.lstrip() - def __contains__(self, k): - self.create_readme() + def __init__(self, parent_inode, inodes, api, num_retries): + super(MagicDirectory, self).__init__(parent_inode) + self.inodes = inodes + self.api = api + self.num_retries = num_retries + + def __setattr__(self, name, value): + super(MagicDirectory, self).__setattr__(name, value) + # When we're assigned an inode, add a README. + if ((name == 'inode') and (self.inode is not None) and + (not self._entries)): + self._entries['README'] = self.inodes.add_entry( + StringFile(self.inode, self.README_TEXT, time.time())) + # If we're the root directory, add an identical by_id subdirectory. + if self.inode == llfuse.ROOT_INODE: + self._entries['by_id'] = self.inodes.add_entry(MagicDirectory( + self.inode, self.inodes, self.api, self.num_retries)) + def __contains__(self, k): if k in self._entries: return True @@ -435,10 +440,6 @@ will appear if it exists. _logger.debug('arv-mount exception keep %s', e) return False - def items(self): - self.create_readme() - return self._entries.items() - def __getitem__(self, item): if item in self: return self._entries[item] @@ -692,7 +693,7 @@ class Inodes(object): def __init__(self): self._entries = {} - self._counter = llfuse.ROOT_INODE + self._counter = itertools.count(llfuse.ROOT_INODE) def __getitem__(self, item): return self._entries[item] @@ -710,9 +711,8 @@ class Inodes(object): return k in self._entries def add_entry(self, entry): - entry.inode = self._counter + entry.inode = next(self._counter) self._entries[entry.inode] = entry - self._counter += 1 return entry def del_entry(self, entry): diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py index b4eb27f52c..f0f375c831 100644 --- a/services/fuse/tests/test_mount.py +++ b/services/fuse/tests/test_mount.py @@ -140,8 +140,6 @@ class FuseMagicTest(MountTestBase): operations = fuse.Operations(os.getuid(), os.getgid()) e = operations.inodes.add_entry(fuse.MagicDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0)) - self.mounttmp = tempfile.mkdtemp() - llfuse.init(operations, self.mounttmp, []) t = threading.Thread(None, lambda: llfuse.main()) t.start() @@ -150,17 +148,20 @@ class FuseMagicTest(MountTestBase): operations.initlock.wait() # now check some stuff - d1 = os.listdir(self.mounttmp) - d1.sort() - self.assertEqual(['README'], d1) - - d2 = os.listdir(os.path.join(self.mounttmp, self.testcollection)) - d2.sort() - self.assertEqual(['thing1.txt'], d2) - - d3 = os.listdir(self.mounttmp) - d3.sort() - self.assertEqual([self.testcollection, 'README'], d3) + mount_ls = os.listdir(self.mounttmp) + self.assertIn('README', mount_ls) + self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or + arvados.util.uuid_pattern.match(fn) + for fn in mount_ls), + "new FUSE MagicDirectory lists Collection") + self.assertDirContents(self.testcollection, ['thing1.txt']) + self.assertDirContents(os.path.join('by_id', self.testcollection), + ['thing1.txt']) + mount_ls = os.listdir(self.mounttmp) + self.assertIn('README', mount_ls) + self.assertIn(self.testcollection, mount_ls) + self.assertIn(self.testcollection, + os.listdir(os.path.join(self.mounttmp, 'by_id'))) files = {} files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1' -- 2.39.5