def __init__(self, cap, min_entries=4):
self._entries = collections.OrderedDict()
self._by_uuid = {}
- self._counter = itertools.count(1)
+ self._counter = itertools.count(0)
self.cap = cap
self._total = 0
self.min_entries = min_entries
def add_entry(self, entry):
entry.inode = next(self._counter)
+ if entry.inode == llfuse.ROOT_INODE:
+ entry.inc_ref()
self._entries[entry.inode] = entry
self.inode_cache.manage(entry)
return entry
def del_entry(self, entry):
if entry.ref_count == 0:
- _logger.warn("Deleting inode %i", entry.inode)
+ _logger.debug("Deleting inode %i", entry.inode)
self.inode_cache.unmanage(entry)
llfuse.invalidate_inode(entry.inode)
del self._entries[entry.inode]
+ entry.inode = None
else:
- _logger.warn("Inode %i has refcount %i", entry.inode, entry.ref_count)
entry.dead = True
+ _logger.debug("del_entry on inode %i with refcount %i", entry.inode, entry.ref_count)
def catch_exceptions(orig_func):
@functools.wraps(orig_func)
# dict of inode to filehandle
self._filehandles = {}
- self._filehandles_counter = 1
+ self._filehandles_counter = itertools.count(0)
# Other threads that need to wait until the fuse driver
# is fully initialized should wait() on this event object.
@catch_exceptions
def forget(self, inodes):
for inode, nlookup in inodes:
- _logger.debug("arv-mount forget: %i %i", inode, nlookup)
ent = self.inodes[inode]
+ _logger.debug("arv-mount forget: inode %i nlookup %i ref_count %i", inode, nlookup, ent.ref_count)
if ent.dec_ref(nlookup) == 0 and ent.dead:
self.inodes.del_entry(ent)
if ((flags & os.O_WRONLY) or (flags & os.O_RDWR)) and not p.writable():
raise llfuse.FUSEError(errno.EPERM)
- fh = self._filehandles_counter
- self._filehandles_counter += 1
+ fh = next(self._filehandles_counter)
self._filehandles[fh] = FileHandle(fh, p)
self.inodes.touch(p)
return fh
if not isinstance(p, Directory):
raise llfuse.FUSEError(errno.ENOTDIR)
- fh = self._filehandles_counter
- self._filehandles_counter += 1
+ fh = next(self._filehandles_counter)
if p.parent_inode in self.inodes:
parent = self.inodes[p.parent_inode]
else:
# The file entry should have been implicitly created by callback.
f = p[name]
- fh = self._filehandles_counter
- self._filehandles_counter += 1
+ fh = next(self._filehandles_counter)
self._filehandles[fh] = FileHandle(fh, f)
self.inodes.touch(p)
@catch_exceptions
def mkdir(self, inode_parent, name, mode, ctx):
+ _logger.debug("arv-mount mkdir: %i '%s' %o", inode_parent, name, mode)
+
p = self._check_writable(inode_parent)
with llfuse.lock_released:
@catch_exceptions
def unlink(self, inode_parent, name):
+ _logger.debug("arv-mount unlink: %i '%s'", inode_parent, name)
p = self._check_writable(inode_parent)
with llfuse.lock_released:
@catch_exceptions
def rename(self, inode_parent_old, name_old, inode_parent_new, name_new):
+ _logger.debug("arv-mount rename: %i '%s' %i '%s'", inode_parent_old, name_old, inode_parent_new, name_new)
src = self._check_writable(inode_parent_old)
dest = self._check_writable(inode_parent_new)
r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A[a-f0-9]{40}@[a-f0-9]{8} 0:0:file1\.txt$')
def fuseWriteFileTestHelper(mounttmp):
- with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
- return f.read() == "Hello world!"
+ class Test(unittest.TestCase):
+ def runTest(self):
+ with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
+ self.assertEqual(f.read(), "Hello world!")
+ Test().runTest()
class FuseWriteFileTest(MountTestBase):
def runTest(self):
# workaround is to run some of our test code in a separate process.
# Forturnately the multiprocessing module makes this relatively easy.
pool = multiprocessing.Pool(1)
- self.assertTrue(pool.apply(fuseWriteFileTestHelper, (self.mounttmp,)))
+ pool.apply(fuseWriteFileTestHelper, (self.mounttmp,))
pool.close()
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
def fuseUpdateFileTestHelper1(mounttmp):
- with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
- fr = f.read()
- if fr != "Hello world!":
- raise Exception("Got %s expected 'Hello world!'" % fr)
- f.seek(0)
- f.write("Hola mundo!")
- f.seek(0)
- fr = f.read()
- if fr != "Hola mundo!!":
- raise Exception("Got %s expected 'Hola mundo!!'" % fr)
- return True
+ class Test(unittest.TestCase):
+ def runTest(self):
+ with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
+ fr = f.read()
+ self.assertEqual(fr, "Hello world!")
+ f.seek(0)
+ f.write("Hola mundo!")
+ f.seek(0)
+ fr = f.read()
+ self.assertEqual(fr, "Hola mundo!!")
+ return True
+ Test().runTest()
def fuseUpdateFileTestHelper2(mounttmp):
- with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
- return f.read() == "Hola mundo!!"
+ class Test(unittest.TestCase):
+ def runTest(self):
+ with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
+ return f.read() == "Hola mundo!!"
+ Test().runTest()
class FuseUpdateFileTest(MountTestBase):
def runTest(self):
# See note in FuseWriteFileTest
pool = multiprocessing.Pool(1)
- self.assertTrue(pool.apply(fuseUpdateFileTestHelper1, (self.mounttmp,)))
- self.assertTrue(pool.apply(fuseUpdateFileTestHelper2, (self.mounttmp,)))
+ pool.apply(fuseUpdateFileTestHelper1, (self.mounttmp,))
+ pool.apply(fuseUpdateFileTestHelper2, (self.mounttmp,))
pool.close()
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertEqual(collection2["manifest_text"], "")
-class FuseMvTest(MountTestBase):
+class FuseMvFileTest(MountTestBase):
def runTest(self):
arvados.logger.setLevel(logging.DEBUG)
r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+class FuseRenameTest(MountTestBase):
+ def runTest(self):
+ arvados.logger.setLevel(logging.DEBUG)
+
+ collection = arvados.collection.Collection(api_client=self.api)
+ collection.save_new()
+
+ m = self.make_mount(fuse.CollectionDirectory)
+ with llfuse.lock:
+ m.new_collection(collection.api_response(), collection)
+ self.assertTrue(m.writable())
+
+ os.mkdir(os.path.join(self.mounttmp, "testdir"))
+
+ with open(os.path.join(self.mounttmp, "testdir", "file1.txt"), "w") as f:
+ f.write("Hello world!")
+
+ # Starting manifest
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertRegexpMatches(collection2["manifest_text"],
+ r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+ d1 = llfuse.listdir(os.path.join(self.mounttmp))
+ self.assertEqual(["testdir"], d1)
+ d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
+ self.assertEqual(["file1.txt"], d1)
+
+ os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
+
+ d1 = llfuse.listdir(os.path.join(self.mounttmp))
+ self.assertEqual(["testdir2"], sorted(d1))
+ d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
+ self.assertEqual(["file1.txt"], d1)
+
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertRegexpMatches(collection2["manifest_text"],
+ r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+
class FuseUpdateFromEventTest(MountTestBase):
def runTest(self):
arvados.logger.setLevel(logging.DEBUG)
self.assertEqual(["file1.txt"], sorted(d1))
+def fuseFileConflictTestHelper(mounttmp):
+ class Test(unittest.TestCase):
+ def runTest(self):
+ d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
+ self.assertEqual(len(d1), 2)
+
+ with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
+ self.assertEqual(f.read(), "bar")
+
+ self.assertRegexpMatches(d1[1],
+ r'file1\.txt~conflict-\d\d\d\d-\d\d-\d\d-\d\d:\d\d:\d\d~')
+
+ with open(os.path.join(mounttmp, d1[1]), "r") as f:
+ self.assertEqual(f.read(), "foo")
+
+ Test().runTest()
+
+class FuseFileConflictTest(MountTestBase):
+ def runTest(self):
+ arvados.logger.setLevel(logging.DEBUG)
+
+ collection = arvados.collection.Collection(api_client=self.api)
+ collection.save_new()
+
+ m = self.make_mount(fuse.CollectionDirectory)
+ with llfuse.lock:
+ m.new_collection(collection.api_response(), collection)
+
+ d1 = llfuse.listdir(os.path.join(self.mounttmp))
+ self.assertEqual([], sorted(d1))
+
+ with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
+ with collection2.open("file1.txt", "w") as f:
+ f.write("foo")
+
+ with open(os.path.join(self.mounttmp, "file1.txt"), "w") as f:
+ f.write("bar")
+
+ # See comment in FuseWriteFileTest
+ pool = multiprocessing.Pool(1)
+ pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
+ pool.close()
+
+
+def fuseUnlinkOpenFileTest(mounttmp):
+ class Test(unittest.TestCase):
+ def runTest(self):
+ with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
+ f.write("foo")
+
+ d1 = llfuse.listdir(os.path.join(mounttmp))
+ self.assertEqual(["file1.txt"], sorted(d1))
+
+ os.remove(os.path.join(mounttmp, "file1.txt"))
+
+ d1 = llfuse.listdir(os.path.join(mounttmp))
+ self.assertEqual([], sorted(d1))
+
+ f.seek(0)
+ self.assertEqual(f.read(), "foo")
+ f.write("bar")
+
+ f.seek(0)
+ self.assertEqual(f.read(), "foobar")
+
+ Test().runTest()
+
+class FuseUnlinkOpenFileTest(MountTestBase):
+ def runTest(self):
+ arvados.logger.setLevel(logging.DEBUG)
+
+ collection = arvados.collection.Collection(api_client=self.api)
+ collection.save_new()
+
+ m = self.make_mount(fuse.CollectionDirectory)
+ with llfuse.lock:
+ m.new_collection(collection.api_response(), collection)
+
+ # See comment in FuseWriteFileTest
+ pool = multiprocessing.Pool(1)
+ pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
+ pool.close()
+
+ self.assertEqual(collection.manifest_text(), "")
+
+
+def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
+ class Test(unittest.TestCase):
+ def runTest(self):
+ with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
+ f.write("Hello world!")
+
+ d1 = os.listdir(os.path.join(mounttmp, uuid1))
+ self.assertEqual(["file1.txt"], sorted(d1))
+ d1 = os.listdir(os.path.join(mounttmp, uuid2))
+ self.assertEqual([], sorted(d1))
+
+ Test().runTest()
+
+def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
+ class Test(unittest.TestCase):
+ def runTest(self):
+ os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
+
+ d1 = os.listdir(os.path.join(mounttmp, uuid1))
+ self.assertEqual([], sorted(d1))
+ d1 = os.listdir(os.path.join(mounttmp, uuid2))
+ self.assertEqual(["file2.txt"], sorted(d1))
+
+ Test().runTest()
+
+class FuseMvFileBetweenCollectionsTest(MountTestBase):
+ def runTest(self):
+ arvados.logger.setLevel(logging.DEBUG)
+
+ collection1 = arvados.collection.Collection(api_client=self.api)
+ collection1.save_new()
+
+ collection2 = arvados.collection.Collection(api_client=self.api)
+ collection2.save_new()
+
+ m = self.make_mount(fuse.MagicDirectory)
+
+ # See comment in FuseWriteFileTest
+ pool = multiprocessing.Pool(1)
+ pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
+ collection1.manifest_locator(),
+ collection2.manifest_locator()))
+
+ collection1.update()
+ collection2.update()
+
+ self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+ self.assertEqual(collection2.manifest_text(), "")
+
+ pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
+ collection1.manifest_locator(),
+ collection2.manifest_locator()))
+ pool.close()
+
+ collection1.update()
+ collection2.update()
+
+ self.assertEqual(collection1.manifest_text(), "")
+ self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file2\.txt$")
+
+
+def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
+ class Test(unittest.TestCase):
+ def runTest(self):
+ os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
+ with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
+ f.write("Hello world!")
+
+ d1 = os.listdir(os.path.join(mounttmp, uuid1))
+ self.assertEqual(["testdir"], sorted(d1))
+ d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
+ self.assertEqual(["file1.txt"], sorted(d1))
+
+ d1 = os.listdir(os.path.join(mounttmp, uuid2))
+ self.assertEqual([], sorted(d1))
+
+ Test().runTest()
+
+def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
+ class Test(unittest.TestCase):
+ def runTest(self):
+ os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
+
+ d1 = os.listdir(os.path.join(mounttmp, uuid1))
+ self.assertEqual([], sorted(d1))
+
+ d1 = os.listdir(os.path.join(mounttmp, uuid2))
+ self.assertEqual(["testdir2"], sorted(d1))
+ d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
+ self.assertEqual(["file1.txt"], sorted(d1))
+
+ Test().runTest()
+
+class FuseMvDirBetweenCollectionsTest(MountTestBase):
+ def runTest(self):
+ arvados.logger.setLevel(logging.DEBUG)
+
+ collection1 = arvados.collection.Collection(api_client=self.api)
+ collection1.save_new()
+
+ collection2 = arvados.collection.Collection(api_client=self.api)
+ collection2.save_new()
+
+ m = self.make_mount(fuse.MagicDirectory)
+
+ # See comment in FuseWriteFileTest
+ pool = multiprocessing.Pool(1)
+ pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
+ collection1.manifest_locator(),
+ collection2.manifest_locator()))
+
+ collection1.update()
+ collection2.update()
+
+ self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+ self.assertEqual(collection2.manifest_text(), "")
+
+ pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
+ collection1.manifest_locator(),
+ collection2.manifest_locator()))
+ pool.close()
+
+ collection1.update()
+ collection2.update()
+
+ self.assertEqual(collection1.manifest_text(), "")
+ self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+
+
class FuseUnitTest(unittest.TestCase):
def test_sanitize_filename(self):
acceptable = [