#
# SPDX-License-Identifier: AGPL-3.0
-from __future__ import absolute_import
-from future.utils import viewitems
-from builtins import str
-from builtins import object
-from six import assertRegex
+import errno
import json
import llfuse
import logging
-import mock
import os
import subprocess
import time
import unittest
+import tempfile
+
+from pathlib import Path
+from unittest import mock
import arvados
import arvados_fuse as fuse
-from . import run_test_server
+import parameterized
+
+from arvados_fuse import fusedir
+from . import run_test_server
from .integration_test import IntegrationTest
from .mount_test_base import MountTestBase
from .test_tmp_collection import storage_classes_desired
logger = logging.getLogger('arvados.arv-mount')
-
class AssertWithTimeout(object):
"""Allow some time for an assertion to pass."""
else:
self.done = True
-
+@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
class FuseMountTest(MountTestBase):
def setUp(self):
super(FuseMountTest, self).setUp()
'dir2/dir3/thing7.txt': 'data 7',
'dir2/dir3/thing8.txt': 'data 8'}
- for k, v in viewitems(files):
+ for k, v in files.items():
with open(os.path.join(self.mounttmp, k), 'rb') as f:
self.assertEqual(v, f.read().decode())
+@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
class FuseMagicTest(MountTestBase):
def setUp(self, api=None):
super(FuseMagicTest, self).setUp(api=api)
files = {}
files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
- for k, v in viewitems(files):
+ for k, v in files.items():
with open(os.path.join(self.mounttmp, k), 'rb') as f:
self.assertEqual(v, f.read().decode())
Test().runTest()
+@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
class FuseSharedTest(MountTestBase):
def runTest(self):
self.make_mount(fuse.SharedDirectory,
'anonymously_accessible_project']
found_in = 0
found_not_in = 0
- for name, item in viewitems(run_test_server.fixture('collections')):
+ for name, item in run_test_server.fixture('collections').items():
if 'name' not in item:
pass
elif item['owner_uuid'] == public_project['uuid']:
self.assertEqual("plnp", f.read())
Test().runTest()
+@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
class FuseModifyFileTest(MountTestBase):
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
+@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
class FuseAddFileToCollectionTest(MountTestBase):
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
+@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
class FuseRemoveFileFromCollectionTest(MountTestBase):
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
pass
Test().runTest()
+@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
class FuseCreateFileTest(MountTestBase):
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
self.assertEqual(["file1.txt"], d1)
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
- assertRegex(self, collection2["manifest_text"],
+ self.assertRegex(collection2["manifest_text"],
r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
self.assertEqual(f.read(), "Hello world!")
Test().runTest()
+@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
class FuseWriteFileTest(MountTestBase):
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
self.assertEqual(12, self.operations.read_counter.get())
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
- assertRegex(self, collection2["manifest_text"],
+ self.assertRegex(collection2["manifest_text"],
r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
Test().runTest()
+@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
class FuseUpdateFileTest(MountTestBase):
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
- assertRegex(self, collection2["manifest_text"],
+ self.assertRegex(collection2["manifest_text"],
r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
- assertRegex(self, collection2["manifest_text"],
+ self.assertRegex(collection2["manifest_text"],
r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
# Starting manifest
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
- assertRegex(self, collection2["manifest_text"],
+ self.assertRegex(collection2["manifest_text"],
r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
# Empty directories are represented by an empty file named "."
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
- assertRegex(self, collection2["manifest_text"],
+ self.assertRegex(collection2["manifest_text"],
r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
# Starting manifest
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
- assertRegex(self, collection2["manifest_text"],
+ self.assertRegex(collection2["manifest_text"],
r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
- assertRegex(self, collection2["manifest_text"],
+ self.assertRegex(collection2["manifest_text"],
r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
# Starting manifest
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
- assertRegex(self, collection2["manifest_text"],
+ self.assertRegex(collection2["manifest_text"],
r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual(["file1.txt"], d1)
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
- assertRegex(self, collection2["manifest_text"],
+ self.assertRegex(collection2["manifest_text"],
r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
-def fuseFileConflictTestHelper(mounttmp):
+def fuseFileConflictTestHelper(mounttmp, uuid, keeptmp, settings):
class Test(unittest.TestCase):
def runTest(self):
+ os.environ['KEEP_LOCAL_STORE'] = keeptmp
+
with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
+ with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
+ with collection2.open("file1.txt", "w") as f2:
+ f2.write("foo")
f.write("bar")
d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
self.assertEqual(f.read(), "bar")
- assertRegex(self, d1[1],
+ self.assertRegex(d1[1],
r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
with open(os.path.join(mounttmp, d1[1]), "r") as f:
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual([], sorted(d1))
- with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
- with collection2.open("file1.txt", "w") as f:
- f.write("foo")
-
# See note in MountTestBase.setUp
- self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
+ self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
def fuseUnlinkOpenFileTest(mounttmp):
collection1.update()
collection2.update()
- assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
+ self.assertRegex(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
self.assertEqual(collection2.manifest_text(), "")
self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
collection2.update()
self.assertEqual(collection1.manifest_text(), "")
- assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
+ self.assertRegex(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
collection1.stop_threads()
collection2.stop_threads()
collection1.update()
collection2.update()
- assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
+ self.assertRegex(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
self.assertEqual(collection2.manifest_text(), "")
self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
collection2.update()
self.assertEqual(collection1.manifest_text(), "")
- assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
+ self.assertRegex(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
collection1.stop_threads()
collection2.stop_threads()
class MagicDirApiError(FuseMagicTest):
def setUp(self):
api = mock.MagicMock()
+ api.keep.block_cache = mock.MagicMock(cache_max=1)
super(MagicDirApiError, self).setUp(api=api)
api.collections().get().execute.side_effect = iter([
Exception('API fail'),
class SanitizeFilenameTest(MountTestBase):
def test_sanitize_filename(self):
- pdir = fuse.ProjectDirectory(1, {}, self.api, 0, project_object=self.api.users().current().execute())
+ pdir = fuse.ProjectDirectory(
+ 1, fuse.Inodes(None), self.api, 0, False, None,
+ project_object=self.api.users().current().execute(),
+ )
acceptable = [
"foo.txt",
".foo",
files = {}
files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
- for k, v in viewitems(files):
+ for k, v in files.items():
with open(os.path.join(self.mounttmp, k), 'rb') as f:
self.assertEqual(v, f.read().decode())
mnt_args = [
'--read-write',
'--mount-home', 'zzz',
+ '--fsns', '[SLASH]'
]
def setUp(self):
super(SlashSubstitutionTest, self).setUp()
- self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
- self.api.config = lambda: {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
+
+ self.api = arvados.safeapi.ThreadSafeApiCache(
+ arvados.config.settings(),
+ version='v1'
+ )
self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
self.fusename = 'foo[SLASH]bar[SLASH]baz'
@IntegrationTest.mount(argv=mnt_args)
- @mock.patch('arvados.util.get_config_once')
- def test_slash_substitution_before_listing(self, get_config_once):
- get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
+ def test_slash_substitution_before_listing(self):
self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
self.checkContents()
@staticmethod
def setUp(self):
super(StorageClassesTest, self).setUp()
- self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
+ self.api = arvados.safeapi.ThreadSafeApiCache(
+ arvados.config.settings(),
+ version='v1',
+ )
@IntegrationTest.mount(argv=mnt_args)
def test_collection_default_storage_classes(self):
@staticmethod
def _test_collection_custom_storage_classes(self, coll):
self.assertEqual(storage_classes_desired(coll), ['foo'])
+
+def _readonlyCollectionTestHelper(mounttmp):
+ f = open(os.path.join(mounttmp, 'thing1.txt'), 'rt')
+ # Testing that close() doesn't raise an error.
+ f.close()
+
+class ReadonlyCollectionTest(MountTestBase):
+ def setUp(self):
+ super(ReadonlyCollectionTest, self).setUp()
+ cw = arvados.collection.Collection()
+ with cw.open('thing1.txt', 'wt') as f:
+ f.write("data 1")
+ cw.save_new(owner_uuid=run_test_server.fixture("groups")["aproject"]["uuid"])
+ self.testcollection = cw.api_response()
+
+ def runTest(self):
+ settings = arvados.config.settings().copy()
+ settings["ARVADOS_API_TOKEN"] = run_test_server.fixture("api_client_authorizations")["project_viewer"]["api_token"]
+ self.api = arvados.safeapi.ThreadSafeApiCache(settings, version='v1')
+ self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection, enable_write=False)
+
+ self.pool.apply(_readonlyCollectionTestHelper, (self.mounttmp,))
+
+
+@parameterized.parameterized_class([
+ {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
+ 'project_object': run_test_server.fixture('users')['admin'],
+ }},
+ {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
+ 'project_object': run_test_server.fixture('groups')['public'],
+ }},
+])
+class UnsupportedCreateTest(MountTestBase):
+ root_class = None
+ root_kwargs = {}
+
+ def setUp(self):
+ super().setUp()
+ if 'prefs' in self.root_kwargs.get('project_object', ()):
+ self.root_kwargs['project_object']['prefs'] = {}
+ self.make_mount(self.root_class, **self.root_kwargs)
+ # Make sure the directory knows about its top-level ents.
+ os.listdir(self.mounttmp)
+
+ def test_create(self):
+ test_path = Path(self.mounttmp, 'test_create')
+ with self.assertRaises(OSError) as exc_check:
+ with test_path.open('w'):
+ pass
+ self.assertEqual(exc_check.exception.errno, errno.ENOTSUP)
+
+
+# FIXME: IMO, for consistency with the "create inside a project" case,
+# these operations should also return ENOTSUP instead of EPERM.
+# Right now they're returning EPERM because the clasess' writable() method
+# usually returns False, and the Operations class transforms that accordingly.
+# However, for cases where the mount will never be writable, I think ENOTSUP
+# is a clearer error: it lets the user know they can't fix the problem by
+# adding permissions in Arvados, etc.
+@parameterized.parameterized_class([
+ {'root_class': fusedir.MagicDirectory,
+ 'preset_dir': 'by_id',
+ 'preset_file': 'README',
+ },
+
+ {'root_class': fusedir.SharedDirectory,
+ 'root_kwargs': {
+ 'exclude': run_test_server.fixture('users')['admin']['uuid'],
+ },
+ 'preset_dir': 'Active User',
+ },
+
+ {'root_class': fusedir.TagDirectory,
+ 'root_kwargs': {
+ 'tag': run_test_server.fixture('links')['foo_collection_tag']['name'],
+ },
+ 'preset_dir': run_test_server.fixture('collections')['foo_collection_in_aproject']['uuid'],
+ },
+
+ {'root_class': fusedir.TagsDirectory,
+ 'preset_dir': run_test_server.fixture('links')['foo_collection_tag']['name'],
+ },
+])
+class UnsupportedOperationsTest(UnsupportedCreateTest):
+ preset_dir = None
+ preset_file = None
+
+ def test_create(self):
+ test_path = Path(self.mounttmp, 'test_create')
+ with self.assertRaises(OSError) as exc_check:
+ with test_path.open('w'):
+ pass
+ self.assertEqual(exc_check.exception.errno, errno.EPERM)
+
+ def test_mkdir(self):
+ test_path = Path(self.mounttmp, 'test_mkdir')
+ with self.assertRaises(OSError) as exc_check:
+ test_path.mkdir()
+ self.assertEqual(exc_check.exception.errno, errno.EPERM)
+
+ def test_rename(self):
+ src_name = self.preset_dir or self.preset_file
+ if src_name is None:
+ return
+ test_src = Path(self.mounttmp, src_name)
+ test_dst = test_src.with_name('test_dst')
+ with self.assertRaises(OSError) as exc_check:
+ test_src.rename(test_dst)
+ self.assertEqual(exc_check.exception.errno, errno.EPERM)
+
+ def test_rmdir(self):
+ if self.preset_dir is None:
+ return
+ test_path = Path(self.mounttmp, self.preset_dir)
+ with self.assertRaises(OSError) as exc_check:
+ test_path.rmdir()
+ self.assertEqual(exc_check.exception.errno, errno.EPERM)
+
+ def test_unlink(self):
+ if self.preset_file is None:
+ return
+ test_path = Path(self.mounttmp, self.preset_file)
+ with self.assertRaises(OSError) as exc_check:
+ test_path.unlink()
+ self.assertEqual(exc_check.exception.errno, errno.EPERM)
projects / arvados.git / blobdiff