1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: AGPL-3.0
5 from __future__ import absolute_import
6 from future.utils import viewitems
7 from builtins import str
8 from builtins import object
9 from six import assertRegex
22 import arvados_fuse as fuse
23 from . import run_test_server
25 from .integration_test import IntegrationTest
26 from .mount_test_base import MountTestBase
27 from .test_tmp_collection import storage_classes_desired
29 logger = logging.getLogger('arvados.arv-mount')
32 class AssertWithTimeout(object):
33 """Allow some time for an assertion to pass."""
35 def __init__(self, timeout=0):
36 self.timeout = timeout
39 self.deadline = time.time() + self.timeout
48 def attempt(self, fn, *args, **kwargs):
51 except AssertionError:
52 if time.time() > self.deadline:
58 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
59 class FuseMountTest(MountTestBase):
61 super(FuseMountTest, self).setUp()
63 cw = arvados.CollectionWriter()
65 cw.start_new_file('thing1.txt')
67 cw.start_new_file('thing2.txt')
70 cw.start_new_stream('dir1')
71 cw.start_new_file('thing3.txt')
73 cw.start_new_file('thing4.txt')
76 cw.start_new_stream('dir2')
77 cw.start_new_file('thing5.txt')
79 cw.start_new_file('thing6.txt')
82 cw.start_new_stream('dir2/dir3')
83 cw.start_new_file('thing7.txt')
86 cw.start_new_file('thing8.txt')
89 cw.start_new_stream('edgecases')
90 for f in ":/.../-/*/ ".split("/"):
94 for f in ":/.../-/*/ ".split("/"):
95 cw.start_new_stream('edgecases/dirs/' + f)
96 cw.start_new_file('x/x')
99 self.testcollection = cw.finish()
100 self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
103 self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
105 self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
106 'edgecases', 'dir1', 'dir2'])
107 self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
108 self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
109 self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
110 self.assertDirContents('edgecases',
111 "dirs/:/.../-/*/ ".split("/"))
112 self.assertDirContents('edgecases/dirs',
113 ":/.../-/*/ ".split("/"))
115 files = {'thing1.txt': 'data 1',
116 'thing2.txt': 'data 2',
117 'dir1/thing3.txt': 'data 3',
118 'dir1/thing4.txt': 'data 4',
119 'dir2/thing5.txt': 'data 5',
120 'dir2/thing6.txt': 'data 6',
121 'dir2/dir3/thing7.txt': 'data 7',
122 'dir2/dir3/thing8.txt': 'data 8'}
124 for k, v in viewitems(files):
125 with open(os.path.join(self.mounttmp, k), 'rb') as f:
126 self.assertEqual(v, f.read().decode())
129 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
130 class FuseMagicTest(MountTestBase):
131 def setUp(self, api=None):
132 super(FuseMagicTest, self).setUp(api=api)
134 self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
135 self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
136 self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
137 self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
138 self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
140 cw = arvados.CollectionWriter()
142 cw.start_new_file('thing1.txt')
145 self.testcollection = cw.finish()
146 self.test_manifest = cw.manifest_text()
147 coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
148 self.test_manifest_pdh = coll['portable_data_hash']
151 self.make_mount(fuse.MagicDirectory)
153 mount_ls = llfuse.listdir(self.mounttmp)
154 self.assertIn('README', mount_ls)
155 self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
156 arvados.util.uuid_pattern.match(fn)
158 "new FUSE MagicDirectory has no collections or projects")
159 self.assertDirContents(self.testcollection, ['thing1.txt'])
160 self.assertDirContents(os.path.join('by_id', self.testcollection),
162 self.assertIn(self.collection_in_test_project,
163 llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
164 self.assertIn(self.collection_in_test_project,
165 llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
166 self.assertIn(self.collection_in_filter_group,
167 llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
168 self.assertIn(self.collection_in_filter_group,
169 llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
172 mount_ls = llfuse.listdir(self.mounttmp)
173 self.assertIn('README', mount_ls)
174 self.assertIn(self.testcollection, mount_ls)
175 self.assertIn(self.testcollection,
176 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
177 self.assertIn(self.test_project, mount_ls)
178 self.assertIn(self.test_project,
179 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
180 self.assertIn(self.filter_group,
181 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
183 with self.assertRaises(OSError):
184 llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
187 files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
189 for k, v in viewitems(files):
190 with open(os.path.join(self.mounttmp, k), 'rb') as f:
191 self.assertEqual(v, f.read().decode())
194 class FuseTagsTest(MountTestBase):
196 self.make_mount(fuse.TagsDirectory)
198 d1 = llfuse.listdir(self.mounttmp)
200 self.assertEqual(['foo_tag'], d1)
202 d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
204 self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
206 d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
208 self.assertEqual(['foo'], d3)
211 class FuseTagsUpdateTest(MountTestBase):
212 def tag_collection(self, coll_uuid, tag_name):
213 return self.api.links().create(
214 body={'link': {'head_uuid': coll_uuid,
220 self.make_mount(fuse.TagsDirectory, poll_time=1)
222 self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
224 bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
225 self.tag_collection(bar_uuid, 'fuse_test_tag')
226 for attempt in AssertWithTimeout(10):
227 attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
228 self.assertDirContents('fuse_test_tag', [bar_uuid])
230 baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
231 l = self.tag_collection(baz_uuid, 'fuse_test_tag')
232 for attempt in AssertWithTimeout(10):
233 attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
235 self.api.links().delete(uuid=l['uuid']).execute()
236 for attempt in AssertWithTimeout(10):
237 attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
240 def fuseSharedTestHelper(mounttmp):
241 class Test(unittest.TestCase):
243 # Double check that we can open and read objects in this folder as a file,
244 # and that its contents are what we expect.
245 baz_path = os.path.join(
249 'collection in FUSE project',
251 with open(baz_path) as f:
252 self.assertEqual("baz", f.read())
254 # check mtime on collection
255 st = os.stat(baz_path)
257 mtime = st.st_mtime_ns // 1000000000
258 except AttributeError:
260 self.assertEqual(mtime, 1391448174)
262 # shared_dirs is a list of the directories exposed
263 # by fuse.SharedDirectory (i.e. any object visible
264 # to the current user)
265 shared_dirs = llfuse.listdir(mounttmp)
267 self.assertIn('FUSE User', shared_dirs)
269 # fuse_user_objs is a list of the objects owned by the FUSE
270 # test user (which present as files in the 'FUSE User'
272 fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
273 fuse_user_objs.sort()
274 self.assertEqual(['FUSE Test Project', # project owned by user
275 'collection #1 owned by FUSE', # collection owned by user
276 'collection #2 owned by FUSE' # collection owned by user
279 # test_proj_files is a list of the files in the FUSE Test Project.
280 test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
281 test_proj_files.sort()
282 self.assertEqual(['collection in FUSE project'
288 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
289 class FuseSharedTest(MountTestBase):
291 self.make_mount(fuse.SharedDirectory,
292 exclude=self.api.users().current().execute()['uuid'])
293 keep = arvados.keep.KeepClient()
294 keep.put("baz".encode())
296 self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
299 class FuseHomeTest(MountTestBase):
301 self.make_mount(fuse.ProjectDirectory,
302 project_object=self.api.users().current().execute())
304 d1 = llfuse.listdir(self.mounttmp)
305 self.assertIn('Unrestricted public data', d1)
307 d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
308 public_project = run_test_server.fixture('groups')[
309 'anonymously_accessible_project']
312 for name, item in viewitems(run_test_server.fixture('collections')):
313 if 'name' not in item:
315 elif item['owner_uuid'] == public_project['uuid']:
316 self.assertIn(item['name'], d2)
319 # Artificial assumption here: there is no public
320 # collection fixture with the same name as a
321 # non-public collection.
322 self.assertNotIn(item['name'], d2)
324 self.assertNotEqual(0, found_in)
325 self.assertNotEqual(0, found_not_in)
327 d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
328 self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
331 def fuseModifyFileTestHelperReadStartContents(mounttmp):
332 class Test(unittest.TestCase):
334 d1 = llfuse.listdir(mounttmp)
335 self.assertEqual(["file1.txt"], d1)
336 with open(os.path.join(mounttmp, "file1.txt")) as f:
337 self.assertEqual("blub", f.read())
340 def fuseModifyFileTestHelperReadEndContents(mounttmp):
341 class Test(unittest.TestCase):
343 d1 = llfuse.listdir(mounttmp)
344 self.assertEqual(["file1.txt"], d1)
345 with open(os.path.join(mounttmp, "file1.txt")) as f:
346 self.assertEqual("plnp", f.read())
349 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
350 class FuseModifyFileTest(MountTestBase):
352 collection = arvados.collection.Collection(api_client=self.api)
353 with collection.open("file1.txt", "w") as f:
356 collection.save_new()
358 m = self.make_mount(fuse.CollectionDirectory)
360 m.new_collection(collection.api_response(), collection)
362 self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
364 with collection.open("file1.txt", "w") as f:
367 self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
370 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
371 class FuseAddFileToCollectionTest(MountTestBase):
373 collection = arvados.collection.Collection(api_client=self.api)
374 with collection.open("file1.txt", "w") as f:
377 collection.save_new()
379 m = self.make_mount(fuse.CollectionDirectory)
381 m.new_collection(collection.api_response(), collection)
383 d1 = llfuse.listdir(self.mounttmp)
384 self.assertEqual(["file1.txt"], d1)
386 with collection.open("file2.txt", "w") as f:
389 d1 = llfuse.listdir(self.mounttmp)
390 self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
393 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
394 class FuseRemoveFileFromCollectionTest(MountTestBase):
396 collection = arvados.collection.Collection(api_client=self.api)
397 with collection.open("file1.txt", "w") as f:
400 with collection.open("file2.txt", "w") as f:
403 collection.save_new()
405 m = self.make_mount(fuse.CollectionDirectory)
407 m.new_collection(collection.api_response(), collection)
409 d1 = llfuse.listdir(self.mounttmp)
410 self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
412 collection.remove("file2.txt")
414 d1 = llfuse.listdir(self.mounttmp)
415 self.assertEqual(["file1.txt"], d1)
418 def fuseCreateFileTestHelper(mounttmp):
419 class Test(unittest.TestCase):
421 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
425 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
426 class FuseCreateFileTest(MountTestBase):
428 collection = arvados.collection.Collection(api_client=self.api)
429 collection.save_new()
431 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
432 self.assertEqual(collection2["manifest_text"], "")
434 collection.save_new()
436 m = self.make_mount(fuse.CollectionDirectory)
438 m.new_collection(collection.api_response(), collection)
439 self.assertTrue(m.writable())
441 self.assertNotIn("file1.txt", collection)
443 self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
445 self.assertIn("file1.txt", collection)
447 d1 = llfuse.listdir(self.mounttmp)
448 self.assertEqual(["file1.txt"], d1)
450 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
451 assertRegex(self, collection2["manifest_text"],
452 r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
455 def fuseWriteFileTestHelperWriteFile(mounttmp):
456 class Test(unittest.TestCase):
458 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
459 f.write("Hello world!")
462 def fuseWriteFileTestHelperReadFile(mounttmp):
463 class Test(unittest.TestCase):
465 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
466 self.assertEqual(f.read(), "Hello world!")
469 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
470 class FuseWriteFileTest(MountTestBase):
472 collection = arvados.collection.Collection(api_client=self.api)
473 collection.save_new()
475 m = self.make_mount(fuse.CollectionDirectory)
477 m.new_collection(collection.api_response(), collection)
478 self.assertTrue(m.writable())
480 self.assertNotIn("file1.txt", collection)
482 self.assertEqual(0, self.operations.write_counter.get())
483 self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
484 self.assertEqual(12, self.operations.write_counter.get())
486 with collection.open("file1.txt") as f:
487 self.assertEqual(f.read(), "Hello world!")
489 self.assertEqual(0, self.operations.read_counter.get())
490 self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
491 self.assertEqual(12, self.operations.read_counter.get())
493 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
494 assertRegex(self, collection2["manifest_text"],
495 r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
498 def fuseUpdateFileTestHelper(mounttmp):
499 class Test(unittest.TestCase):
501 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
502 f.write("Hello world!")
504 with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
506 self.assertEqual(fr, "Hello world!")
508 f.write("Hola mundo!")
511 self.assertEqual(fr, "Hola mundo!!")
513 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
514 self.assertEqual(f.read(), "Hola mundo!!")
518 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
519 class FuseUpdateFileTest(MountTestBase):
521 collection = arvados.collection.Collection(api_client=self.api)
522 collection.save_new()
524 m = self.make_mount(fuse.CollectionDirectory)
526 m.new_collection(collection.api_response(), collection)
527 self.assertTrue(m.writable())
529 # See note in MountTestBase.setUp
530 self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
532 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
533 assertRegex(self, collection2["manifest_text"],
534 r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
537 def fuseMkdirTestHelper(mounttmp):
538 class Test(unittest.TestCase):
540 with self.assertRaises(IOError):
541 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
542 f.write("Hello world!")
544 os.mkdir(os.path.join(mounttmp, "testdir"))
546 with self.assertRaises(OSError):
547 os.mkdir(os.path.join(mounttmp, "testdir"))
549 d1 = llfuse.listdir(mounttmp)
550 self.assertEqual(["testdir"], d1)
552 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
553 f.write("Hello world!")
555 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
556 self.assertEqual(["file1.txt"], d1)
560 class FuseMkdirTest(MountTestBase):
562 collection = arvados.collection.Collection(api_client=self.api)
563 collection.save_new()
565 m = self.make_mount(fuse.CollectionDirectory)
567 m.new_collection(collection.api_response(), collection)
568 self.assertTrue(m.writable())
570 self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
572 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
573 assertRegex(self, collection2["manifest_text"],
574 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
577 def fuseRmTestHelperWriteFile(mounttmp):
578 class Test(unittest.TestCase):
580 os.mkdir(os.path.join(mounttmp, "testdir"))
582 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
583 f.write("Hello world!")
587 def fuseRmTestHelperDeleteFile(mounttmp):
588 class Test(unittest.TestCase):
590 # Can't delete because it's not empty
591 with self.assertRaises(OSError):
592 os.rmdir(os.path.join(mounttmp, "testdir"))
594 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
595 self.assertEqual(["file1.txt"], d1)
598 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
600 # Make sure it's empty
601 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
602 self.assertEqual([], d1)
604 # Try to delete it again
605 with self.assertRaises(OSError):
606 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
610 def fuseRmTestHelperRmdir(mounttmp):
611 class Test(unittest.TestCase):
613 # Should be able to delete now that it is empty
614 os.rmdir(os.path.join(mounttmp, "testdir"))
616 # Make sure it's empty
617 d1 = llfuse.listdir(os.path.join(mounttmp))
618 self.assertEqual([], d1)
620 # Try to delete it again
621 with self.assertRaises(OSError):
622 os.rmdir(os.path.join(mounttmp, "testdir"))
626 class FuseRmTest(MountTestBase):
628 collection = arvados.collection.Collection(api_client=self.api)
629 collection.save_new()
631 m = self.make_mount(fuse.CollectionDirectory)
633 m.new_collection(collection.api_response(), collection)
634 self.assertTrue(m.writable())
636 self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
639 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
640 assertRegex(self, collection2["manifest_text"],
641 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
642 self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
644 # Empty directories are represented by an empty file named "."
645 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
646 assertRegex(self, collection2["manifest_text"],
647 r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
649 self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
651 # manifest should be empty now.
652 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
653 self.assertEqual(collection2["manifest_text"], "")
656 def fuseMvFileTestHelperWriteFile(mounttmp):
657 class Test(unittest.TestCase):
659 os.mkdir(os.path.join(mounttmp, "testdir"))
661 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
662 f.write("Hello world!")
666 def fuseMvFileTestHelperMoveFile(mounttmp):
667 class Test(unittest.TestCase):
669 d1 = llfuse.listdir(os.path.join(mounttmp))
670 self.assertEqual(["testdir"], d1)
671 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
672 self.assertEqual(["file1.txt"], d1)
674 os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
676 d1 = llfuse.listdir(os.path.join(mounttmp))
677 self.assertEqual(["file1.txt", "testdir"], sorted(d1))
678 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
679 self.assertEqual([], d1)
683 class FuseMvFileTest(MountTestBase):
685 collection = arvados.collection.Collection(api_client=self.api)
686 collection.save_new()
688 m = self.make_mount(fuse.CollectionDirectory)
690 m.new_collection(collection.api_response(), collection)
691 self.assertTrue(m.writable())
693 self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
696 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
697 assertRegex(self, collection2["manifest_text"],
698 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
700 self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
702 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
703 assertRegex(self, collection2["manifest_text"],
704 r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
707 def fuseRenameTestHelper(mounttmp):
708 class Test(unittest.TestCase):
710 os.mkdir(os.path.join(mounttmp, "testdir"))
712 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
713 f.write("Hello world!")
717 class FuseRenameTest(MountTestBase):
719 collection = arvados.collection.Collection(api_client=self.api)
720 collection.save_new()
722 m = self.make_mount(fuse.CollectionDirectory)
724 m.new_collection(collection.api_response(), collection)
725 self.assertTrue(m.writable())
727 self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
730 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
731 assertRegex(self, collection2["manifest_text"],
732 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
734 d1 = llfuse.listdir(os.path.join(self.mounttmp))
735 self.assertEqual(["testdir"], d1)
736 d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
737 self.assertEqual(["file1.txt"], d1)
739 os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
741 d1 = llfuse.listdir(os.path.join(self.mounttmp))
742 self.assertEqual(["testdir2"], sorted(d1))
743 d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
744 self.assertEqual(["file1.txt"], d1)
746 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
747 assertRegex(self, collection2["manifest_text"],
748 r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
751 class FuseUpdateFromEventTest(MountTestBase):
753 collection = arvados.collection.Collection(api_client=self.api)
754 collection.save_new()
756 m = self.make_mount(fuse.CollectionDirectory)
758 m.new_collection(collection.api_response(), collection)
760 self.operations.listen_for_events()
762 d1 = llfuse.listdir(os.path.join(self.mounttmp))
763 self.assertEqual([], sorted(d1))
765 with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
766 with collection2.open("file1.txt", "w") as f:
769 for attempt in AssertWithTimeout(10):
770 attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
773 class FuseDeleteProjectEventTest(MountTestBase):
776 aproject = self.api.groups().create(body={
778 "group_class": "project"
781 bproject = self.api.groups().create(body={
783 "group_class": "project",
784 "owner_uuid": aproject["uuid"]
787 self.make_mount(fuse.ProjectDirectory,
788 project_object=self.api.users().current().execute())
790 self.operations.listen_for_events()
792 d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
793 self.assertEqual(["bproject"], sorted(d1))
795 self.api.groups().delete(uuid=bproject["uuid"]).execute()
797 for attempt in AssertWithTimeout(10):
798 attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
801 def fuseFileConflictTestHelper(mounttmp, uuid, keeptmp, settings):
802 class Test(unittest.TestCase):
804 os.environ['KEEP_LOCAL_STORE'] = keeptmp
806 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
807 with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
808 with collection2.open("file1.txt", "w") as f2:
812 d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
813 self.assertEqual(len(d1), 2)
815 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
816 self.assertEqual(f.read(), "bar")
818 assertRegex(self, d1[1],
819 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
821 with open(os.path.join(mounttmp, d1[1]), "r") as f:
822 self.assertEqual(f.read(), "foo")
826 class FuseFileConflictTest(MountTestBase):
828 collection = arvados.collection.Collection(api_client=self.api)
829 collection.save_new()
831 m = self.make_mount(fuse.CollectionDirectory)
833 m.new_collection(collection.api_response(), collection)
835 d1 = llfuse.listdir(os.path.join(self.mounttmp))
836 self.assertEqual([], sorted(d1))
838 # See note in MountTestBase.setUp
839 self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
842 def fuseUnlinkOpenFileTest(mounttmp):
843 class Test(unittest.TestCase):
845 with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
848 d1 = llfuse.listdir(os.path.join(mounttmp))
849 self.assertEqual(["file1.txt"], sorted(d1))
851 os.remove(os.path.join(mounttmp, "file1.txt"))
853 d1 = llfuse.listdir(os.path.join(mounttmp))
854 self.assertEqual([], sorted(d1))
857 self.assertEqual(f.read(), "foo")
861 self.assertEqual(f.read(), "foobar")
865 class FuseUnlinkOpenFileTest(MountTestBase):
867 collection = arvados.collection.Collection(api_client=self.api)
868 collection.save_new()
870 m = self.make_mount(fuse.CollectionDirectory)
872 m.new_collection(collection.api_response(), collection)
874 # See note in MountTestBase.setUp
875 self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
877 self.assertEqual(collection.manifest_text(), "")
880 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
881 class Test(unittest.TestCase):
883 with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
884 f.write("Hello world!")
886 d1 = os.listdir(os.path.join(mounttmp, uuid1))
887 self.assertEqual(["file1.txt"], sorted(d1))
888 d1 = os.listdir(os.path.join(mounttmp, uuid2))
889 self.assertEqual([], sorted(d1))
893 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
894 class Test(unittest.TestCase):
896 os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
898 d1 = os.listdir(os.path.join(mounttmp, uuid1))
899 self.assertEqual([], sorted(d1))
900 d1 = os.listdir(os.path.join(mounttmp, uuid2))
901 self.assertEqual(["file2.txt"], sorted(d1))
905 class FuseMvFileBetweenCollectionsTest(MountTestBase):
907 collection1 = arvados.collection.Collection(api_client=self.api)
908 collection1.save_new()
910 collection2 = arvados.collection.Collection(api_client=self.api)
911 collection2.save_new()
913 m = self.make_mount(fuse.MagicDirectory)
915 # See note in MountTestBase.setUp
916 self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
917 collection1.manifest_locator(),
918 collection2.manifest_locator()))
923 assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
924 self.assertEqual(collection2.manifest_text(), "")
926 self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
927 collection1.manifest_locator(),
928 collection2.manifest_locator()))
933 self.assertEqual(collection1.manifest_text(), "")
934 assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
936 collection1.stop_threads()
937 collection2.stop_threads()
940 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
941 class Test(unittest.TestCase):
943 os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
944 with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
945 f.write("Hello world!")
947 d1 = os.listdir(os.path.join(mounttmp, uuid1))
948 self.assertEqual(["testdir"], sorted(d1))
949 d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
950 self.assertEqual(["file1.txt"], sorted(d1))
952 d1 = os.listdir(os.path.join(mounttmp, uuid2))
953 self.assertEqual([], sorted(d1))
958 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
959 class Test(unittest.TestCase):
961 os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
963 d1 = os.listdir(os.path.join(mounttmp, uuid1))
964 self.assertEqual([], sorted(d1))
966 d1 = os.listdir(os.path.join(mounttmp, uuid2))
967 self.assertEqual(["testdir2"], sorted(d1))
968 d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
969 self.assertEqual(["file1.txt"], sorted(d1))
971 with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
972 self.assertEqual(f.read(), "Hello world!")
976 class FuseMvDirBetweenCollectionsTest(MountTestBase):
978 collection1 = arvados.collection.Collection(api_client=self.api)
979 collection1.save_new()
981 collection2 = arvados.collection.Collection(api_client=self.api)
982 collection2.save_new()
984 m = self.make_mount(fuse.MagicDirectory)
986 # See note in MountTestBase.setUp
987 self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
988 collection1.manifest_locator(),
989 collection2.manifest_locator()))
994 assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
995 self.assertEqual(collection2.manifest_text(), "")
997 self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
998 collection1.manifest_locator(),
999 collection2.manifest_locator()))
1001 collection1.update()
1002 collection2.update()
1004 self.assertEqual(collection1.manifest_text(), "")
1005 assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
1007 collection1.stop_threads()
1008 collection2.stop_threads()
1010 def fuseProjectMkdirTestHelper1(mounttmp):
1011 class Test(unittest.TestCase):
1013 os.mkdir(os.path.join(mounttmp, "testcollection"))
1014 with self.assertRaises(OSError):
1015 os.mkdir(os.path.join(mounttmp, "testcollection"))
1018 def fuseProjectMkdirTestHelper2(mounttmp):
1019 class Test(unittest.TestCase):
1021 with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1022 f.write("Hello world!")
1023 with self.assertRaises(OSError):
1024 os.rmdir(os.path.join(mounttmp, "testcollection"))
1025 os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1026 with self.assertRaises(OSError):
1027 os.remove(os.path.join(mounttmp, "testcollection"))
1028 os.rmdir(os.path.join(mounttmp, "testcollection"))
1031 class FuseProjectMkdirRmdirTest(MountTestBase):
1033 self.make_mount(fuse.ProjectDirectory,
1034 project_object=self.api.users().current().execute())
1036 d1 = llfuse.listdir(self.mounttmp)
1037 self.assertNotIn('testcollection', d1)
1039 self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1041 d1 = llfuse.listdir(self.mounttmp)
1042 self.assertIn('testcollection', d1)
1044 self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1046 d1 = llfuse.listdir(self.mounttmp)
1047 self.assertNotIn('testcollection', d1)
1050 def fuseProjectMvTestHelper1(mounttmp):
1051 class Test(unittest.TestCase):
1053 d1 = llfuse.listdir(mounttmp)
1054 self.assertNotIn('testcollection', d1)
1056 os.mkdir(os.path.join(mounttmp, "testcollection"))
1058 d1 = llfuse.listdir(mounttmp)
1059 self.assertIn('testcollection', d1)
1061 with self.assertRaises(OSError):
1062 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1064 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1066 d1 = llfuse.listdir(mounttmp)
1067 self.assertNotIn('testcollection', d1)
1069 d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1070 self.assertIn('testcollection', d1)
1074 class FuseProjectMvTest(MountTestBase):
1076 self.make_mount(fuse.ProjectDirectory,
1077 project_object=self.api.users().current().execute())
1079 self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1082 def fuseFsyncTestHelper(mounttmp, k):
1083 class Test(unittest.TestCase):
1085 fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1091 class FuseFsyncTest(FuseMagicTest):
1093 self.make_mount(fuse.MagicDirectory)
1094 self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1097 class MagicDirApiError(FuseMagicTest):
1099 api = mock.MagicMock()
1100 api.keep.block_cache = mock.MagicMock(cache_max=1)
1101 super(MagicDirApiError, self).setUp(api=api)
1102 api.collections().get().execute.side_effect = iter([
1103 Exception('API fail'),
1105 "manifest_text": self.test_manifest,
1106 "portable_data_hash": self.test_manifest_pdh,
1109 api.keep.get.side_effect = Exception('Keep fail')
1112 with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1113 self.make_mount(fuse.MagicDirectory)
1115 self.operations.inodes.inode_cache.cap = 1
1116 self.operations.inodes.inode_cache.min_entries = 2
1118 with self.assertRaises(OSError):
1119 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1121 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1124 class SanitizeFilenameTest(MountTestBase):
1125 def test_sanitize_filename(self):
1126 pdir = fuse.ProjectDirectory(1, {}, self.api, 0, False, project_object=self.api.users().current().execute())
1145 for f in acceptable:
1146 self.assertEqual(f, pdir.sanitize_filename(f))
1147 for f in unacceptable:
1148 self.assertNotEqual(f, pdir.sanitize_filename(f))
1149 # The sanitized filename should be the same length, though.
1150 self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1152 self.assertEqual("_", pdir.sanitize_filename(""))
1153 self.assertEqual("_", pdir.sanitize_filename("."))
1154 self.assertEqual("__", pdir.sanitize_filename(".."))
1157 class FuseMagicTestPDHOnly(MountTestBase):
1158 def setUp(self, api=None):
1159 super(FuseMagicTestPDHOnly, self).setUp(api=api)
1161 cw = arvados.CollectionWriter()
1163 cw.start_new_file('thing1.txt')
1166 self.testcollection = cw.finish()
1167 self.test_manifest = cw.manifest_text()
1168 created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1169 self.testcollectionuuid = str(created['uuid'])
1171 def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1172 if skip_pdh_only is True:
1173 self.make_mount(fuse.MagicDirectory) # in this case, the default by_id applies
1175 self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1177 mount_ls = llfuse.listdir(self.mounttmp)
1178 self.assertIn('README', mount_ls)
1179 self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1180 arvados.util.uuid_pattern.match(fn)
1181 for fn in mount_ls),
1182 "new FUSE MagicDirectory lists Collection")
1184 # look up using pdh should succeed in all cases
1185 self.assertDirContents(self.testcollection, ['thing1.txt'])
1186 self.assertDirContents(os.path.join('by_id', self.testcollection),
1188 mount_ls = llfuse.listdir(self.mounttmp)
1189 self.assertIn('README', mount_ls)
1190 self.assertIn(self.testcollection, mount_ls)
1191 self.assertIn(self.testcollection,
1192 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1195 files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1197 for k, v in viewitems(files):
1198 with open(os.path.join(self.mounttmp, k), 'rb') as f:
1199 self.assertEqual(v, f.read().decode())
1201 # look up using uuid should fail when pdh_only is set
1202 if pdh_only is True:
1203 with self.assertRaises(OSError):
1204 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1207 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1210 def test_with_pdh_only_true(self):
1211 self.verify_pdh_only(pdh_only=True)
1213 def test_with_pdh_only_false(self):
1214 self.verify_pdh_only(pdh_only=False)
1216 def test_with_default_by_id(self):
1217 self.verify_pdh_only(skip_pdh_only=True)
1220 class SlashSubstitutionTest(IntegrationTest):
1223 '--mount-home', 'zzz',
1227 super(SlashSubstitutionTest, self).setUp()
1228 self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1229 self.api.config = lambda: {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1230 self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1231 self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1232 self.fusename = 'foo[SLASH]bar[SLASH]baz'
1234 @IntegrationTest.mount(argv=mnt_args)
1235 @mock.patch('arvados.util.get_config_once')
1236 def test_slash_substitution_before_listing(self, get_config_once):
1237 get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1238 self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1239 self.checkContents()
1241 def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1242 with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1244 with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1247 @IntegrationTest.mount(argv=mnt_args)
1248 @mock.patch('arvados.util.get_config_once')
1249 def test_slash_substitution_after_listing(self, get_config_once):
1250 get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1251 self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1252 self.checkContents()
1254 def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1255 with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1258 with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1261 def checkContents(self):
1262 self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1263 self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1265 @IntegrationTest.mount(argv=mnt_args)
1266 @mock.patch('arvados.util.get_config_once')
1267 def test_slash_substitution_conflict(self, get_config_once):
1268 self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1269 get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1270 self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1271 self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1272 # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1273 self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1275 def _test_slash_substitution_conflict(self, tmpdir, fusename):
1276 with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1279 class StorageClassesTest(IntegrationTest):
1282 '--mount-home', 'homedir',
1286 super(StorageClassesTest, self).setUp()
1287 self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1289 @IntegrationTest.mount(argv=mnt_args)
1290 def test_collection_default_storage_classes(self):
1291 coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
1292 self.api.collections().create(body={'name':'a_collection'}).execute()
1293 self.pool_test(coll_path)
1295 def _test_collection_default_storage_classes(self, coll):
1296 self.assertEqual(storage_classes_desired(coll), ['default'])
1298 @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
1299 def test_collection_custom_storage_classes(self):
1300 coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
1302 self.pool_test(coll_path)
1304 def _test_collection_custom_storage_classes(self, coll):
1305 self.assertEqual(storage_classes_desired(coll), ['foo'])
1307 def _readonlyCollectionTestHelper(mounttmp):
1308 f = open(os.path.join(mounttmp, 'thing1.txt'), 'rt')
1309 # Testing that close() doesn't raise an error.
1312 class ReadonlyCollectionTest(MountTestBase):
1314 super(ReadonlyCollectionTest, self).setUp()
1315 cw = arvados.collection.Collection()
1316 with cw.open('thing1.txt', 'wt') as f:
1318 cw.save_new(owner_uuid=run_test_server.fixture("groups")["aproject"]["uuid"])
1319 self.testcollection = cw.api_response()
1322 settings = arvados.config.settings().copy()
1323 settings["ARVADOS_API_TOKEN"] = run_test_server.fixture("api_client_authorizations")["project_viewer"]["api_token"]
1324 self.api = arvados.safeapi.ThreadSafeApiCache(settings)
1325 self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection, enable_write=False)
1327 self.pool.apply(_readonlyCollectionTestHelper, (self.mounttmp,))