1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: AGPL-3.0
15 import arvados_fuse as fuse
16 import run_test_server
18 from mount_test_base import MountTestBase
20 logger = logging.getLogger('arvados.arv-mount')
23 class AssertWithTimeout(object):
24 """Allow some time for an assertion to pass."""
26 def __init__(self, timeout=0):
27 self.timeout = timeout
30 self.deadline = time.time() + self.timeout
39 def attempt(self, fn, *args, **kwargs):
42 except AssertionError:
43 if time.time() > self.deadline:
50 class FuseMountTest(MountTestBase):
52 super(FuseMountTest, self).setUp()
54 cw = arvados.CollectionWriter()
56 cw.start_new_file('thing1.txt')
58 cw.start_new_file('thing2.txt')
61 cw.start_new_stream('dir1')
62 cw.start_new_file('thing3.txt')
64 cw.start_new_file('thing4.txt')
67 cw.start_new_stream('dir2')
68 cw.start_new_file('thing5.txt')
70 cw.start_new_file('thing6.txt')
73 cw.start_new_stream('dir2/dir3')
74 cw.start_new_file('thing7.txt')
77 cw.start_new_file('thing8.txt')
80 cw.start_new_stream('edgecases')
81 for f in ":/.../-/*/\x01\\/ ".split("/"):
85 for f in ":/.../-/*/\x01\\/ ".split("/"):
86 cw.start_new_stream('edgecases/dirs/' + f)
87 cw.start_new_file('x/x')
90 self.testcollection = cw.finish()
91 self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
94 self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
96 self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
97 'edgecases', 'dir1', 'dir2'])
98 self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
99 self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
100 self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
101 self.assertDirContents('edgecases',
102 "dirs/:/.../-/*/\x01\\/ ".split("/"))
103 self.assertDirContents('edgecases/dirs',
104 ":/.../-/*/\x01\\/ ".split("/"))
106 files = {'thing1.txt': 'data 1',
107 'thing2.txt': 'data 2',
108 'dir1/thing3.txt': 'data 3',
109 'dir1/thing4.txt': 'data 4',
110 'dir2/thing5.txt': 'data 5',
111 'dir2/thing6.txt': 'data 6',
112 'dir2/dir3/thing7.txt': 'data 7',
113 'dir2/dir3/thing8.txt': 'data 8'}
115 for k, v in files.items():
116 with open(os.path.join(self.mounttmp, k)) as f:
117 self.assertEqual(v, f.read())
120 class FuseMagicTest(MountTestBase):
121 def setUp(self, api=None):
122 super(FuseMagicTest, self).setUp(api=api)
124 self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
125 self.non_project_group = run_test_server.fixture('groups')['public']['uuid']
126 self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
128 cw = arvados.CollectionWriter()
130 cw.start_new_file('thing1.txt')
133 self.testcollection = cw.finish()
134 self.test_manifest = cw.manifest_text()
135 coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
136 self.test_manifest_pdh = coll['portable_data_hash']
139 self.make_mount(fuse.MagicDirectory)
141 mount_ls = llfuse.listdir(self.mounttmp)
142 self.assertIn('README', mount_ls)
143 self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
144 arvados.util.uuid_pattern.match(fn)
146 "new FUSE MagicDirectory has no collections or projects")
147 self.assertDirContents(self.testcollection, ['thing1.txt'])
148 self.assertDirContents(os.path.join('by_id', self.testcollection),
150 self.assertIn(self.collection_in_test_project,
151 llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
152 self.assertIn(self.collection_in_test_project,
153 llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
155 mount_ls = llfuse.listdir(self.mounttmp)
156 self.assertIn('README', mount_ls)
157 self.assertIn(self.testcollection, mount_ls)
158 self.assertIn(self.testcollection,
159 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
160 self.assertIn(self.test_project, mount_ls)
161 self.assertIn(self.test_project,
162 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
164 with self.assertRaises(OSError):
165 llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
168 files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
170 for k, v in files.items():
171 with open(os.path.join(self.mounttmp, k)) as f:
172 self.assertEqual(v, f.read())
175 class FuseTagsTest(MountTestBase):
177 self.make_mount(fuse.TagsDirectory)
179 d1 = llfuse.listdir(self.mounttmp)
181 self.assertEqual(['foo_tag'], d1)
183 d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
185 self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
187 d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
189 self.assertEqual(['foo'], d3)
192 class FuseTagsUpdateTest(MountTestBase):
193 def tag_collection(self, coll_uuid, tag_name):
194 return self.api.links().create(
195 body={'link': {'head_uuid': coll_uuid,
201 self.make_mount(fuse.TagsDirectory, poll_time=1)
203 self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
205 bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
206 self.tag_collection(bar_uuid, 'fuse_test_tag')
207 for attempt in AssertWithTimeout(10):
208 attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
209 self.assertDirContents('fuse_test_tag', [bar_uuid])
211 baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
212 l = self.tag_collection(baz_uuid, 'fuse_test_tag')
213 for attempt in AssertWithTimeout(10):
214 attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
216 self.api.links().delete(uuid=l['uuid']).execute()
217 for attempt in AssertWithTimeout(10):
218 attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
221 def fuseSharedTestHelper(mounttmp):
222 class Test(unittest.TestCase):
224 # Double check that we can open and read objects in this folder as a file,
225 # and that its contents are what we expect.
226 baz_path = os.path.join(
230 'collection in FUSE project',
232 with open(baz_path) as f:
233 self.assertEqual("baz", f.read())
235 # check mtime on collection
236 st = os.stat(baz_path)
238 mtime = st.st_mtime_ns / 1000000000
239 except AttributeError:
241 self.assertEqual(mtime, 1391448174)
243 # shared_dirs is a list of the directories exposed
244 # by fuse.SharedDirectory (i.e. any object visible
245 # to the current user)
246 shared_dirs = llfuse.listdir(mounttmp)
248 self.assertIn('FUSE User', shared_dirs)
250 # fuse_user_objs is a list of the objects owned by the FUSE
251 # test user (which present as files in the 'FUSE User'
253 fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
254 fuse_user_objs.sort()
255 self.assertEqual(['FUSE Test Project', # project owned by user
256 'collection #1 owned by FUSE', # collection owned by user
257 'collection #2 owned by FUSE' # collection owned by user
260 # test_proj_files is a list of the files in the FUSE Test Project.
261 test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
262 test_proj_files.sort()
263 self.assertEqual(['collection in FUSE project'
269 class FuseSharedTest(MountTestBase):
271 self.make_mount(fuse.SharedDirectory,
272 exclude=self.api.users().current().execute()['uuid'])
273 keep = arvados.keep.KeepClient()
276 self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
279 class FuseHomeTest(MountTestBase):
281 self.make_mount(fuse.ProjectDirectory,
282 project_object=self.api.users().current().execute())
284 d1 = llfuse.listdir(self.mounttmp)
285 self.assertIn('Unrestricted public data', d1)
287 d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
288 public_project = run_test_server.fixture('groups')[
289 'anonymously_accessible_project']
292 for name, item in run_test_server.fixture('collections').iteritems():
293 if 'name' not in item:
295 elif item['owner_uuid'] == public_project['uuid']:
296 self.assertIn(item['name'], d2)
299 # Artificial assumption here: there is no public
300 # collection fixture with the same name as a
301 # non-public collection.
302 self.assertNotIn(item['name'], d2)
304 self.assertNotEqual(0, found_in)
305 self.assertNotEqual(0, found_not_in)
307 d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
308 self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
311 def fuseModifyFileTestHelperReadStartContents(mounttmp):
312 class Test(unittest.TestCase):
314 d1 = llfuse.listdir(mounttmp)
315 self.assertEqual(["file1.txt"], d1)
316 with open(os.path.join(mounttmp, "file1.txt")) as f:
317 self.assertEqual("blub", f.read())
320 def fuseModifyFileTestHelperReadEndContents(mounttmp):
321 class Test(unittest.TestCase):
323 d1 = llfuse.listdir(mounttmp)
324 self.assertEqual(["file1.txt"], d1)
325 with open(os.path.join(mounttmp, "file1.txt")) as f:
326 self.assertEqual("plnp", f.read())
329 class FuseModifyFileTest(MountTestBase):
331 collection = arvados.collection.Collection(api_client=self.api)
332 with collection.open("file1.txt", "w") as f:
335 collection.save_new()
337 m = self.make_mount(fuse.CollectionDirectory)
339 m.new_collection(collection.api_response(), collection)
341 self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
343 with collection.open("file1.txt", "w") as f:
346 self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
349 class FuseAddFileToCollectionTest(MountTestBase):
351 collection = arvados.collection.Collection(api_client=self.api)
352 with collection.open("file1.txt", "w") as f:
355 collection.save_new()
357 m = self.make_mount(fuse.CollectionDirectory)
359 m.new_collection(collection.api_response(), collection)
361 d1 = llfuse.listdir(self.mounttmp)
362 self.assertEqual(["file1.txt"], d1)
364 with collection.open("file2.txt", "w") as f:
367 d1 = llfuse.listdir(self.mounttmp)
368 self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
371 class FuseRemoveFileFromCollectionTest(MountTestBase):
373 collection = arvados.collection.Collection(api_client=self.api)
374 with collection.open("file1.txt", "w") as f:
377 with collection.open("file2.txt", "w") as f:
380 collection.save_new()
382 m = self.make_mount(fuse.CollectionDirectory)
384 m.new_collection(collection.api_response(), collection)
386 d1 = llfuse.listdir(self.mounttmp)
387 self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
389 collection.remove("file2.txt")
391 d1 = llfuse.listdir(self.mounttmp)
392 self.assertEqual(["file1.txt"], d1)
395 def fuseCreateFileTestHelper(mounttmp):
396 class Test(unittest.TestCase):
398 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
402 class FuseCreateFileTest(MountTestBase):
404 collection = arvados.collection.Collection(api_client=self.api)
405 collection.save_new()
407 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
408 self.assertEqual(collection2["manifest_text"], "")
410 collection.save_new()
412 m = self.make_mount(fuse.CollectionDirectory)
414 m.new_collection(collection.api_response(), collection)
415 self.assertTrue(m.writable())
417 self.assertNotIn("file1.txt", collection)
419 self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
421 self.assertIn("file1.txt", collection)
423 d1 = llfuse.listdir(self.mounttmp)
424 self.assertEqual(["file1.txt"], d1)
426 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
427 self.assertRegexpMatches(collection2["manifest_text"],
428 r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
431 def fuseWriteFileTestHelperWriteFile(mounttmp):
432 class Test(unittest.TestCase):
434 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
435 f.write("Hello world!")
438 def fuseWriteFileTestHelperReadFile(mounttmp):
439 class Test(unittest.TestCase):
441 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
442 self.assertEqual(f.read(), "Hello world!")
445 class FuseWriteFileTest(MountTestBase):
447 collection = arvados.collection.Collection(api_client=self.api)
448 collection.save_new()
450 m = self.make_mount(fuse.CollectionDirectory)
452 m.new_collection(collection.api_response(), collection)
453 self.assertTrue(m.writable())
455 self.assertNotIn("file1.txt", collection)
457 self.assertEqual(0, self.operations.write_counter.get())
458 self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
459 self.assertEqual(12, self.operations.write_counter.get())
461 with collection.open("file1.txt") as f:
462 self.assertEqual(f.read(), "Hello world!")
464 self.assertEqual(0, self.operations.read_counter.get())
465 self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
466 self.assertEqual(12, self.operations.read_counter.get())
468 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
469 self.assertRegexpMatches(collection2["manifest_text"],
470 r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
473 def fuseUpdateFileTestHelper(mounttmp):
474 class Test(unittest.TestCase):
476 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
477 f.write("Hello world!")
479 with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
481 self.assertEqual(fr, "Hello world!")
483 f.write("Hola mundo!")
486 self.assertEqual(fr, "Hola mundo!!")
488 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
489 self.assertEqual(f.read(), "Hola mundo!!")
493 class FuseUpdateFileTest(MountTestBase):
495 collection = arvados.collection.Collection(api_client=self.api)
496 collection.save_new()
498 m = self.make_mount(fuse.CollectionDirectory)
500 m.new_collection(collection.api_response(), collection)
501 self.assertTrue(m.writable())
503 # See note in MountTestBase.setUp
504 self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
506 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
507 self.assertRegexpMatches(collection2["manifest_text"],
508 r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
511 def fuseMkdirTestHelper(mounttmp):
512 class Test(unittest.TestCase):
514 with self.assertRaises(IOError):
515 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
516 f.write("Hello world!")
518 os.mkdir(os.path.join(mounttmp, "testdir"))
520 with self.assertRaises(OSError):
521 os.mkdir(os.path.join(mounttmp, "testdir"))
523 d1 = llfuse.listdir(mounttmp)
524 self.assertEqual(["testdir"], d1)
526 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
527 f.write("Hello world!")
529 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
530 self.assertEqual(["file1.txt"], d1)
534 class FuseMkdirTest(MountTestBase):
536 collection = arvados.collection.Collection(api_client=self.api)
537 collection.save_new()
539 m = self.make_mount(fuse.CollectionDirectory)
541 m.new_collection(collection.api_response(), collection)
542 self.assertTrue(m.writable())
544 self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
546 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
547 self.assertRegexpMatches(collection2["manifest_text"],
548 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
551 def fuseRmTestHelperWriteFile(mounttmp):
552 class Test(unittest.TestCase):
554 os.mkdir(os.path.join(mounttmp, "testdir"))
556 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
557 f.write("Hello world!")
561 def fuseRmTestHelperDeleteFile(mounttmp):
562 class Test(unittest.TestCase):
564 # Can't delete because it's not empty
565 with self.assertRaises(OSError):
566 os.rmdir(os.path.join(mounttmp, "testdir"))
568 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
569 self.assertEqual(["file1.txt"], d1)
572 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
574 # Make sure it's empty
575 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
576 self.assertEqual([], d1)
578 # Try to delete it again
579 with self.assertRaises(OSError):
580 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
584 def fuseRmTestHelperRmdir(mounttmp):
585 class Test(unittest.TestCase):
587 # Should be able to delete now that it is empty
588 os.rmdir(os.path.join(mounttmp, "testdir"))
590 # Make sure it's empty
591 d1 = llfuse.listdir(os.path.join(mounttmp))
592 self.assertEqual([], d1)
594 # Try to delete it again
595 with self.assertRaises(OSError):
596 os.rmdir(os.path.join(mounttmp, "testdir"))
600 class FuseRmTest(MountTestBase):
602 collection = arvados.collection.Collection(api_client=self.api)
603 collection.save_new()
605 m = self.make_mount(fuse.CollectionDirectory)
607 m.new_collection(collection.api_response(), collection)
608 self.assertTrue(m.writable())
610 self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
613 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
614 self.assertRegexpMatches(collection2["manifest_text"],
615 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
616 self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
618 # Can't have empty directories :-( so manifest will be empty.
619 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
620 self.assertEqual(collection2["manifest_text"], "")
622 self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
624 # manifest should be empty now.
625 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
626 self.assertEqual(collection2["manifest_text"], "")
629 def fuseMvFileTestHelperWriteFile(mounttmp):
630 class Test(unittest.TestCase):
632 os.mkdir(os.path.join(mounttmp, "testdir"))
634 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
635 f.write("Hello world!")
639 def fuseMvFileTestHelperMoveFile(mounttmp):
640 class Test(unittest.TestCase):
642 d1 = llfuse.listdir(os.path.join(mounttmp))
643 self.assertEqual(["testdir"], d1)
644 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
645 self.assertEqual(["file1.txt"], d1)
647 os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
649 d1 = llfuse.listdir(os.path.join(mounttmp))
650 self.assertEqual(["file1.txt", "testdir"], sorted(d1))
651 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
652 self.assertEqual([], d1)
656 class FuseMvFileTest(MountTestBase):
658 collection = arvados.collection.Collection(api_client=self.api)
659 collection.save_new()
661 m = self.make_mount(fuse.CollectionDirectory)
663 m.new_collection(collection.api_response(), collection)
664 self.assertTrue(m.writable())
666 self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
669 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
670 self.assertRegexpMatches(collection2["manifest_text"],
671 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
673 self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
675 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
676 self.assertRegexpMatches(collection2["manifest_text"],
677 r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
680 def fuseRenameTestHelper(mounttmp):
681 class Test(unittest.TestCase):
683 os.mkdir(os.path.join(mounttmp, "testdir"))
685 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
686 f.write("Hello world!")
690 class FuseRenameTest(MountTestBase):
692 collection = arvados.collection.Collection(api_client=self.api)
693 collection.save_new()
695 m = self.make_mount(fuse.CollectionDirectory)
697 m.new_collection(collection.api_response(), collection)
698 self.assertTrue(m.writable())
700 self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
703 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
704 self.assertRegexpMatches(collection2["manifest_text"],
705 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
707 d1 = llfuse.listdir(os.path.join(self.mounttmp))
708 self.assertEqual(["testdir"], d1)
709 d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
710 self.assertEqual(["file1.txt"], d1)
712 os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
714 d1 = llfuse.listdir(os.path.join(self.mounttmp))
715 self.assertEqual(["testdir2"], sorted(d1))
716 d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
717 self.assertEqual(["file1.txt"], d1)
719 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
720 self.assertRegexpMatches(collection2["manifest_text"],
721 r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
724 class FuseUpdateFromEventTest(MountTestBase):
726 collection = arvados.collection.Collection(api_client=self.api)
727 collection.save_new()
729 m = self.make_mount(fuse.CollectionDirectory)
731 m.new_collection(collection.api_response(), collection)
733 self.operations.listen_for_events()
735 d1 = llfuse.listdir(os.path.join(self.mounttmp))
736 self.assertEqual([], sorted(d1))
738 with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
739 with collection2.open("file1.txt", "w") as f:
742 for attempt in AssertWithTimeout(10):
743 attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
746 class FuseDeleteProjectEventTest(MountTestBase):
749 aproject = self.api.groups().create(body={
751 "group_class": "project"
754 bproject = self.api.groups().create(body={
756 "group_class": "project",
757 "owner_uuid": aproject["uuid"]
760 self.make_mount(fuse.ProjectDirectory,
761 project_object=self.api.users().current().execute())
763 self.operations.listen_for_events()
765 d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
766 self.assertEqual(["bproject"], sorted(d1))
768 self.api.groups().delete(uuid=bproject["uuid"]).execute()
770 for attempt in AssertWithTimeout(10):
771 attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
774 def fuseFileConflictTestHelper(mounttmp):
775 class Test(unittest.TestCase):
777 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
780 d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
781 self.assertEqual(len(d1), 2)
783 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
784 self.assertEqual(f.read(), "bar")
786 self.assertRegexpMatches(d1[1],
787 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
789 with open(os.path.join(mounttmp, d1[1]), "r") as f:
790 self.assertEqual(f.read(), "foo")
794 class FuseFileConflictTest(MountTestBase):
796 collection = arvados.collection.Collection(api_client=self.api)
797 collection.save_new()
799 m = self.make_mount(fuse.CollectionDirectory)
801 m.new_collection(collection.api_response(), collection)
803 d1 = llfuse.listdir(os.path.join(self.mounttmp))
804 self.assertEqual([], sorted(d1))
806 with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
807 with collection2.open("file1.txt", "w") as f:
810 # See note in MountTestBase.setUp
811 self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
814 def fuseUnlinkOpenFileTest(mounttmp):
815 class Test(unittest.TestCase):
817 with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
820 d1 = llfuse.listdir(os.path.join(mounttmp))
821 self.assertEqual(["file1.txt"], sorted(d1))
823 os.remove(os.path.join(mounttmp, "file1.txt"))
825 d1 = llfuse.listdir(os.path.join(mounttmp))
826 self.assertEqual([], sorted(d1))
829 self.assertEqual(f.read(), "foo")
833 self.assertEqual(f.read(), "foobar")
837 class FuseUnlinkOpenFileTest(MountTestBase):
839 collection = arvados.collection.Collection(api_client=self.api)
840 collection.save_new()
842 m = self.make_mount(fuse.CollectionDirectory)
844 m.new_collection(collection.api_response(), collection)
846 # See note in MountTestBase.setUp
847 self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
849 self.assertEqual(collection.manifest_text(), "")
852 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
853 class Test(unittest.TestCase):
855 with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
856 f.write("Hello world!")
858 d1 = os.listdir(os.path.join(mounttmp, uuid1))
859 self.assertEqual(["file1.txt"], sorted(d1))
860 d1 = os.listdir(os.path.join(mounttmp, uuid2))
861 self.assertEqual([], sorted(d1))
865 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
866 class Test(unittest.TestCase):
868 os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
870 d1 = os.listdir(os.path.join(mounttmp, uuid1))
871 self.assertEqual([], sorted(d1))
872 d1 = os.listdir(os.path.join(mounttmp, uuid2))
873 self.assertEqual(["file2.txt"], sorted(d1))
877 class FuseMvFileBetweenCollectionsTest(MountTestBase):
879 collection1 = arvados.collection.Collection(api_client=self.api)
880 collection1.save_new()
882 collection2 = arvados.collection.Collection(api_client=self.api)
883 collection2.save_new()
885 m = self.make_mount(fuse.MagicDirectory)
887 # See note in MountTestBase.setUp
888 self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
889 collection1.manifest_locator(),
890 collection2.manifest_locator()))
895 self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
896 self.assertEqual(collection2.manifest_text(), "")
898 self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
899 collection1.manifest_locator(),
900 collection2.manifest_locator()))
905 self.assertEqual(collection1.manifest_text(), "")
906 self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
908 collection1.stop_threads()
909 collection2.stop_threads()
912 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
913 class Test(unittest.TestCase):
915 os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
916 with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
917 f.write("Hello world!")
919 d1 = os.listdir(os.path.join(mounttmp, uuid1))
920 self.assertEqual(["testdir"], sorted(d1))
921 d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
922 self.assertEqual(["file1.txt"], sorted(d1))
924 d1 = os.listdir(os.path.join(mounttmp, uuid2))
925 self.assertEqual([], sorted(d1))
930 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
931 class Test(unittest.TestCase):
933 os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
935 d1 = os.listdir(os.path.join(mounttmp, uuid1))
936 self.assertEqual([], sorted(d1))
938 d1 = os.listdir(os.path.join(mounttmp, uuid2))
939 self.assertEqual(["testdir2"], sorted(d1))
940 d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
941 self.assertEqual(["file1.txt"], sorted(d1))
943 with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
944 self.assertEqual(f.read(), "Hello world!")
948 class FuseMvDirBetweenCollectionsTest(MountTestBase):
950 collection1 = arvados.collection.Collection(api_client=self.api)
951 collection1.save_new()
953 collection2 = arvados.collection.Collection(api_client=self.api)
954 collection2.save_new()
956 m = self.make_mount(fuse.MagicDirectory)
958 # See note in MountTestBase.setUp
959 self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
960 collection1.manifest_locator(),
961 collection2.manifest_locator()))
966 self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
967 self.assertEqual(collection2.manifest_text(), "")
969 self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
970 collection1.manifest_locator(),
971 collection2.manifest_locator()))
976 self.assertEqual(collection1.manifest_text(), "")
977 self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
979 collection1.stop_threads()
980 collection2.stop_threads()
982 def fuseProjectMkdirTestHelper1(mounttmp):
983 class Test(unittest.TestCase):
985 os.mkdir(os.path.join(mounttmp, "testcollection"))
986 with self.assertRaises(OSError):
987 os.mkdir(os.path.join(mounttmp, "testcollection"))
990 def fuseProjectMkdirTestHelper2(mounttmp):
991 class Test(unittest.TestCase):
993 with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
994 f.write("Hello world!")
995 with self.assertRaises(OSError):
996 os.rmdir(os.path.join(mounttmp, "testcollection"))
997 os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
998 with self.assertRaises(OSError):
999 os.remove(os.path.join(mounttmp, "testcollection"))
1000 os.rmdir(os.path.join(mounttmp, "testcollection"))
1003 class FuseProjectMkdirRmdirTest(MountTestBase):
1005 self.make_mount(fuse.ProjectDirectory,
1006 project_object=self.api.users().current().execute())
1008 d1 = llfuse.listdir(self.mounttmp)
1009 self.assertNotIn('testcollection', d1)
1011 self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1013 d1 = llfuse.listdir(self.mounttmp)
1014 self.assertIn('testcollection', d1)
1016 self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1018 d1 = llfuse.listdir(self.mounttmp)
1019 self.assertNotIn('testcollection', d1)
1022 def fuseProjectMvTestHelper1(mounttmp):
1023 class Test(unittest.TestCase):
1025 d1 = llfuse.listdir(mounttmp)
1026 self.assertNotIn('testcollection', d1)
1028 os.mkdir(os.path.join(mounttmp, "testcollection"))
1030 d1 = llfuse.listdir(mounttmp)
1031 self.assertIn('testcollection', d1)
1033 with self.assertRaises(OSError):
1034 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1036 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1038 d1 = llfuse.listdir(mounttmp)
1039 self.assertNotIn('testcollection', d1)
1041 d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1042 self.assertIn('testcollection', d1)
1046 class FuseProjectMvTest(MountTestBase):
1048 self.make_mount(fuse.ProjectDirectory,
1049 project_object=self.api.users().current().execute())
1051 self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1054 def fuseFsyncTestHelper(mounttmp, k):
1055 class Test(unittest.TestCase):
1057 fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1063 class FuseFsyncTest(FuseMagicTest):
1065 self.make_mount(fuse.MagicDirectory)
1066 self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1069 class MagicDirApiError(FuseMagicTest):
1071 api = mock.MagicMock()
1072 super(MagicDirApiError, self).setUp(api=api)
1073 api.collections().get().execute.side_effect = iter([
1074 Exception('API fail'),
1076 "manifest_text": self.test_manifest,
1077 "portable_data_hash": self.test_manifest_pdh,
1080 api.keep.get.side_effect = Exception('Keep fail')
1083 self.make_mount(fuse.MagicDirectory)
1085 self.operations.inodes.inode_cache.cap = 1
1086 self.operations.inodes.inode_cache.min_entries = 2
1088 with self.assertRaises(OSError):
1089 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1091 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1094 class FuseUnitTest(unittest.TestCase):
1095 def test_sanitize_filename(self):
1114 for f in acceptable:
1115 self.assertEqual(f, fuse.sanitize_filename(f))
1116 for f in unacceptable:
1117 self.assertNotEqual(f, fuse.sanitize_filename(f))
1118 # The sanitized filename should be the same length, though.
1119 self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1121 self.assertEqual("_", fuse.sanitize_filename(""))
1122 self.assertEqual("_", fuse.sanitize_filename("."))
1123 self.assertEqual("__", fuse.sanitize_filename(".."))
1126 class FuseMagicTestPDHOnly(MountTestBase):
1127 def setUp(self, api=None):
1128 super(FuseMagicTestPDHOnly, self).setUp(api=api)
1130 cw = arvados.CollectionWriter()
1132 cw.start_new_file('thing1.txt')
1135 self.testcollection = cw.finish()
1136 self.test_manifest = cw.manifest_text()
1137 created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1138 self.testcollectionuuid = str(created['uuid'])
1140 def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1141 if skip_pdh_only is True:
1142 self.make_mount(fuse.MagicDirectory) # in this case, the default by_id applies
1144 self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1146 mount_ls = llfuse.listdir(self.mounttmp)
1147 self.assertIn('README', mount_ls)
1148 self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1149 arvados.util.uuid_pattern.match(fn)
1150 for fn in mount_ls),
1151 "new FUSE MagicDirectory lists Collection")
1153 # look up using pdh should succeed in all cases
1154 self.assertDirContents(self.testcollection, ['thing1.txt'])
1155 self.assertDirContents(os.path.join('by_id', self.testcollection),
1157 mount_ls = llfuse.listdir(self.mounttmp)
1158 self.assertIn('README', mount_ls)
1159 self.assertIn(self.testcollection, mount_ls)
1160 self.assertIn(self.testcollection,
1161 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1164 files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1166 for k, v in files.items():
1167 with open(os.path.join(self.mounttmp, k)) as f:
1168 self.assertEqual(v, f.read())
1170 # look up using uuid should fail when pdh_only is set
1171 if pdh_only is True:
1172 with self.assertRaises(OSError):
1173 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1176 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1179 def test_with_pdh_only_true(self):
1180 self.verify_pdh_only(pdh_only=True)
1182 def test_with_pdh_only_false(self):
1183 self.verify_pdh_only(pdh_only=False)
1185 def test_with_default_by_id(self):
1186 self.verify_pdh_only(skip_pdh_only=True)