1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: AGPL-3.0
15 import arvados_fuse as fuse
16 import run_test_server
18 from mount_test_base import MountTestBase
20 logger = logging.getLogger('arvados.arv-mount')
23 class AssertWithTimeout(object):
24 """Allow some time for an assertion to pass."""
26 def __init__(self, timeout=0):
27 self.timeout = timeout
30 self.deadline = time.time() + self.timeout
39 def attempt(self, fn, *args, **kwargs):
42 except AssertionError:
43 if time.time() > self.deadline:
50 class FuseMountTest(MountTestBase):
52 super(FuseMountTest, self).setUp()
54 cw = arvados.CollectionWriter()
56 cw.start_new_file('thing1.txt')
58 cw.start_new_file('thing2.txt')
61 cw.start_new_stream('dir1')
62 cw.start_new_file('thing3.txt')
64 cw.start_new_file('thing4.txt')
67 cw.start_new_stream('dir2')
68 cw.start_new_file('thing5.txt')
70 cw.start_new_file('thing6.txt')
73 cw.start_new_stream('dir2/dir3')
74 cw.start_new_file('thing7.txt')
77 cw.start_new_file('thing8.txt')
80 cw.start_new_stream('edgecases')
81 for f in ":/.../-/*/\x01\\/ ".split("/"):
85 for f in ":/.../-/*/\x01\\/ ".split("/"):
86 cw.start_new_stream('edgecases/dirs/' + f)
87 cw.start_new_file('x/x')
90 self.testcollection = cw.finish()
91 self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
94 self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
96 self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
97 'edgecases', 'dir1', 'dir2'])
98 self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
99 self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
100 self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
101 self.assertDirContents('edgecases',
102 "dirs/:/.../-/*/\x01\\/ ".split("/"))
103 self.assertDirContents('edgecases/dirs',
104 ":/.../-/*/\x01\\/ ".split("/"))
106 files = {'thing1.txt': 'data 1',
107 'thing2.txt': 'data 2',
108 'dir1/thing3.txt': 'data 3',
109 'dir1/thing4.txt': 'data 4',
110 'dir2/thing5.txt': 'data 5',
111 'dir2/thing6.txt': 'data 6',
112 'dir2/dir3/thing7.txt': 'data 7',
113 'dir2/dir3/thing8.txt': 'data 8'}
115 for k, v in files.items():
116 with open(os.path.join(self.mounttmp, k)) as f:
117 self.assertEqual(v, f.read())
120 class FuseNoAPITest(MountTestBase):
122 super(FuseNoAPITest, self).setUp()
123 keep = arvados.keep.KeepClient(local_store=self.keeptmp)
124 self.file_data = "API-free text\n"
125 self.file_loc = keep.put(self.file_data)
126 self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
127 self.file_loc, len(self.file_data)))
130 self.make_mount(fuse.MagicDirectory)
131 self.assertDirContents(self.coll_loc, ['api-free.txt'])
132 with open(os.path.join(
133 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
134 actual = keep_file.read(-1)
135 self.assertEqual(self.file_data, actual)
138 class FuseMagicTest(MountTestBase):
139 def setUp(self, api=None):
140 super(FuseMagicTest, self).setUp(api=api)
142 cw = arvados.CollectionWriter()
144 cw.start_new_file('thing1.txt')
147 self.testcollection = cw.finish()
148 self.test_manifest = cw.manifest_text()
149 self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
152 self.make_mount(fuse.MagicDirectory)
154 mount_ls = llfuse.listdir(self.mounttmp)
155 self.assertIn('README', mount_ls)
156 self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
157 arvados.util.uuid_pattern.match(fn)
159 "new FUSE MagicDirectory lists Collection")
160 self.assertDirContents(self.testcollection, ['thing1.txt'])
161 self.assertDirContents(os.path.join('by_id', self.testcollection),
163 mount_ls = llfuse.listdir(self.mounttmp)
164 self.assertIn('README', mount_ls)
165 self.assertIn(self.testcollection, mount_ls)
166 self.assertIn(self.testcollection,
167 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
170 files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
172 for k, v in files.items():
173 with open(os.path.join(self.mounttmp, k)) as f:
174 self.assertEqual(v, f.read())
177 class FuseTagsTest(MountTestBase):
179 self.make_mount(fuse.TagsDirectory)
181 d1 = llfuse.listdir(self.mounttmp)
183 self.assertEqual(['foo_tag'], d1)
185 d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
187 self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
189 d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
191 self.assertEqual(['foo'], d3)
194 class FuseTagsUpdateTest(MountTestBase):
195 def tag_collection(self, coll_uuid, tag_name):
196 return self.api.links().create(
197 body={'link': {'head_uuid': coll_uuid,
203 self.make_mount(fuse.TagsDirectory, poll_time=1)
205 self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
207 bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
208 self.tag_collection(bar_uuid, 'fuse_test_tag')
209 for attempt in AssertWithTimeout(10):
210 attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
211 self.assertDirContents('fuse_test_tag', [bar_uuid])
213 baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
214 l = self.tag_collection(baz_uuid, 'fuse_test_tag')
215 for attempt in AssertWithTimeout(10):
216 attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
218 self.api.links().delete(uuid=l['uuid']).execute()
219 for attempt in AssertWithTimeout(10):
220 attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
223 class FuseSharedTest(MountTestBase):
225 self.make_mount(fuse.SharedDirectory,
226 exclude=self.api.users().current().execute()['uuid'])
228 # shared_dirs is a list of the directories exposed
229 # by fuse.SharedDirectory (i.e. any object visible
230 # to the current user)
231 shared_dirs = llfuse.listdir(self.mounttmp)
233 self.assertIn('FUSE User', shared_dirs)
235 # fuse_user_objs is a list of the objects owned by the FUSE
236 # test user (which present as files in the 'FUSE User'
238 fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
239 fuse_user_objs.sort()
240 self.assertEqual(['FUSE Test Project', # project owned by user
241 'collection #1 owned by FUSE', # collection owned by user
242 'collection #2 owned by FUSE', # collection owned by user
243 'pipeline instance owned by FUSE.pipelineInstance', # pipeline instance owned by user
246 # test_proj_files is a list of the files in the FUSE Test Project.
247 test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
248 test_proj_files.sort()
249 self.assertEqual(['collection in FUSE project',
250 'pipeline instance in FUSE project.pipelineInstance',
251 'pipeline template in FUSE project.pipelineTemplate'
254 # Double check that we can open and read objects in this folder as a file,
255 # and that its contents are what we expect.
256 pipeline_template_path = os.path.join(
260 'pipeline template in FUSE project.pipelineTemplate')
261 with open(pipeline_template_path) as f:
263 self.assertEqual("pipeline template in FUSE project", j['name'])
265 # check mtime on template
266 st = os.stat(pipeline_template_path)
268 mtime = st.st_mtime_ns / 1000000000
269 except AttributeError:
271 self.assertEqual(mtime, 1397493304)
273 # check mtime on collection
274 st = os.stat(os.path.join(
277 'collection #1 owned by FUSE'))
279 mtime = st.st_mtime_ns / 1000000000
280 except AttributeError:
282 self.assertEqual(mtime, 1391448174)
285 class FuseHomeTest(MountTestBase):
287 self.make_mount(fuse.ProjectDirectory,
288 project_object=self.api.users().current().execute())
290 d1 = llfuse.listdir(self.mounttmp)
291 self.assertIn('Unrestricted public data', d1)
293 d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
294 public_project = run_test_server.fixture('groups')[
295 'anonymously_accessible_project']
298 for name, item in run_test_server.fixture('collections').iteritems():
299 if 'name' not in item:
301 elif item['owner_uuid'] == public_project['uuid']:
302 self.assertIn(item['name'], d2)
305 # Artificial assumption here: there is no public
306 # collection fixture with the same name as a
307 # non-public collection.
308 self.assertNotIn(item['name'], d2)
310 self.assertNotEqual(0, found_in)
311 self.assertNotEqual(0, found_not_in)
313 d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
314 self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
317 def fuseModifyFileTestHelperReadStartContents(mounttmp):
318 class Test(unittest.TestCase):
320 d1 = llfuse.listdir(mounttmp)
321 self.assertEqual(["file1.txt"], d1)
322 with open(os.path.join(mounttmp, "file1.txt")) as f:
323 self.assertEqual("blub", f.read())
326 def fuseModifyFileTestHelperReadEndContents(mounttmp):
327 class Test(unittest.TestCase):
329 d1 = llfuse.listdir(mounttmp)
330 self.assertEqual(["file1.txt"], d1)
331 with open(os.path.join(mounttmp, "file1.txt")) as f:
332 self.assertEqual("plnp", f.read())
335 class FuseModifyFileTest(MountTestBase):
337 collection = arvados.collection.Collection(api_client=self.api)
338 with collection.open("file1.txt", "w") as f:
341 collection.save_new()
343 m = self.make_mount(fuse.CollectionDirectory)
345 m.new_collection(collection.api_response(), collection)
347 self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
349 with collection.open("file1.txt", "w") as f:
352 self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
355 class FuseAddFileToCollectionTest(MountTestBase):
357 collection = arvados.collection.Collection(api_client=self.api)
358 with collection.open("file1.txt", "w") as f:
361 collection.save_new()
363 m = self.make_mount(fuse.CollectionDirectory)
365 m.new_collection(collection.api_response(), collection)
367 d1 = llfuse.listdir(self.mounttmp)
368 self.assertEqual(["file1.txt"], d1)
370 with collection.open("file2.txt", "w") as f:
373 d1 = llfuse.listdir(self.mounttmp)
374 self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
377 class FuseRemoveFileFromCollectionTest(MountTestBase):
379 collection = arvados.collection.Collection(api_client=self.api)
380 with collection.open("file1.txt", "w") as f:
383 with collection.open("file2.txt", "w") as f:
386 collection.save_new()
388 m = self.make_mount(fuse.CollectionDirectory)
390 m.new_collection(collection.api_response(), collection)
392 d1 = llfuse.listdir(self.mounttmp)
393 self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
395 collection.remove("file2.txt")
397 d1 = llfuse.listdir(self.mounttmp)
398 self.assertEqual(["file1.txt"], d1)
401 def fuseCreateFileTestHelper(mounttmp):
402 class Test(unittest.TestCase):
404 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
408 class FuseCreateFileTest(MountTestBase):
410 collection = arvados.collection.Collection(api_client=self.api)
411 collection.save_new()
413 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
414 self.assertEqual(collection2["manifest_text"], "")
416 collection.save_new()
418 m = self.make_mount(fuse.CollectionDirectory)
420 m.new_collection(collection.api_response(), collection)
421 self.assertTrue(m.writable())
423 self.assertNotIn("file1.txt", collection)
425 self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
427 self.assertIn("file1.txt", collection)
429 d1 = llfuse.listdir(self.mounttmp)
430 self.assertEqual(["file1.txt"], d1)
432 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
433 self.assertRegexpMatches(collection2["manifest_text"],
434 r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
437 def fuseWriteFileTestHelperWriteFile(mounttmp):
438 class Test(unittest.TestCase):
440 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
441 f.write("Hello world!")
444 def fuseWriteFileTestHelperReadFile(mounttmp):
445 class Test(unittest.TestCase):
447 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
448 self.assertEqual(f.read(), "Hello world!")
451 class FuseWriteFileTest(MountTestBase):
453 collection = arvados.collection.Collection(api_client=self.api)
454 collection.save_new()
456 m = self.make_mount(fuse.CollectionDirectory)
458 m.new_collection(collection.api_response(), collection)
459 self.assertTrue(m.writable())
461 self.assertNotIn("file1.txt", collection)
463 self.assertEqual(0, self.operations.write_counter.get())
464 self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
465 self.assertEqual(12, self.operations.write_counter.get())
467 with collection.open("file1.txt") as f:
468 self.assertEqual(f.read(), "Hello world!")
470 self.assertEqual(0, self.operations.read_counter.get())
471 self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
472 self.assertEqual(12, self.operations.read_counter.get())
474 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
475 self.assertRegexpMatches(collection2["manifest_text"],
476 r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
479 def fuseUpdateFileTestHelper(mounttmp):
480 class Test(unittest.TestCase):
482 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
483 f.write("Hello world!")
485 with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
487 self.assertEqual(fr, "Hello world!")
489 f.write("Hola mundo!")
492 self.assertEqual(fr, "Hola mundo!!")
494 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
495 self.assertEqual(f.read(), "Hola mundo!!")
499 class FuseUpdateFileTest(MountTestBase):
501 collection = arvados.collection.Collection(api_client=self.api)
502 collection.save_new()
504 m = self.make_mount(fuse.CollectionDirectory)
506 m.new_collection(collection.api_response(), collection)
507 self.assertTrue(m.writable())
509 # See note in MountTestBase.setUp
510 self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
512 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
513 self.assertRegexpMatches(collection2["manifest_text"],
514 r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
517 def fuseMkdirTestHelper(mounttmp):
518 class Test(unittest.TestCase):
520 with self.assertRaises(IOError):
521 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
522 f.write("Hello world!")
524 os.mkdir(os.path.join(mounttmp, "testdir"))
526 with self.assertRaises(OSError):
527 os.mkdir(os.path.join(mounttmp, "testdir"))
529 d1 = llfuse.listdir(mounttmp)
530 self.assertEqual(["testdir"], d1)
532 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
533 f.write("Hello world!")
535 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
536 self.assertEqual(["file1.txt"], d1)
540 class FuseMkdirTest(MountTestBase):
542 collection = arvados.collection.Collection(api_client=self.api)
543 collection.save_new()
545 m = self.make_mount(fuse.CollectionDirectory)
547 m.new_collection(collection.api_response(), collection)
548 self.assertTrue(m.writable())
550 self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
552 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
553 self.assertRegexpMatches(collection2["manifest_text"],
554 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
557 def fuseRmTestHelperWriteFile(mounttmp):
558 class Test(unittest.TestCase):
560 os.mkdir(os.path.join(mounttmp, "testdir"))
562 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
563 f.write("Hello world!")
567 def fuseRmTestHelperDeleteFile(mounttmp):
568 class Test(unittest.TestCase):
570 # Can't delete because it's not empty
571 with self.assertRaises(OSError):
572 os.rmdir(os.path.join(mounttmp, "testdir"))
574 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
575 self.assertEqual(["file1.txt"], d1)
578 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
580 # Make sure it's empty
581 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
582 self.assertEqual([], d1)
584 # Try to delete it again
585 with self.assertRaises(OSError):
586 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
590 def fuseRmTestHelperRmdir(mounttmp):
591 class Test(unittest.TestCase):
593 # Should be able to delete now that it is empty
594 os.rmdir(os.path.join(mounttmp, "testdir"))
596 # Make sure it's empty
597 d1 = llfuse.listdir(os.path.join(mounttmp))
598 self.assertEqual([], d1)
600 # Try to delete it again
601 with self.assertRaises(OSError):
602 os.rmdir(os.path.join(mounttmp, "testdir"))
606 class FuseRmTest(MountTestBase):
608 collection = arvados.collection.Collection(api_client=self.api)
609 collection.save_new()
611 m = self.make_mount(fuse.CollectionDirectory)
613 m.new_collection(collection.api_response(), collection)
614 self.assertTrue(m.writable())
616 self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
619 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
620 self.assertRegexpMatches(collection2["manifest_text"],
621 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
622 self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
624 # Can't have empty directories :-( so manifest will be empty.
625 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
626 self.assertEqual(collection2["manifest_text"], "")
628 self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
630 # manifest should be empty now.
631 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
632 self.assertEqual(collection2["manifest_text"], "")
635 def fuseMvFileTestHelperWriteFile(mounttmp):
636 class Test(unittest.TestCase):
638 os.mkdir(os.path.join(mounttmp, "testdir"))
640 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
641 f.write("Hello world!")
645 def fuseMvFileTestHelperMoveFile(mounttmp):
646 class Test(unittest.TestCase):
648 d1 = llfuse.listdir(os.path.join(mounttmp))
649 self.assertEqual(["testdir"], d1)
650 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
651 self.assertEqual(["file1.txt"], d1)
653 os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
655 d1 = llfuse.listdir(os.path.join(mounttmp))
656 self.assertEqual(["file1.txt", "testdir"], sorted(d1))
657 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
658 self.assertEqual([], d1)
662 class FuseMvFileTest(MountTestBase):
664 collection = arvados.collection.Collection(api_client=self.api)
665 collection.save_new()
667 m = self.make_mount(fuse.CollectionDirectory)
669 m.new_collection(collection.api_response(), collection)
670 self.assertTrue(m.writable())
672 self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
675 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
676 self.assertRegexpMatches(collection2["manifest_text"],
677 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
679 self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
681 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
682 self.assertRegexpMatches(collection2["manifest_text"],
683 r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
686 def fuseRenameTestHelper(mounttmp):
687 class Test(unittest.TestCase):
689 os.mkdir(os.path.join(mounttmp, "testdir"))
691 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
692 f.write("Hello world!")
696 class FuseRenameTest(MountTestBase):
698 collection = arvados.collection.Collection(api_client=self.api)
699 collection.save_new()
701 m = self.make_mount(fuse.CollectionDirectory)
703 m.new_collection(collection.api_response(), collection)
704 self.assertTrue(m.writable())
706 self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
709 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
710 self.assertRegexpMatches(collection2["manifest_text"],
711 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
713 d1 = llfuse.listdir(os.path.join(self.mounttmp))
714 self.assertEqual(["testdir"], d1)
715 d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
716 self.assertEqual(["file1.txt"], d1)
718 os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
720 d1 = llfuse.listdir(os.path.join(self.mounttmp))
721 self.assertEqual(["testdir2"], sorted(d1))
722 d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
723 self.assertEqual(["file1.txt"], d1)
725 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
726 self.assertRegexpMatches(collection2["manifest_text"],
727 r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
730 class FuseUpdateFromEventTest(MountTestBase):
732 collection = arvados.collection.Collection(api_client=self.api)
733 collection.save_new()
735 m = self.make_mount(fuse.CollectionDirectory)
737 m.new_collection(collection.api_response(), collection)
739 self.operations.listen_for_events()
741 d1 = llfuse.listdir(os.path.join(self.mounttmp))
742 self.assertEqual([], sorted(d1))
744 with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
745 with collection2.open("file1.txt", "w") as f:
748 for attempt in AssertWithTimeout(10):
749 attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
752 def fuseFileConflictTestHelper(mounttmp):
753 class Test(unittest.TestCase):
755 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
758 d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
759 self.assertEqual(len(d1), 2)
761 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
762 self.assertEqual(f.read(), "bar")
764 self.assertRegexpMatches(d1[1],
765 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
767 with open(os.path.join(mounttmp, d1[1]), "r") as f:
768 self.assertEqual(f.read(), "foo")
772 class FuseFileConflictTest(MountTestBase):
774 collection = arvados.collection.Collection(api_client=self.api)
775 collection.save_new()
777 m = self.make_mount(fuse.CollectionDirectory)
779 m.new_collection(collection.api_response(), collection)
781 d1 = llfuse.listdir(os.path.join(self.mounttmp))
782 self.assertEqual([], sorted(d1))
784 with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
785 with collection2.open("file1.txt", "w") as f:
788 # See note in MountTestBase.setUp
789 self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
792 def fuseUnlinkOpenFileTest(mounttmp):
793 class Test(unittest.TestCase):
795 with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
798 d1 = llfuse.listdir(os.path.join(mounttmp))
799 self.assertEqual(["file1.txt"], sorted(d1))
801 os.remove(os.path.join(mounttmp, "file1.txt"))
803 d1 = llfuse.listdir(os.path.join(mounttmp))
804 self.assertEqual([], sorted(d1))
807 self.assertEqual(f.read(), "foo")
811 self.assertEqual(f.read(), "foobar")
815 class FuseUnlinkOpenFileTest(MountTestBase):
817 collection = arvados.collection.Collection(api_client=self.api)
818 collection.save_new()
820 m = self.make_mount(fuse.CollectionDirectory)
822 m.new_collection(collection.api_response(), collection)
824 # See note in MountTestBase.setUp
825 self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
827 self.assertEqual(collection.manifest_text(), "")
830 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
831 class Test(unittest.TestCase):
833 with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
834 f.write("Hello world!")
836 d1 = os.listdir(os.path.join(mounttmp, uuid1))
837 self.assertEqual(["file1.txt"], sorted(d1))
838 d1 = os.listdir(os.path.join(mounttmp, uuid2))
839 self.assertEqual([], sorted(d1))
843 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
844 class Test(unittest.TestCase):
846 os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
848 d1 = os.listdir(os.path.join(mounttmp, uuid1))
849 self.assertEqual([], sorted(d1))
850 d1 = os.listdir(os.path.join(mounttmp, uuid2))
851 self.assertEqual(["file2.txt"], sorted(d1))
855 class FuseMvFileBetweenCollectionsTest(MountTestBase):
857 collection1 = arvados.collection.Collection(api_client=self.api)
858 collection1.save_new()
860 collection2 = arvados.collection.Collection(api_client=self.api)
861 collection2.save_new()
863 m = self.make_mount(fuse.MagicDirectory)
865 # See note in MountTestBase.setUp
866 self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
867 collection1.manifest_locator(),
868 collection2.manifest_locator()))
873 self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
874 self.assertEqual(collection2.manifest_text(), "")
876 self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
877 collection1.manifest_locator(),
878 collection2.manifest_locator()))
883 self.assertEqual(collection1.manifest_text(), "")
884 self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
886 collection1.stop_threads()
887 collection2.stop_threads()
890 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
891 class Test(unittest.TestCase):
893 os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
894 with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
895 f.write("Hello world!")
897 d1 = os.listdir(os.path.join(mounttmp, uuid1))
898 self.assertEqual(["testdir"], sorted(d1))
899 d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
900 self.assertEqual(["file1.txt"], sorted(d1))
902 d1 = os.listdir(os.path.join(mounttmp, uuid2))
903 self.assertEqual([], sorted(d1))
908 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
909 class Test(unittest.TestCase):
911 os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
913 d1 = os.listdir(os.path.join(mounttmp, uuid1))
914 self.assertEqual([], sorted(d1))
916 d1 = os.listdir(os.path.join(mounttmp, uuid2))
917 self.assertEqual(["testdir2"], sorted(d1))
918 d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
919 self.assertEqual(["file1.txt"], sorted(d1))
921 with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
922 self.assertEqual(f.read(), "Hello world!")
926 class FuseMvDirBetweenCollectionsTest(MountTestBase):
928 collection1 = arvados.collection.Collection(api_client=self.api)
929 collection1.save_new()
931 collection2 = arvados.collection.Collection(api_client=self.api)
932 collection2.save_new()
934 m = self.make_mount(fuse.MagicDirectory)
936 # See note in MountTestBase.setUp
937 self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
938 collection1.manifest_locator(),
939 collection2.manifest_locator()))
944 self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
945 self.assertEqual(collection2.manifest_text(), "")
947 self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
948 collection1.manifest_locator(),
949 collection2.manifest_locator()))
954 self.assertEqual(collection1.manifest_text(), "")
955 self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
957 collection1.stop_threads()
958 collection2.stop_threads()
960 def fuseProjectMkdirTestHelper1(mounttmp):
961 class Test(unittest.TestCase):
963 os.mkdir(os.path.join(mounttmp, "testcollection"))
964 with self.assertRaises(OSError):
965 os.mkdir(os.path.join(mounttmp, "testcollection"))
968 def fuseProjectMkdirTestHelper2(mounttmp):
969 class Test(unittest.TestCase):
971 with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
972 f.write("Hello world!")
973 with self.assertRaises(OSError):
974 os.rmdir(os.path.join(mounttmp, "testcollection"))
975 os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
976 with self.assertRaises(OSError):
977 os.remove(os.path.join(mounttmp, "testcollection"))
978 os.rmdir(os.path.join(mounttmp, "testcollection"))
981 class FuseProjectMkdirRmdirTest(MountTestBase):
983 self.make_mount(fuse.ProjectDirectory,
984 project_object=self.api.users().current().execute())
986 d1 = llfuse.listdir(self.mounttmp)
987 self.assertNotIn('testcollection', d1)
989 self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
991 d1 = llfuse.listdir(self.mounttmp)
992 self.assertIn('testcollection', d1)
994 self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
996 d1 = llfuse.listdir(self.mounttmp)
997 self.assertNotIn('testcollection', d1)
1000 def fuseProjectMvTestHelper1(mounttmp):
1001 class Test(unittest.TestCase):
1003 d1 = llfuse.listdir(mounttmp)
1004 self.assertNotIn('testcollection', d1)
1006 os.mkdir(os.path.join(mounttmp, "testcollection"))
1008 d1 = llfuse.listdir(mounttmp)
1009 self.assertIn('testcollection', d1)
1011 with self.assertRaises(OSError):
1012 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1014 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1016 d1 = llfuse.listdir(mounttmp)
1017 self.assertNotIn('testcollection', d1)
1019 d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1020 self.assertIn('testcollection', d1)
1024 class FuseProjectMvTest(MountTestBase):
1026 self.make_mount(fuse.ProjectDirectory,
1027 project_object=self.api.users().current().execute())
1029 self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1032 def fuseFsyncTestHelper(mounttmp, k):
1033 class Test(unittest.TestCase):
1035 fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1041 class FuseFsyncTest(FuseMagicTest):
1043 self.make_mount(fuse.MagicDirectory)
1044 self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1047 class MagicDirApiError(FuseMagicTest):
1049 api = mock.MagicMock()
1050 super(MagicDirApiError, self).setUp(api=api)
1051 api.collections().get().execute.side_effect = iter([Exception('API fail'), {"manifest_text": self.test_manifest}])
1052 api.keep.get.side_effect = Exception('Keep fail')
1055 self.make_mount(fuse.MagicDirectory)
1057 self.operations.inodes.inode_cache.cap = 1
1058 self.operations.inodes.inode_cache.min_entries = 2
1060 with self.assertRaises(OSError):
1061 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1063 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1066 class FuseUnitTest(unittest.TestCase):
1067 def test_sanitize_filename(self):
1086 for f in acceptable:
1087 self.assertEqual(f, fuse.sanitize_filename(f))
1088 for f in unacceptable:
1089 self.assertNotEqual(f, fuse.sanitize_filename(f))
1090 # The sanitized filename should be the same length, though.
1091 self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1093 self.assertEqual("_", fuse.sanitize_filename(""))
1094 self.assertEqual("_", fuse.sanitize_filename("."))
1095 self.assertEqual("__", fuse.sanitize_filename(".."))
1098 class FuseMagicTestPDHOnly(MountTestBase):
1099 def setUp(self, api=None):
1100 super(FuseMagicTestPDHOnly, self).setUp(api=api)
1102 cw = arvados.CollectionWriter()
1104 cw.start_new_file('thing1.txt')
1107 self.testcollection = cw.finish()
1108 self.test_manifest = cw.manifest_text()
1109 created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1110 self.testcollectionuuid = str(created['uuid'])
1112 def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1113 if skip_pdh_only is True:
1114 self.make_mount(fuse.MagicDirectory) # in this case, the default by_id applies
1116 self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1118 mount_ls = llfuse.listdir(self.mounttmp)
1119 self.assertIn('README', mount_ls)
1120 self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1121 arvados.util.uuid_pattern.match(fn)
1122 for fn in mount_ls),
1123 "new FUSE MagicDirectory lists Collection")
1125 # look up using pdh should succeed in all cases
1126 self.assertDirContents(self.testcollection, ['thing1.txt'])
1127 self.assertDirContents(os.path.join('by_id', self.testcollection),
1129 mount_ls = llfuse.listdir(self.mounttmp)
1130 self.assertIn('README', mount_ls)
1131 self.assertIn(self.testcollection, mount_ls)
1132 self.assertIn(self.testcollection,
1133 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1136 files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1138 for k, v in files.items():
1139 with open(os.path.join(self.mounttmp, k)) as f:
1140 self.assertEqual(v, f.read())
1142 # look up using uuid should fail when pdh_only is set
1143 if pdh_only is True:
1144 with self.assertRaises(OSError):
1145 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1148 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1151 def test_with_pdh_only_true(self):
1152 self.verify_pdh_only(pdh_only=True)
1154 def test_with_pdh_only_false(self):
1155 self.verify_pdh_only(pdh_only=False)
1157 def test_with_default_by_id(self):
1158 self.verify_pdh_only(skip_pdh_only=True)