3 import arvados_fuse as fuse
16 import multiprocessing
17 import run_test_server
20 from mount_test_base import MountTestBase
22 logger = logging.getLogger('arvados.arv-mount')
25 class FuseMountTest(MountTestBase):
27 super(FuseMountTest, self).setUp()
29 cw = arvados.CollectionWriter()
31 cw.start_new_file('thing1.txt')
33 cw.start_new_file('thing2.txt')
36 cw.start_new_stream('dir1')
37 cw.start_new_file('thing3.txt')
39 cw.start_new_file('thing4.txt')
42 cw.start_new_stream('dir2')
43 cw.start_new_file('thing5.txt')
45 cw.start_new_file('thing6.txt')
48 cw.start_new_stream('dir2/dir3')
49 cw.start_new_file('thing7.txt')
52 cw.start_new_file('thing8.txt')
55 cw.start_new_stream('edgecases')
56 for f in ":/.../-/*/\x01\\/ ".split("/"):
60 for f in ":/.../-/*/\x01\\/ ".split("/"):
61 cw.start_new_stream('edgecases/dirs/' + f)
62 cw.start_new_file('x/x')
65 self.testcollection = cw.finish()
66 self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
69 self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
71 self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
72 'edgecases', 'dir1', 'dir2'])
73 self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
74 self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
75 self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
76 self.assertDirContents('edgecases',
77 "dirs/:/.../-/*/\x01\\/ ".split("/"))
78 self.assertDirContents('edgecases/dirs',
79 ":/.../-/*/\x01\\/ ".split("/"))
81 files = {'thing1.txt': 'data 1',
82 'thing2.txt': 'data 2',
83 'dir1/thing3.txt': 'data 3',
84 'dir1/thing4.txt': 'data 4',
85 'dir2/thing5.txt': 'data 5',
86 'dir2/thing6.txt': 'data 6',
87 'dir2/dir3/thing7.txt': 'data 7',
88 'dir2/dir3/thing8.txt': 'data 8'}
90 for k, v in files.items():
91 with open(os.path.join(self.mounttmp, k)) as f:
92 self.assertEqual(v, f.read())
95 class FuseNoAPITest(MountTestBase):
97 super(FuseNoAPITest, self).setUp()
98 keep = arvados.keep.KeepClient(local_store=self.keeptmp)
99 self.file_data = "API-free text\n"
100 self.file_loc = keep.put(self.file_data)
101 self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
102 self.file_loc, len(self.file_data)))
105 self.make_mount(fuse.MagicDirectory)
106 self.assertDirContents(self.coll_loc, ['api-free.txt'])
107 with open(os.path.join(
108 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
109 actual = keep_file.read(-1)
110 self.assertEqual(self.file_data, actual)
113 class FuseMagicTest(MountTestBase):
114 def setUp(self, api=None):
115 super(FuseMagicTest, self).setUp(api=api)
117 cw = arvados.CollectionWriter()
119 cw.start_new_file('thing1.txt')
122 self.testcollection = cw.finish()
123 self.test_manifest = cw.manifest_text()
124 self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
127 self.make_mount(fuse.MagicDirectory)
129 mount_ls = llfuse.listdir(self.mounttmp)
130 self.assertIn('README', mount_ls)
131 self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
132 arvados.util.uuid_pattern.match(fn)
134 "new FUSE MagicDirectory lists Collection")
135 self.assertDirContents(self.testcollection, ['thing1.txt'])
136 self.assertDirContents(os.path.join('by_id', self.testcollection),
138 mount_ls = llfuse.listdir(self.mounttmp)
139 self.assertIn('README', mount_ls)
140 self.assertIn(self.testcollection, mount_ls)
141 self.assertIn(self.testcollection,
142 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
145 files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
147 for k, v in files.items():
148 with open(os.path.join(self.mounttmp, k)) as f:
149 self.assertEqual(v, f.read())
152 class FuseTagsTest(MountTestBase):
154 self.make_mount(fuse.TagsDirectory)
156 d1 = llfuse.listdir(self.mounttmp)
158 self.assertEqual(['foo_tag'], d1)
160 d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
162 self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
164 d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
166 self.assertEqual(['foo'], d3)
169 class FuseTagsUpdateTest(MountTestBase):
170 def tag_collection(self, coll_uuid, tag_name):
171 return self.api.links().create(
172 body={'link': {'head_uuid': coll_uuid,
178 self.make_mount(fuse.TagsDirectory, poll_time=1)
180 self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
182 bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
183 self.tag_collection(bar_uuid, 'fuse_test_tag')
185 self.assertIn('fuse_test_tag', llfuse.listdir(self.mounttmp))
186 self.assertDirContents('fuse_test_tag', [bar_uuid])
188 baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
189 l = self.tag_collection(baz_uuid, 'fuse_test_tag')
191 self.assertDirContents('fuse_test_tag', [bar_uuid, baz_uuid])
193 self.api.links().delete(uuid=l['uuid']).execute()
195 self.assertDirContents('fuse_test_tag', [bar_uuid])
198 class FuseSharedTest(MountTestBase):
200 self.make_mount(fuse.SharedDirectory,
201 exclude=self.api.users().current().execute()['uuid'])
203 # shared_dirs is a list of the directories exposed
204 # by fuse.SharedDirectory (i.e. any object visible
205 # to the current user)
206 shared_dirs = llfuse.listdir(self.mounttmp)
208 self.assertIn('FUSE User', shared_dirs)
210 # fuse_user_objs is a list of the objects owned by the FUSE
211 # test user (which present as files in the 'FUSE User'
213 fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
214 fuse_user_objs.sort()
215 self.assertEqual(['FUSE Test Project', # project owned by user
216 'collection #1 owned by FUSE', # collection owned by user
217 'collection #2 owned by FUSE', # collection owned by user
218 'pipeline instance owned by FUSE.pipelineInstance', # pipeline instance owned by user
221 # test_proj_files is a list of the files in the FUSE Test Project.
222 test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
223 test_proj_files.sort()
224 self.assertEqual(['collection in FUSE project',
225 'pipeline instance in FUSE project.pipelineInstance',
226 'pipeline template in FUSE project.pipelineTemplate'
229 # Double check that we can open and read objects in this folder as a file,
230 # and that its contents are what we expect.
231 pipeline_template_path = os.path.join(
235 'pipeline template in FUSE project.pipelineTemplate')
236 with open(pipeline_template_path) as f:
238 self.assertEqual("pipeline template in FUSE project", j['name'])
240 # check mtime on template
241 st = os.stat(pipeline_template_path)
242 self.assertEqual(st.st_mtime, 1397493304)
244 # check mtime on collection
245 st = os.stat(os.path.join(
248 'collection #1 owned by FUSE'))
249 self.assertEqual(st.st_mtime, 1391448174)
252 class FuseHomeTest(MountTestBase):
254 self.make_mount(fuse.ProjectDirectory,
255 project_object=self.api.users().current().execute())
257 d1 = llfuse.listdir(self.mounttmp)
258 self.assertIn('Unrestricted public data', d1)
260 d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
261 public_project = run_test_server.fixture('groups')[
262 'anonymously_accessible_project']
265 for name, item in run_test_server.fixture('collections').iteritems():
266 if 'name' not in item:
268 elif item['owner_uuid'] == public_project['uuid']:
269 self.assertIn(item['name'], d2)
272 # Artificial assumption here: there is no public
273 # collection fixture with the same name as a
274 # non-public collection.
275 self.assertNotIn(item['name'], d2)
277 self.assertNotEqual(0, found_in)
278 self.assertNotEqual(0, found_not_in)
280 d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
281 self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
284 def fuseModifyFileTestHelperReadStartContents(mounttmp):
285 class Test(unittest.TestCase):
287 d1 = llfuse.listdir(mounttmp)
288 self.assertEqual(["file1.txt"], d1)
289 with open(os.path.join(mounttmp, "file1.txt")) as f:
290 self.assertEqual("blub", f.read())
293 def fuseModifyFileTestHelperReadEndContents(mounttmp):
294 class Test(unittest.TestCase):
296 d1 = llfuse.listdir(mounttmp)
297 self.assertEqual(["file1.txt"], d1)
298 with open(os.path.join(mounttmp, "file1.txt")) as f:
299 self.assertEqual("plnp", f.read())
302 class FuseModifyFileTest(MountTestBase):
304 collection = arvados.collection.Collection(api_client=self.api)
305 with collection.open("file1.txt", "w") as f:
308 collection.save_new()
310 m = self.make_mount(fuse.CollectionDirectory)
312 m.new_collection(collection.api_response(), collection)
314 self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
316 with collection.open("file1.txt", "w") as f:
319 self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
322 class FuseAddFileToCollectionTest(MountTestBase):
324 collection = arvados.collection.Collection(api_client=self.api)
325 with collection.open("file1.txt", "w") as f:
328 collection.save_new()
330 m = self.make_mount(fuse.CollectionDirectory)
332 m.new_collection(collection.api_response(), collection)
334 d1 = llfuse.listdir(self.mounttmp)
335 self.assertEqual(["file1.txt"], d1)
337 with collection.open("file2.txt", "w") as f:
340 d1 = llfuse.listdir(self.mounttmp)
341 self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
344 class FuseRemoveFileFromCollectionTest(MountTestBase):
346 collection = arvados.collection.Collection(api_client=self.api)
347 with collection.open("file1.txt", "w") as f:
350 with collection.open("file2.txt", "w") as f:
353 collection.save_new()
355 m = self.make_mount(fuse.CollectionDirectory)
357 m.new_collection(collection.api_response(), collection)
359 d1 = llfuse.listdir(self.mounttmp)
360 self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
362 collection.remove("file2.txt")
364 d1 = llfuse.listdir(self.mounttmp)
365 self.assertEqual(["file1.txt"], d1)
368 def fuseCreateFileTestHelper(mounttmp):
369 class Test(unittest.TestCase):
371 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
375 class FuseCreateFileTest(MountTestBase):
377 collection = arvados.collection.Collection(api_client=self.api)
378 collection.save_new()
380 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
381 self.assertEqual(collection2["manifest_text"], "")
383 collection.save_new()
385 m = self.make_mount(fuse.CollectionDirectory)
387 m.new_collection(collection.api_response(), collection)
388 self.assertTrue(m.writable())
390 self.assertNotIn("file1.txt", collection)
392 self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
394 self.assertIn("file1.txt", collection)
396 d1 = llfuse.listdir(self.mounttmp)
397 self.assertEqual(["file1.txt"], d1)
399 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
400 self.assertRegexpMatches(collection2["manifest_text"],
401 r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
404 def fuseWriteFileTestHelperWriteFile(mounttmp):
405 class Test(unittest.TestCase):
407 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
408 f.write("Hello world!")
411 def fuseWriteFileTestHelperReadFile(mounttmp):
412 class Test(unittest.TestCase):
414 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
415 self.assertEqual(f.read(), "Hello world!")
418 class FuseWriteFileTest(MountTestBase):
420 collection = arvados.collection.Collection(api_client=self.api)
421 collection.save_new()
423 m = self.make_mount(fuse.CollectionDirectory)
425 m.new_collection(collection.api_response(), collection)
426 self.assertTrue(m.writable())
428 self.assertNotIn("file1.txt", collection)
430 self.assertEqual(0, self.operations.write_counter.get())
431 self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
432 self.assertEqual(12, self.operations.write_counter.get())
434 with collection.open("file1.txt") as f:
435 self.assertEqual(f.read(), "Hello world!")
437 self.assertEqual(0, self.operations.read_counter.get())
438 self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
439 self.assertEqual(12, self.operations.read_counter.get())
441 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
442 self.assertRegexpMatches(collection2["manifest_text"],
443 r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
446 def fuseUpdateFileTestHelper(mounttmp):
447 class Test(unittest.TestCase):
449 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
450 f.write("Hello world!")
452 with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
454 self.assertEqual(fr, "Hello world!")
456 f.write("Hola mundo!")
459 self.assertEqual(fr, "Hola mundo!!")
461 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
462 self.assertEqual(f.read(), "Hola mundo!!")
466 class FuseUpdateFileTest(MountTestBase):
468 collection = arvados.collection.Collection(api_client=self.api)
469 collection.save_new()
471 m = self.make_mount(fuse.CollectionDirectory)
473 m.new_collection(collection.api_response(), collection)
474 self.assertTrue(m.writable())
476 # See note in MountTestBase.setUp
477 self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
479 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
480 self.assertRegexpMatches(collection2["manifest_text"],
481 r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
484 def fuseMkdirTestHelper(mounttmp):
485 class Test(unittest.TestCase):
487 with self.assertRaises(IOError):
488 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
489 f.write("Hello world!")
491 os.mkdir(os.path.join(mounttmp, "testdir"))
493 with self.assertRaises(OSError):
494 os.mkdir(os.path.join(mounttmp, "testdir"))
496 d1 = llfuse.listdir(mounttmp)
497 self.assertEqual(["testdir"], d1)
499 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
500 f.write("Hello world!")
502 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
503 self.assertEqual(["file1.txt"], d1)
507 class FuseMkdirTest(MountTestBase):
509 collection = arvados.collection.Collection(api_client=self.api)
510 collection.save_new()
512 m = self.make_mount(fuse.CollectionDirectory)
514 m.new_collection(collection.api_response(), collection)
515 self.assertTrue(m.writable())
517 self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
519 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
520 self.assertRegexpMatches(collection2["manifest_text"],
521 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
524 def fuseRmTestHelperWriteFile(mounttmp):
525 class Test(unittest.TestCase):
527 os.mkdir(os.path.join(mounttmp, "testdir"))
529 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
530 f.write("Hello world!")
534 def fuseRmTestHelperDeleteFile(mounttmp):
535 class Test(unittest.TestCase):
537 # Can't delete because it's not empty
538 with self.assertRaises(OSError):
539 os.rmdir(os.path.join(mounttmp, "testdir"))
541 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
542 self.assertEqual(["file1.txt"], d1)
545 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
547 # Make sure it's empty
548 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
549 self.assertEqual([], d1)
551 # Try to delete it again
552 with self.assertRaises(OSError):
553 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
557 def fuseRmTestHelperRmdir(mounttmp):
558 class Test(unittest.TestCase):
560 # Should be able to delete now that it is empty
561 os.rmdir(os.path.join(mounttmp, "testdir"))
563 # Make sure it's empty
564 d1 = llfuse.listdir(os.path.join(mounttmp))
565 self.assertEqual([], d1)
567 # Try to delete it again
568 with self.assertRaises(OSError):
569 os.rmdir(os.path.join(mounttmp, "testdir"))
573 class FuseRmTest(MountTestBase):
575 collection = arvados.collection.Collection(api_client=self.api)
576 collection.save_new()
578 m = self.make_mount(fuse.CollectionDirectory)
580 m.new_collection(collection.api_response(), collection)
581 self.assertTrue(m.writable())
583 self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
586 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
587 self.assertRegexpMatches(collection2["manifest_text"],
588 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
589 self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
591 # Can't have empty directories :-( so manifest will be empty.
592 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
593 self.assertEqual(collection2["manifest_text"], "")
595 self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
597 # manifest should be empty now.
598 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
599 self.assertEqual(collection2["manifest_text"], "")
602 def fuseMvFileTestHelperWriteFile(mounttmp):
603 class Test(unittest.TestCase):
605 os.mkdir(os.path.join(mounttmp, "testdir"))
607 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
608 f.write("Hello world!")
612 def fuseMvFileTestHelperMoveFile(mounttmp):
613 class Test(unittest.TestCase):
615 d1 = llfuse.listdir(os.path.join(mounttmp))
616 self.assertEqual(["testdir"], d1)
617 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
618 self.assertEqual(["file1.txt"], d1)
620 os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
622 d1 = llfuse.listdir(os.path.join(mounttmp))
623 self.assertEqual(["file1.txt", "testdir"], sorted(d1))
624 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
625 self.assertEqual([], d1)
629 class FuseMvFileTest(MountTestBase):
631 collection = arvados.collection.Collection(api_client=self.api)
632 collection.save_new()
634 m = self.make_mount(fuse.CollectionDirectory)
636 m.new_collection(collection.api_response(), collection)
637 self.assertTrue(m.writable())
639 self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
642 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
643 self.assertRegexpMatches(collection2["manifest_text"],
644 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
646 self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
648 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
649 self.assertRegexpMatches(collection2["manifest_text"],
650 r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
653 def fuseRenameTestHelper(mounttmp):
654 class Test(unittest.TestCase):
656 os.mkdir(os.path.join(mounttmp, "testdir"))
658 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
659 f.write("Hello world!")
663 class FuseRenameTest(MountTestBase):
665 collection = arvados.collection.Collection(api_client=self.api)
666 collection.save_new()
668 m = self.make_mount(fuse.CollectionDirectory)
670 m.new_collection(collection.api_response(), collection)
671 self.assertTrue(m.writable())
673 self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
676 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
677 self.assertRegexpMatches(collection2["manifest_text"],
678 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
680 d1 = llfuse.listdir(os.path.join(self.mounttmp))
681 self.assertEqual(["testdir"], d1)
682 d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
683 self.assertEqual(["file1.txt"], d1)
685 os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
687 d1 = llfuse.listdir(os.path.join(self.mounttmp))
688 self.assertEqual(["testdir2"], sorted(d1))
689 d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
690 self.assertEqual(["file1.txt"], d1)
692 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
693 self.assertRegexpMatches(collection2["manifest_text"],
694 r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
697 class FuseUpdateFromEventTest(MountTestBase):
699 collection = arvados.collection.Collection(api_client=self.api)
700 collection.save_new()
702 m = self.make_mount(fuse.CollectionDirectory)
704 m.new_collection(collection.api_response(), collection)
706 self.operations.listen_for_events()
708 d1 = llfuse.listdir(os.path.join(self.mounttmp))
709 self.assertEqual([], sorted(d1))
711 with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
712 with collection2.open("file1.txt", "w") as f:
717 # should show up via event bus notify
719 d1 = llfuse.listdir(os.path.join(self.mounttmp))
720 self.assertEqual(["file1.txt"], sorted(d1))
723 def fuseFileConflictTestHelper(mounttmp):
724 class Test(unittest.TestCase):
726 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
729 d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
730 self.assertEqual(len(d1), 2)
732 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
733 self.assertEqual(f.read(), "bar")
735 self.assertRegexpMatches(d1[1],
736 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
738 with open(os.path.join(mounttmp, d1[1]), "r") as f:
739 self.assertEqual(f.read(), "foo")
743 class FuseFileConflictTest(MountTestBase):
745 collection = arvados.collection.Collection(api_client=self.api)
746 collection.save_new()
748 m = self.make_mount(fuse.CollectionDirectory)
750 m.new_collection(collection.api_response(), collection)
752 d1 = llfuse.listdir(os.path.join(self.mounttmp))
753 self.assertEqual([], sorted(d1))
755 with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
756 with collection2.open("file1.txt", "w") as f:
759 # See note in MountTestBase.setUp
760 self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
763 def fuseUnlinkOpenFileTest(mounttmp):
764 class Test(unittest.TestCase):
766 with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
769 d1 = llfuse.listdir(os.path.join(mounttmp))
770 self.assertEqual(["file1.txt"], sorted(d1))
772 os.remove(os.path.join(mounttmp, "file1.txt"))
774 d1 = llfuse.listdir(os.path.join(mounttmp))
775 self.assertEqual([], sorted(d1))
778 self.assertEqual(f.read(), "foo")
782 self.assertEqual(f.read(), "foobar")
786 class FuseUnlinkOpenFileTest(MountTestBase):
788 collection = arvados.collection.Collection(api_client=self.api)
789 collection.save_new()
791 m = self.make_mount(fuse.CollectionDirectory)
793 m.new_collection(collection.api_response(), collection)
795 # See note in MountTestBase.setUp
796 self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
798 self.assertEqual(collection.manifest_text(), "")
801 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
802 class Test(unittest.TestCase):
804 with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
805 f.write("Hello world!")
807 d1 = os.listdir(os.path.join(mounttmp, uuid1))
808 self.assertEqual(["file1.txt"], sorted(d1))
809 d1 = os.listdir(os.path.join(mounttmp, uuid2))
810 self.assertEqual([], sorted(d1))
814 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
815 class Test(unittest.TestCase):
817 os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
819 d1 = os.listdir(os.path.join(mounttmp, uuid1))
820 self.assertEqual([], sorted(d1))
821 d1 = os.listdir(os.path.join(mounttmp, uuid2))
822 self.assertEqual(["file2.txt"], sorted(d1))
826 class FuseMvFileBetweenCollectionsTest(MountTestBase):
828 collection1 = arvados.collection.Collection(api_client=self.api)
829 collection1.save_new()
831 collection2 = arvados.collection.Collection(api_client=self.api)
832 collection2.save_new()
834 m = self.make_mount(fuse.MagicDirectory)
836 # See note in MountTestBase.setUp
837 self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
838 collection1.manifest_locator(),
839 collection2.manifest_locator()))
844 self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
845 self.assertEqual(collection2.manifest_text(), "")
847 self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
848 collection1.manifest_locator(),
849 collection2.manifest_locator()))
854 self.assertEqual(collection1.manifest_text(), "")
855 self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
857 collection1.stop_threads()
858 collection2.stop_threads()
861 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
862 class Test(unittest.TestCase):
864 os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
865 with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
866 f.write("Hello world!")
868 d1 = os.listdir(os.path.join(mounttmp, uuid1))
869 self.assertEqual(["testdir"], sorted(d1))
870 d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
871 self.assertEqual(["file1.txt"], sorted(d1))
873 d1 = os.listdir(os.path.join(mounttmp, uuid2))
874 self.assertEqual([], sorted(d1))
879 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
880 class Test(unittest.TestCase):
882 os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
884 d1 = os.listdir(os.path.join(mounttmp, uuid1))
885 self.assertEqual([], sorted(d1))
887 d1 = os.listdir(os.path.join(mounttmp, uuid2))
888 self.assertEqual(["testdir2"], sorted(d1))
889 d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
890 self.assertEqual(["file1.txt"], sorted(d1))
892 with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
893 self.assertEqual(f.read(), "Hello world!")
897 class FuseMvDirBetweenCollectionsTest(MountTestBase):
899 collection1 = arvados.collection.Collection(api_client=self.api)
900 collection1.save_new()
902 collection2 = arvados.collection.Collection(api_client=self.api)
903 collection2.save_new()
905 m = self.make_mount(fuse.MagicDirectory)
907 # See note in MountTestBase.setUp
908 self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
909 collection1.manifest_locator(),
910 collection2.manifest_locator()))
915 self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
916 self.assertEqual(collection2.manifest_text(), "")
918 self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
919 collection1.manifest_locator(),
920 collection2.manifest_locator()))
925 self.assertEqual(collection1.manifest_text(), "")
926 self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
928 collection1.stop_threads()
929 collection2.stop_threads()
931 def fuseProjectMkdirTestHelper1(mounttmp):
932 class Test(unittest.TestCase):
934 os.mkdir(os.path.join(mounttmp, "testcollection"))
935 with self.assertRaises(OSError):
936 os.mkdir(os.path.join(mounttmp, "testcollection"))
939 def fuseProjectMkdirTestHelper2(mounttmp):
940 class Test(unittest.TestCase):
942 with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
943 f.write("Hello world!")
944 with self.assertRaises(OSError):
945 os.rmdir(os.path.join(mounttmp, "testcollection"))
946 os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
947 with self.assertRaises(OSError):
948 os.remove(os.path.join(mounttmp, "testcollection"))
949 os.rmdir(os.path.join(mounttmp, "testcollection"))
952 class FuseProjectMkdirRmdirTest(MountTestBase):
954 self.make_mount(fuse.ProjectDirectory,
955 project_object=self.api.users().current().execute())
957 d1 = llfuse.listdir(self.mounttmp)
958 self.assertNotIn('testcollection', d1)
960 self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
962 d1 = llfuse.listdir(self.mounttmp)
963 self.assertIn('testcollection', d1)
965 self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
967 d1 = llfuse.listdir(self.mounttmp)
968 self.assertNotIn('testcollection', d1)
971 def fuseProjectMvTestHelper1(mounttmp):
972 class Test(unittest.TestCase):
974 d1 = llfuse.listdir(mounttmp)
975 self.assertNotIn('testcollection', d1)
977 os.mkdir(os.path.join(mounttmp, "testcollection"))
979 d1 = llfuse.listdir(mounttmp)
980 self.assertIn('testcollection', d1)
982 with self.assertRaises(OSError):
983 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
985 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
987 d1 = llfuse.listdir(mounttmp)
988 self.assertNotIn('testcollection', d1)
990 d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
991 self.assertIn('testcollection', d1)
995 class FuseProjectMvTest(MountTestBase):
997 self.make_mount(fuse.ProjectDirectory,
998 project_object=self.api.users().current().execute())
1000 self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1003 def fuseFsyncTestHelper(mounttmp, k):
1004 class Test(unittest.TestCase):
1006 fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1012 class FuseFsyncTest(FuseMagicTest):
1014 self.make_mount(fuse.MagicDirectory)
1015 self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1018 class MagicDirApiError(FuseMagicTest):
1020 api = mock.MagicMock()
1021 super(MagicDirApiError, self).setUp(api=api)
1022 api.collections().get().execute.side_effect = iter([Exception('API fail'), {"manifest_text": self.test_manifest}])
1023 api.keep.get.side_effect = Exception('Keep fail')
1026 self.make_mount(fuse.MagicDirectory)
1028 self.operations.inodes.inode_cache.cap = 1
1029 self.operations.inodes.inode_cache.min_entries = 2
1031 with self.assertRaises(OSError):
1032 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1034 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1037 class FuseUnitTest(unittest.TestCase):
1038 def test_sanitize_filename(self):
1057 for f in acceptable:
1058 self.assertEqual(f, fuse.sanitize_filename(f))
1059 for f in unacceptable:
1060 self.assertNotEqual(f, fuse.sanitize_filename(f))
1061 # The sanitized filename should be the same length, though.
1062 self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1064 self.assertEqual("_", fuse.sanitize_filename(""))
1065 self.assertEqual("_", fuse.sanitize_filename("."))
1066 self.assertEqual("__", fuse.sanitize_filename(".."))
1069 class FuseMagicTestPDHOnly(MountTestBase):
1070 def setUp(self, api=None):
1071 super(FuseMagicTestPDHOnly, self).setUp(api=api)
1073 cw = arvados.CollectionWriter()
1075 cw.start_new_file('thing1.txt')
1078 self.testcollection = cw.finish()
1079 self.test_manifest = cw.manifest_text()
1080 created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1081 self.testcollectionuuid = str(created['uuid'])
1083 def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1084 if skip_pdh_only is True:
1085 self.make_mount(fuse.MagicDirectory) # in this case, the default by_id applies
1087 self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1089 mount_ls = llfuse.listdir(self.mounttmp)
1090 self.assertIn('README', mount_ls)
1091 self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1092 arvados.util.uuid_pattern.match(fn)
1093 for fn in mount_ls),
1094 "new FUSE MagicDirectory lists Collection")
1096 # look up using pdh should succeed in all cases
1097 self.assertDirContents(self.testcollection, ['thing1.txt'])
1098 self.assertDirContents(os.path.join('by_id', self.testcollection),
1100 mount_ls = llfuse.listdir(self.mounttmp)
1101 self.assertIn('README', mount_ls)
1102 self.assertIn(self.testcollection, mount_ls)
1103 self.assertIn(self.testcollection,
1104 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1107 files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1109 for k, v in files.items():
1110 with open(os.path.join(self.mounttmp, k)) as f:
1111 self.assertEqual(v, f.read())
1113 # look up using uuid should fail when pdh_only is set
1114 if pdh_only is True:
1115 with self.assertRaises(OSError):
1116 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1119 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1122 def test_with_pdh_only_true(self):
1123 self.verify_pdh_only(pdh_only=True)
1125 def test_with_pdh_only_false(self):
1126 self.verify_pdh_only(pdh_only=False)
1128 def test_with_default_by_id(self):
1129 self.verify_pdh_only(skip_pdh_only=True)