1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: AGPL-3.0
5 from __future__ import absolute_import
6 from future.utils import viewitems
7 from builtins import str
8 from builtins import object
19 import arvados_fuse as fuse
20 from . import run_test_server
22 from .mount_test_base import MountTestBase
24 logger = logging.getLogger('arvados.arv-mount')
27 class AssertWithTimeout(object):
28 """Allow some time for an assertion to pass."""
30 def __init__(self, timeout=0):
31 self.timeout = timeout
34 self.deadline = time.time() + self.timeout
43 def attempt(self, fn, *args, **kwargs):
46 except AssertionError:
47 if time.time() > self.deadline:
54 class FuseMountTest(MountTestBase):
56 super(FuseMountTest, self).setUp()
58 cw = arvados.CollectionWriter()
60 cw.start_new_file('thing1.txt')
62 cw.start_new_file('thing2.txt')
65 cw.start_new_stream('dir1')
66 cw.start_new_file('thing3.txt')
68 cw.start_new_file('thing4.txt')
71 cw.start_new_stream('dir2')
72 cw.start_new_file('thing5.txt')
74 cw.start_new_file('thing6.txt')
77 cw.start_new_stream('dir2/dir3')
78 cw.start_new_file('thing7.txt')
81 cw.start_new_file('thing8.txt')
84 cw.start_new_stream('edgecases')
85 for f in ":/.../-/*/ ".split("/"):
89 for f in ":/.../-/*/ ".split("/"):
90 cw.start_new_stream('edgecases/dirs/' + f)
91 cw.start_new_file('x/x')
94 self.testcollection = cw.finish()
95 self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
98 self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
100 self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
101 'edgecases', 'dir1', 'dir2'])
102 self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
103 self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
104 self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
105 self.assertDirContents('edgecases',
106 "dirs/:/.../-/*/ ".split("/"))
107 self.assertDirContents('edgecases/dirs',
108 ":/.../-/*/ ".split("/"))
110 files = {'thing1.txt': 'data 1',
111 'thing2.txt': 'data 2',
112 'dir1/thing3.txt': 'data 3',
113 'dir1/thing4.txt': 'data 4',
114 'dir2/thing5.txt': 'data 5',
115 'dir2/thing6.txt': 'data 6',
116 'dir2/dir3/thing7.txt': 'data 7',
117 'dir2/dir3/thing8.txt': 'data 8'}
119 for k, v in viewitems(files):
120 with open(os.path.join(self.mounttmp, k), 'rb') as f:
121 self.assertEqual(v, f.read().decode())
124 class FuseMagicTest(MountTestBase):
125 def setUp(self, api=None):
126 super(FuseMagicTest, self).setUp(api=api)
128 self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
129 self.non_project_group = run_test_server.fixture('groups')['public']['uuid']
130 self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
132 cw = arvados.CollectionWriter()
134 cw.start_new_file('thing1.txt')
137 self.testcollection = cw.finish()
138 self.test_manifest = cw.manifest_text()
139 coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
140 self.test_manifest_pdh = coll['portable_data_hash']
143 self.make_mount(fuse.MagicDirectory)
145 mount_ls = llfuse.listdir(self.mounttmp)
146 self.assertIn('README', mount_ls)
147 self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
148 arvados.util.uuid_pattern.match(fn)
150 "new FUSE MagicDirectory has no collections or projects")
151 self.assertDirContents(self.testcollection, ['thing1.txt'])
152 self.assertDirContents(os.path.join('by_id', self.testcollection),
154 self.assertIn(self.collection_in_test_project,
155 llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
156 self.assertIn(self.collection_in_test_project,
157 llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
159 mount_ls = llfuse.listdir(self.mounttmp)
160 self.assertIn('README', mount_ls)
161 self.assertIn(self.testcollection, mount_ls)
162 self.assertIn(self.testcollection,
163 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
164 self.assertIn(self.test_project, mount_ls)
165 self.assertIn(self.test_project,
166 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
168 with self.assertRaises(OSError):
169 llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
172 files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = b'data 1'
174 for k, v in viewitems(files):
175 with open(os.path.join(self.mounttmp, k), 'rb') as f:
176 self.assertEqual(v, f.read())
179 class FuseTagsTest(MountTestBase):
181 self.make_mount(fuse.TagsDirectory)
183 d1 = llfuse.listdir(self.mounttmp)
185 self.assertEqual(['foo_tag'], d1)
187 d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
189 self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
191 d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
193 self.assertEqual(['foo'], d3)
196 class FuseTagsUpdateTest(MountTestBase):
197 def tag_collection(self, coll_uuid, tag_name):
198 return self.api.links().create(
199 body={'link': {'head_uuid': coll_uuid,
205 self.make_mount(fuse.TagsDirectory, poll_time=1)
207 self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
209 bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
210 self.tag_collection(bar_uuid, 'fuse_test_tag')
211 for attempt in AssertWithTimeout(10):
212 attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
213 self.assertDirContents('fuse_test_tag', [bar_uuid])
215 baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
216 l = self.tag_collection(baz_uuid, 'fuse_test_tag')
217 for attempt in AssertWithTimeout(10):
218 attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
220 self.api.links().delete(uuid=l['uuid']).execute()
221 for attempt in AssertWithTimeout(10):
222 attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
225 def fuseSharedTestHelper(mounttmp):
226 class Test(unittest.TestCase):
228 # Double check that we can open and read objects in this folder as a file,
229 # and that its contents are what we expect.
230 baz_path = os.path.join(
234 'collection in FUSE project',
236 with open(baz_path) as f:
237 self.assertEqual("baz", f.read())
239 # check mtime on collection
240 st = os.stat(baz_path)
242 mtime = st.st_mtime_ns // 1000000000
243 except AttributeError:
245 self.assertEqual(mtime, 1391448174)
247 # shared_dirs is a list of the directories exposed
248 # by fuse.SharedDirectory (i.e. any object visible
249 # to the current user)
250 shared_dirs = llfuse.listdir(mounttmp)
252 self.assertIn('FUSE User', shared_dirs)
254 # fuse_user_objs is a list of the objects owned by the FUSE
255 # test user (which present as files in the 'FUSE User'
257 fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
258 fuse_user_objs.sort()
259 self.assertEqual(['FUSE Test Project', # project owned by user
260 'collection #1 owned by FUSE', # collection owned by user
261 'collection #2 owned by FUSE' # collection owned by user
264 # test_proj_files is a list of the files in the FUSE Test Project.
265 test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
266 test_proj_files.sort()
267 self.assertEqual(['collection in FUSE project'
273 class FuseSharedTest(MountTestBase):
275 self.make_mount(fuse.SharedDirectory,
276 exclude=self.api.users().current().execute()['uuid'])
277 keep = arvados.keep.KeepClient()
280 self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
283 class FuseHomeTest(MountTestBase):
285 self.make_mount(fuse.ProjectDirectory,
286 project_object=self.api.users().current().execute())
288 d1 = llfuse.listdir(self.mounttmp)
289 self.assertIn('Unrestricted public data', d1)
291 d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
292 public_project = run_test_server.fixture('groups')[
293 'anonymously_accessible_project']
296 for name, item in viewitems(run_test_server.fixture('collections')):
297 if 'name' not in item:
299 elif item['owner_uuid'] == public_project['uuid']:
300 self.assertIn(item['name'], d2)
303 # Artificial assumption here: there is no public
304 # collection fixture with the same name as a
305 # non-public collection.
306 self.assertNotIn(item['name'], d2)
308 self.assertNotEqual(0, found_in)
309 self.assertNotEqual(0, found_not_in)
311 d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
312 self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
315 def fuseModifyFileTestHelperReadStartContents(mounttmp):
316 class Test(unittest.TestCase):
318 d1 = llfuse.listdir(mounttmp)
319 self.assertEqual(["file1.txt"], d1)
320 with open(os.path.join(mounttmp, "file1.txt")) as f:
321 self.assertEqual("blub", f.read())
324 def fuseModifyFileTestHelperReadEndContents(mounttmp):
325 class Test(unittest.TestCase):
327 d1 = llfuse.listdir(mounttmp)
328 self.assertEqual(["file1.txt"], d1)
329 with open(os.path.join(mounttmp, "file1.txt")) as f:
330 self.assertEqual("plnp", f.read())
333 class FuseModifyFileTest(MountTestBase):
335 collection = arvados.collection.Collection(api_client=self.api)
336 with collection.open("file1.txt", "w") as f:
339 collection.save_new()
341 m = self.make_mount(fuse.CollectionDirectory)
343 m.new_collection(collection.api_response(), collection)
345 self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
347 with collection.open("file1.txt", "w") as f:
350 self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
353 class FuseAddFileToCollectionTest(MountTestBase):
355 collection = arvados.collection.Collection(api_client=self.api)
356 with collection.open("file1.txt", "w") as f:
359 collection.save_new()
361 m = self.make_mount(fuse.CollectionDirectory)
363 m.new_collection(collection.api_response(), collection)
365 d1 = llfuse.listdir(self.mounttmp)
366 self.assertEqual(["file1.txt"], d1)
368 with collection.open("file2.txt", "w") as f:
371 d1 = llfuse.listdir(self.mounttmp)
372 self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
375 class FuseRemoveFileFromCollectionTest(MountTestBase):
377 collection = arvados.collection.Collection(api_client=self.api)
378 with collection.open("file1.txt", "w") as f:
381 with collection.open("file2.txt", "w") as f:
384 collection.save_new()
386 m = self.make_mount(fuse.CollectionDirectory)
388 m.new_collection(collection.api_response(), collection)
390 d1 = llfuse.listdir(self.mounttmp)
391 self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
393 collection.remove("file2.txt")
395 d1 = llfuse.listdir(self.mounttmp)
396 self.assertEqual(["file1.txt"], d1)
399 def fuseCreateFileTestHelper(mounttmp):
400 class Test(unittest.TestCase):
402 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
406 class FuseCreateFileTest(MountTestBase):
408 collection = arvados.collection.Collection(api_client=self.api)
409 collection.save_new()
411 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
412 self.assertEqual(collection2["manifest_text"], "")
414 collection.save_new()
416 m = self.make_mount(fuse.CollectionDirectory)
418 m.new_collection(collection.api_response(), collection)
419 self.assertTrue(m.writable())
421 self.assertNotIn("file1.txt", collection)
423 self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
425 self.assertIn("file1.txt", collection)
427 d1 = llfuse.listdir(self.mounttmp)
428 self.assertEqual(["file1.txt"], d1)
430 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
431 self.assertRegexpMatches(collection2["manifest_text"],
432 r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
435 def fuseWriteFileTestHelperWriteFile(mounttmp):
436 class Test(unittest.TestCase):
438 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
439 f.write("Hello world!")
442 def fuseWriteFileTestHelperReadFile(mounttmp):
443 class Test(unittest.TestCase):
445 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
446 self.assertEqual(f.read(), "Hello world!")
449 class FuseWriteFileTest(MountTestBase):
451 collection = arvados.collection.Collection(api_client=self.api)
452 collection.save_new()
454 m = self.make_mount(fuse.CollectionDirectory)
456 m.new_collection(collection.api_response(), collection)
457 self.assertTrue(m.writable())
459 self.assertNotIn("file1.txt", collection)
461 self.assertEqual(0, self.operations.write_counter.get())
462 self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
463 self.assertEqual(12, self.operations.write_counter.get())
465 with collection.open("file1.txt") as f:
466 self.assertEqual(f.read(), "Hello world!")
468 self.assertEqual(0, self.operations.read_counter.get())
469 self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
470 self.assertEqual(12, self.operations.read_counter.get())
472 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
473 self.assertRegexpMatches(collection2["manifest_text"],
474 r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
477 def fuseUpdateFileTestHelper(mounttmp):
478 class Test(unittest.TestCase):
480 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
481 f.write("Hello world!")
483 with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
485 self.assertEqual(fr, "Hello world!")
487 f.write("Hola mundo!")
490 self.assertEqual(fr, "Hola mundo!!")
492 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
493 self.assertEqual(f.read(), "Hola mundo!!")
497 class FuseUpdateFileTest(MountTestBase):
499 collection = arvados.collection.Collection(api_client=self.api)
500 collection.save_new()
502 m = self.make_mount(fuse.CollectionDirectory)
504 m.new_collection(collection.api_response(), collection)
505 self.assertTrue(m.writable())
507 # See note in MountTestBase.setUp
508 self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
510 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
511 self.assertRegexpMatches(collection2["manifest_text"],
512 r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
515 def fuseMkdirTestHelper(mounttmp):
516 class Test(unittest.TestCase):
518 with self.assertRaises(IOError):
519 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
520 f.write("Hello world!")
522 os.mkdir(os.path.join(mounttmp, "testdir"))
524 with self.assertRaises(OSError):
525 os.mkdir(os.path.join(mounttmp, "testdir"))
527 d1 = llfuse.listdir(mounttmp)
528 self.assertEqual(["testdir"], d1)
530 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
531 f.write("Hello world!")
533 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
534 self.assertEqual(["file1.txt"], d1)
538 class FuseMkdirTest(MountTestBase):
540 collection = arvados.collection.Collection(api_client=self.api)
541 collection.save_new()
543 m = self.make_mount(fuse.CollectionDirectory)
545 m.new_collection(collection.api_response(), collection)
546 self.assertTrue(m.writable())
548 self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
550 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
551 self.assertRegexpMatches(collection2["manifest_text"],
552 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
555 def fuseRmTestHelperWriteFile(mounttmp):
556 class Test(unittest.TestCase):
558 os.mkdir(os.path.join(mounttmp, "testdir"))
560 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
561 f.write("Hello world!")
565 def fuseRmTestHelperDeleteFile(mounttmp):
566 class Test(unittest.TestCase):
568 # Can't delete because it's not empty
569 with self.assertRaises(OSError):
570 os.rmdir(os.path.join(mounttmp, "testdir"))
572 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
573 self.assertEqual(["file1.txt"], d1)
576 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
578 # Make sure it's empty
579 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
580 self.assertEqual([], d1)
582 # Try to delete it again
583 with self.assertRaises(OSError):
584 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
588 def fuseRmTestHelperRmdir(mounttmp):
589 class Test(unittest.TestCase):
591 # Should be able to delete now that it is empty
592 os.rmdir(os.path.join(mounttmp, "testdir"))
594 # Make sure it's empty
595 d1 = llfuse.listdir(os.path.join(mounttmp))
596 self.assertEqual([], d1)
598 # Try to delete it again
599 with self.assertRaises(OSError):
600 os.rmdir(os.path.join(mounttmp, "testdir"))
604 class FuseRmTest(MountTestBase):
606 collection = arvados.collection.Collection(api_client=self.api)
607 collection.save_new()
609 m = self.make_mount(fuse.CollectionDirectory)
611 m.new_collection(collection.api_response(), collection)
612 self.assertTrue(m.writable())
614 self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
617 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
618 self.assertRegexpMatches(collection2["manifest_text"],
619 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
620 self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
622 # Empty directories are represented by an empty file named "."
623 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
624 self.assertRegexpMatches(collection2["manifest_text"],
625 r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
627 self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
629 # manifest should be empty now.
630 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
631 self.assertEqual(collection2["manifest_text"], "")
634 def fuseMvFileTestHelperWriteFile(mounttmp):
635 class Test(unittest.TestCase):
637 os.mkdir(os.path.join(mounttmp, "testdir"))
639 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
640 f.write("Hello world!")
644 def fuseMvFileTestHelperMoveFile(mounttmp):
645 class Test(unittest.TestCase):
647 d1 = llfuse.listdir(os.path.join(mounttmp))
648 self.assertEqual(["testdir"], d1)
649 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
650 self.assertEqual(["file1.txt"], d1)
652 os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
654 d1 = llfuse.listdir(os.path.join(mounttmp))
655 self.assertEqual(["file1.txt", "testdir"], sorted(d1))
656 d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
657 self.assertEqual([], d1)
661 class FuseMvFileTest(MountTestBase):
663 collection = arvados.collection.Collection(api_client=self.api)
664 collection.save_new()
666 m = self.make_mount(fuse.CollectionDirectory)
668 m.new_collection(collection.api_response(), collection)
669 self.assertTrue(m.writable())
671 self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
674 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
675 self.assertRegexpMatches(collection2["manifest_text"],
676 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
678 self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
680 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
681 self.assertRegexpMatches(collection2["manifest_text"],
682 r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
685 def fuseRenameTestHelper(mounttmp):
686 class Test(unittest.TestCase):
688 os.mkdir(os.path.join(mounttmp, "testdir"))
690 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
691 f.write("Hello world!")
695 class FuseRenameTest(MountTestBase):
697 collection = arvados.collection.Collection(api_client=self.api)
698 collection.save_new()
700 m = self.make_mount(fuse.CollectionDirectory)
702 m.new_collection(collection.api_response(), collection)
703 self.assertTrue(m.writable())
705 self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
708 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
709 self.assertRegexpMatches(collection2["manifest_text"],
710 r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
712 d1 = llfuse.listdir(os.path.join(self.mounttmp))
713 self.assertEqual(["testdir"], d1)
714 d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
715 self.assertEqual(["file1.txt"], d1)
717 os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
719 d1 = llfuse.listdir(os.path.join(self.mounttmp))
720 self.assertEqual(["testdir2"], sorted(d1))
721 d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
722 self.assertEqual(["file1.txt"], d1)
724 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
725 self.assertRegexpMatches(collection2["manifest_text"],
726 r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
729 class FuseUpdateFromEventTest(MountTestBase):
731 collection = arvados.collection.Collection(api_client=self.api)
732 collection.save_new()
734 m = self.make_mount(fuse.CollectionDirectory)
736 m.new_collection(collection.api_response(), collection)
738 self.operations.listen_for_events()
740 d1 = llfuse.listdir(os.path.join(self.mounttmp))
741 self.assertEqual([], sorted(d1))
743 with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
744 with collection2.open("file1.txt", "w") as f:
747 for attempt in AssertWithTimeout(10):
748 attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
751 class FuseDeleteProjectEventTest(MountTestBase):
754 aproject = self.api.groups().create(body={
756 "group_class": "project"
759 bproject = self.api.groups().create(body={
761 "group_class": "project",
762 "owner_uuid": aproject["uuid"]
765 self.make_mount(fuse.ProjectDirectory,
766 project_object=self.api.users().current().execute())
768 self.operations.listen_for_events()
770 d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
771 self.assertEqual(["bproject"], sorted(d1))
773 self.api.groups().delete(uuid=bproject["uuid"]).execute()
775 for attempt in AssertWithTimeout(10):
776 attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
779 def fuseFileConflictTestHelper(mounttmp):
780 class Test(unittest.TestCase):
782 with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
785 d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
786 self.assertEqual(len(d1), 2)
788 with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
789 self.assertEqual(f.read(), "bar")
791 self.assertRegexpMatches(d1[1],
792 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
794 with open(os.path.join(mounttmp, d1[1]), "r") as f:
795 self.assertEqual(f.read(), "foo")
799 class FuseFileConflictTest(MountTestBase):
801 collection = arvados.collection.Collection(api_client=self.api)
802 collection.save_new()
804 m = self.make_mount(fuse.CollectionDirectory)
806 m.new_collection(collection.api_response(), collection)
808 d1 = llfuse.listdir(os.path.join(self.mounttmp))
809 self.assertEqual([], sorted(d1))
811 with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
812 with collection2.open("file1.txt", "w") as f:
815 # See note in MountTestBase.setUp
816 self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
819 def fuseUnlinkOpenFileTest(mounttmp):
820 class Test(unittest.TestCase):
822 with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
825 d1 = llfuse.listdir(os.path.join(mounttmp))
826 self.assertEqual(["file1.txt"], sorted(d1))
828 os.remove(os.path.join(mounttmp, "file1.txt"))
830 d1 = llfuse.listdir(os.path.join(mounttmp))
831 self.assertEqual([], sorted(d1))
834 self.assertEqual(f.read(), "foo")
838 self.assertEqual(f.read(), "foobar")
842 class FuseUnlinkOpenFileTest(MountTestBase):
844 collection = arvados.collection.Collection(api_client=self.api)
845 collection.save_new()
847 m = self.make_mount(fuse.CollectionDirectory)
849 m.new_collection(collection.api_response(), collection)
851 # See note in MountTestBase.setUp
852 self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
854 self.assertEqual(collection.manifest_text(), "")
857 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
858 class Test(unittest.TestCase):
860 with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
861 f.write("Hello world!")
863 d1 = os.listdir(os.path.join(mounttmp, uuid1))
864 self.assertEqual(["file1.txt"], sorted(d1))
865 d1 = os.listdir(os.path.join(mounttmp, uuid2))
866 self.assertEqual([], sorted(d1))
870 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
871 class Test(unittest.TestCase):
873 os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
875 d1 = os.listdir(os.path.join(mounttmp, uuid1))
876 self.assertEqual([], sorted(d1))
877 d1 = os.listdir(os.path.join(mounttmp, uuid2))
878 self.assertEqual(["file2.txt"], sorted(d1))
882 class FuseMvFileBetweenCollectionsTest(MountTestBase):
884 collection1 = arvados.collection.Collection(api_client=self.api)
885 collection1.save_new()
887 collection2 = arvados.collection.Collection(api_client=self.api)
888 collection2.save_new()
890 m = self.make_mount(fuse.MagicDirectory)
892 # See note in MountTestBase.setUp
893 self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
894 collection1.manifest_locator(),
895 collection2.manifest_locator()))
900 self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
901 self.assertEqual(collection2.manifest_text(), "")
903 self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
904 collection1.manifest_locator(),
905 collection2.manifest_locator()))
910 self.assertEqual(collection1.manifest_text(), "")
911 self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
913 collection1.stop_threads()
914 collection2.stop_threads()
917 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
918 class Test(unittest.TestCase):
920 os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
921 with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
922 f.write("Hello world!")
924 d1 = os.listdir(os.path.join(mounttmp, uuid1))
925 self.assertEqual(["testdir"], sorted(d1))
926 d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
927 self.assertEqual(["file1.txt"], sorted(d1))
929 d1 = os.listdir(os.path.join(mounttmp, uuid2))
930 self.assertEqual([], sorted(d1))
935 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
936 class Test(unittest.TestCase):
938 os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
940 d1 = os.listdir(os.path.join(mounttmp, uuid1))
941 self.assertEqual([], sorted(d1))
943 d1 = os.listdir(os.path.join(mounttmp, uuid2))
944 self.assertEqual(["testdir2"], sorted(d1))
945 d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
946 self.assertEqual(["file1.txt"], sorted(d1))
948 with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
949 self.assertEqual(f.read(), "Hello world!")
953 class FuseMvDirBetweenCollectionsTest(MountTestBase):
955 collection1 = arvados.collection.Collection(api_client=self.api)
956 collection1.save_new()
958 collection2 = arvados.collection.Collection(api_client=self.api)
959 collection2.save_new()
961 m = self.make_mount(fuse.MagicDirectory)
963 # See note in MountTestBase.setUp
964 self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
965 collection1.manifest_locator(),
966 collection2.manifest_locator()))
971 self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
972 self.assertEqual(collection2.manifest_text(), "")
974 self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
975 collection1.manifest_locator(),
976 collection2.manifest_locator()))
981 self.assertEqual(collection1.manifest_text(), "")
982 self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
984 collection1.stop_threads()
985 collection2.stop_threads()
987 def fuseProjectMkdirTestHelper1(mounttmp):
988 class Test(unittest.TestCase):
990 os.mkdir(os.path.join(mounttmp, "testcollection"))
991 with self.assertRaises(OSError):
992 os.mkdir(os.path.join(mounttmp, "testcollection"))
995 def fuseProjectMkdirTestHelper2(mounttmp):
996 class Test(unittest.TestCase):
998 with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
999 f.write("Hello world!")
1000 with self.assertRaises(OSError):
1001 os.rmdir(os.path.join(mounttmp, "testcollection"))
1002 os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1003 with self.assertRaises(OSError):
1004 os.remove(os.path.join(mounttmp, "testcollection"))
1005 os.rmdir(os.path.join(mounttmp, "testcollection"))
1008 class FuseProjectMkdirRmdirTest(MountTestBase):
1010 self.make_mount(fuse.ProjectDirectory,
1011 project_object=self.api.users().current().execute())
1013 d1 = llfuse.listdir(self.mounttmp)
1014 self.assertNotIn('testcollection', d1)
1016 self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1018 d1 = llfuse.listdir(self.mounttmp)
1019 self.assertIn('testcollection', d1)
1021 self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1023 d1 = llfuse.listdir(self.mounttmp)
1024 self.assertNotIn('testcollection', d1)
1027 def fuseProjectMvTestHelper1(mounttmp):
1028 class Test(unittest.TestCase):
1030 d1 = llfuse.listdir(mounttmp)
1031 self.assertNotIn('testcollection', d1)
1033 os.mkdir(os.path.join(mounttmp, "testcollection"))
1035 d1 = llfuse.listdir(mounttmp)
1036 self.assertIn('testcollection', d1)
1038 with self.assertRaises(OSError):
1039 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1041 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1043 d1 = llfuse.listdir(mounttmp)
1044 self.assertNotIn('testcollection', d1)
1046 d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1047 self.assertIn('testcollection', d1)
1051 class FuseProjectMvTest(MountTestBase):
1053 self.make_mount(fuse.ProjectDirectory,
1054 project_object=self.api.users().current().execute())
1056 self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1059 def fuseFsyncTestHelper(mounttmp, k):
1060 class Test(unittest.TestCase):
1062 fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1068 class FuseFsyncTest(FuseMagicTest):
1070 self.make_mount(fuse.MagicDirectory)
1071 self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1074 class MagicDirApiError(FuseMagicTest):
1076 api = mock.MagicMock()
1077 super(MagicDirApiError, self).setUp(api=api)
1078 api.collections().get().execute.side_effect = iter([
1079 Exception('API fail'),
1081 "manifest_text": self.test_manifest,
1082 "portable_data_hash": self.test_manifest_pdh,
1085 api.keep.get.side_effect = Exception('Keep fail')
1088 self.make_mount(fuse.MagicDirectory)
1090 self.operations.inodes.inode_cache.cap = 1
1091 self.operations.inodes.inode_cache.min_entries = 2
1093 with self.assertRaises(OSError):
1094 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1096 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1099 class FuseUnitTest(unittest.TestCase):
1100 def test_sanitize_filename(self):
1119 for f in acceptable:
1120 self.assertEqual(f, fuse.sanitize_filename(f))
1121 for f in unacceptable:
1122 self.assertNotEqual(f, fuse.sanitize_filename(f))
1123 # The sanitized filename should be the same length, though.
1124 self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1126 self.assertEqual("_", fuse.sanitize_filename(""))
1127 self.assertEqual("_", fuse.sanitize_filename("."))
1128 self.assertEqual("__", fuse.sanitize_filename(".."))
1131 class FuseMagicTestPDHOnly(MountTestBase):
1132 def setUp(self, api=None):
1133 super(FuseMagicTestPDHOnly, self).setUp(api=api)
1135 cw = arvados.CollectionWriter()
1137 cw.start_new_file('thing1.txt')
1140 self.testcollection = cw.finish()
1141 self.test_manifest = cw.manifest_text()
1142 created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1143 self.testcollectionuuid = str(created['uuid'])
1145 def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1146 if skip_pdh_only is True:
1147 self.make_mount(fuse.MagicDirectory) # in this case, the default by_id applies
1149 self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1151 mount_ls = llfuse.listdir(self.mounttmp)
1152 self.assertIn('README', mount_ls)
1153 self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1154 arvados.util.uuid_pattern.match(fn)
1155 for fn in mount_ls),
1156 "new FUSE MagicDirectory lists Collection")
1158 # look up using pdh should succeed in all cases
1159 self.assertDirContents(self.testcollection, ['thing1.txt'])
1160 self.assertDirContents(os.path.join('by_id', self.testcollection),
1162 mount_ls = llfuse.listdir(self.mounttmp)
1163 self.assertIn('README', mount_ls)
1164 self.assertIn(self.testcollection, mount_ls)
1165 self.assertIn(self.testcollection,
1166 llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1169 files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = b'data 1'
1171 for k, v in viewitems(files):
1172 with open(os.path.join(self.mounttmp, k), 'rb') as f:
1173 self.assertEqual(v, f.read())
1175 # look up using uuid should fail when pdh_only is set
1176 if pdh_only is True:
1177 with self.assertRaises(OSError):
1178 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1181 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1184 def test_with_pdh_only_true(self):
1185 self.verify_pdh_only(pdh_only=True)
1187 def test_with_pdh_only_false(self):
1188 self.verify_pdh_only(pdh_only=False)
1190 def test_with_default_by_id(self):
1191 self.verify_pdh_only(skip_pdh_only=True)