Merge branch 'master' of git.curoverse.com:arvados into 11876-r-sdk
[arvados.git] / services / fuse / tests / test_mount.py
1 # Copyright (C) The Arvados Authors. All rights reserved.
2 #
3 # SPDX-License-Identifier: AGPL-3.0
4
5 import json
6 import llfuse
7 import logging
8 import mock
9 import os
10 import subprocess
11 import time
12 import unittest
13
14 import arvados
15 import arvados_fuse as fuse
16 import run_test_server
17
18 from mount_test_base import MountTestBase
19
20 logger = logging.getLogger('arvados.arv-mount')
21
22
23 class AssertWithTimeout(object):
24     """Allow some time for an assertion to pass."""
25
26     def __init__(self, timeout=0):
27         self.timeout = timeout
28
29     def __iter__(self):
30         self.deadline = time.time() + self.timeout
31         self.done = False
32         return self
33
34     def next(self):
35         if self.done:
36             raise StopIteration
37         return self.attempt
38
39     def attempt(self, fn, *args, **kwargs):
40         try:
41             fn(*args, **kwargs)
42         except AssertionError:
43             if time.time() > self.deadline:
44                 raise
45             time.sleep(0.1)
46         else:
47             self.done = True
48
49
50 class FuseMountTest(MountTestBase):
51     def setUp(self):
52         super(FuseMountTest, self).setUp()
53
54         cw = arvados.CollectionWriter()
55
56         cw.start_new_file('thing1.txt')
57         cw.write("data 1")
58         cw.start_new_file('thing2.txt')
59         cw.write("data 2")
60
61         cw.start_new_stream('dir1')
62         cw.start_new_file('thing3.txt')
63         cw.write("data 3")
64         cw.start_new_file('thing4.txt')
65         cw.write("data 4")
66
67         cw.start_new_stream('dir2')
68         cw.start_new_file('thing5.txt')
69         cw.write("data 5")
70         cw.start_new_file('thing6.txt')
71         cw.write("data 6")
72
73         cw.start_new_stream('dir2/dir3')
74         cw.start_new_file('thing7.txt')
75         cw.write("data 7")
76
77         cw.start_new_file('thing8.txt')
78         cw.write("data 8")
79
80         cw.start_new_stream('edgecases')
81         for f in ":/.../-/*/\x01\\/ ".split("/"):
82             cw.start_new_file(f)
83             cw.write('x')
84
85         for f in ":/.../-/*/\x01\\/ ".split("/"):
86             cw.start_new_stream('edgecases/dirs/' + f)
87             cw.start_new_file('x/x')
88             cw.write('x')
89
90         self.testcollection = cw.finish()
91         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
92
93     def runTest(self):
94         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
95
96         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
97                                       'edgecases', 'dir1', 'dir2'])
98         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
99         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
100         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
101         self.assertDirContents('edgecases',
102                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
103         self.assertDirContents('edgecases/dirs',
104                                ":/.../-/*/\x01\\/ ".split("/"))
105
106         files = {'thing1.txt': 'data 1',
107                  'thing2.txt': 'data 2',
108                  'dir1/thing3.txt': 'data 3',
109                  'dir1/thing4.txt': 'data 4',
110                  'dir2/thing5.txt': 'data 5',
111                  'dir2/thing6.txt': 'data 6',
112                  'dir2/dir3/thing7.txt': 'data 7',
113                  'dir2/dir3/thing8.txt': 'data 8'}
114
115         for k, v in files.items():
116             with open(os.path.join(self.mounttmp, k)) as f:
117                 self.assertEqual(v, f.read())
118
119
120 class FuseMagicTest(MountTestBase):
121     def setUp(self, api=None):
122         super(FuseMagicTest, self).setUp(api=api)
123
124         cw = arvados.CollectionWriter()
125
126         cw.start_new_file('thing1.txt')
127         cw.write("data 1")
128
129         self.testcollection = cw.finish()
130         self.test_manifest = cw.manifest_text()
131         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
132         self.test_manifest_pdh = coll['portable_data_hash']
133
134     def runTest(self):
135         self.make_mount(fuse.MagicDirectory)
136
137         mount_ls = llfuse.listdir(self.mounttmp)
138         self.assertIn('README', mount_ls)
139         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
140                              arvados.util.uuid_pattern.match(fn)
141                              for fn in mount_ls),
142                          "new FUSE MagicDirectory lists Collection")
143         self.assertDirContents(self.testcollection, ['thing1.txt'])
144         self.assertDirContents(os.path.join('by_id', self.testcollection),
145                                ['thing1.txt'])
146         mount_ls = llfuse.listdir(self.mounttmp)
147         self.assertIn('README', mount_ls)
148         self.assertIn(self.testcollection, mount_ls)
149         self.assertIn(self.testcollection,
150                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
151
152         files = {}
153         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
154
155         for k, v in files.items():
156             with open(os.path.join(self.mounttmp, k)) as f:
157                 self.assertEqual(v, f.read())
158
159
160 class FuseTagsTest(MountTestBase):
161     def runTest(self):
162         self.make_mount(fuse.TagsDirectory)
163
164         d1 = llfuse.listdir(self.mounttmp)
165         d1.sort()
166         self.assertEqual(['foo_tag'], d1)
167
168         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
169         d2.sort()
170         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
171
172         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
173         d3.sort()
174         self.assertEqual(['foo'], d3)
175
176
177 class FuseTagsUpdateTest(MountTestBase):
178     def tag_collection(self, coll_uuid, tag_name):
179         return self.api.links().create(
180             body={'link': {'head_uuid': coll_uuid,
181                            'link_class': 'tag',
182                            'name': tag_name,
183         }}).execute()
184
185     def runTest(self):
186         self.make_mount(fuse.TagsDirectory, poll_time=1)
187
188         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
189
190         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
191         self.tag_collection(bar_uuid, 'fuse_test_tag')
192         for attempt in AssertWithTimeout(10):
193             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
194         self.assertDirContents('fuse_test_tag', [bar_uuid])
195
196         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
197         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
198         for attempt in AssertWithTimeout(10):
199             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
200
201         self.api.links().delete(uuid=l['uuid']).execute()
202         for attempt in AssertWithTimeout(10):
203             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
204
205
206 def fuseSharedTestHelper(mounttmp):
207     class Test(unittest.TestCase):
208         def runTest(self):
209             # Double check that we can open and read objects in this folder as a file,
210             # and that its contents are what we expect.
211             baz_path = os.path.join(
212                 mounttmp,
213                 'FUSE User',
214                 'FUSE Test Project',
215                 'collection in FUSE project',
216                 'baz')
217             with open(baz_path) as f:
218                 self.assertEqual("baz", f.read())
219
220             # check mtime on collection
221             st = os.stat(baz_path)
222             try:
223                 mtime = st.st_mtime_ns / 1000000000
224             except AttributeError:
225                 mtime = st.st_mtime
226             self.assertEqual(mtime, 1391448174)
227
228             # shared_dirs is a list of the directories exposed
229             # by fuse.SharedDirectory (i.e. any object visible
230             # to the current user)
231             shared_dirs = llfuse.listdir(mounttmp)
232             shared_dirs.sort()
233             self.assertIn('FUSE User', shared_dirs)
234
235             # fuse_user_objs is a list of the objects owned by the FUSE
236             # test user (which present as files in the 'FUSE User'
237             # directory)
238             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
239             fuse_user_objs.sort()
240             self.assertEqual(['FUSE Test Project',                    # project owned by user
241                               'collection #1 owned by FUSE',          # collection owned by user
242                               'collection #2 owned by FUSE'          # collection owned by user
243                           ], fuse_user_objs)
244
245             # test_proj_files is a list of the files in the FUSE Test Project.
246             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
247             test_proj_files.sort()
248             self.assertEqual(['collection in FUSE project'
249                           ], test_proj_files)
250
251
252     Test().runTest()
253
254 class FuseSharedTest(MountTestBase):
255     def runTest(self):
256         self.make_mount(fuse.SharedDirectory,
257                         exclude=self.api.users().current().execute()['uuid'])
258         keep = arvados.keep.KeepClient()
259         keep.put("baz")
260
261         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
262
263
264 class FuseHomeTest(MountTestBase):
265     def runTest(self):
266         self.make_mount(fuse.ProjectDirectory,
267                         project_object=self.api.users().current().execute())
268
269         d1 = llfuse.listdir(self.mounttmp)
270         self.assertIn('Unrestricted public data', d1)
271
272         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
273         public_project = run_test_server.fixture('groups')[
274             'anonymously_accessible_project']
275         found_in = 0
276         found_not_in = 0
277         for name, item in run_test_server.fixture('collections').iteritems():
278             if 'name' not in item:
279                 pass
280             elif item['owner_uuid'] == public_project['uuid']:
281                 self.assertIn(item['name'], d2)
282                 found_in += 1
283             else:
284                 # Artificial assumption here: there is no public
285                 # collection fixture with the same name as a
286                 # non-public collection.
287                 self.assertNotIn(item['name'], d2)
288                 found_not_in += 1
289         self.assertNotEqual(0, found_in)
290         self.assertNotEqual(0, found_not_in)
291
292         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
293         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
294
295
296 def fuseModifyFileTestHelperReadStartContents(mounttmp):
297     class Test(unittest.TestCase):
298         def runTest(self):
299             d1 = llfuse.listdir(mounttmp)
300             self.assertEqual(["file1.txt"], d1)
301             with open(os.path.join(mounttmp, "file1.txt")) as f:
302                 self.assertEqual("blub", f.read())
303     Test().runTest()
304
305 def fuseModifyFileTestHelperReadEndContents(mounttmp):
306     class Test(unittest.TestCase):
307         def runTest(self):
308             d1 = llfuse.listdir(mounttmp)
309             self.assertEqual(["file1.txt"], d1)
310             with open(os.path.join(mounttmp, "file1.txt")) as f:
311                 self.assertEqual("plnp", f.read())
312     Test().runTest()
313
314 class FuseModifyFileTest(MountTestBase):
315     def runTest(self):
316         collection = arvados.collection.Collection(api_client=self.api)
317         with collection.open("file1.txt", "w") as f:
318             f.write("blub")
319
320         collection.save_new()
321
322         m = self.make_mount(fuse.CollectionDirectory)
323         with llfuse.lock:
324             m.new_collection(collection.api_response(), collection)
325
326         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
327
328         with collection.open("file1.txt", "w") as f:
329             f.write("plnp")
330
331         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
332
333
334 class FuseAddFileToCollectionTest(MountTestBase):
335     def runTest(self):
336         collection = arvados.collection.Collection(api_client=self.api)
337         with collection.open("file1.txt", "w") as f:
338             f.write("blub")
339
340         collection.save_new()
341
342         m = self.make_mount(fuse.CollectionDirectory)
343         with llfuse.lock:
344             m.new_collection(collection.api_response(), collection)
345
346         d1 = llfuse.listdir(self.mounttmp)
347         self.assertEqual(["file1.txt"], d1)
348
349         with collection.open("file2.txt", "w") as f:
350             f.write("plnp")
351
352         d1 = llfuse.listdir(self.mounttmp)
353         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
354
355
356 class FuseRemoveFileFromCollectionTest(MountTestBase):
357     def runTest(self):
358         collection = arvados.collection.Collection(api_client=self.api)
359         with collection.open("file1.txt", "w") as f:
360             f.write("blub")
361
362         with collection.open("file2.txt", "w") as f:
363             f.write("plnp")
364
365         collection.save_new()
366
367         m = self.make_mount(fuse.CollectionDirectory)
368         with llfuse.lock:
369             m.new_collection(collection.api_response(), collection)
370
371         d1 = llfuse.listdir(self.mounttmp)
372         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
373
374         collection.remove("file2.txt")
375
376         d1 = llfuse.listdir(self.mounttmp)
377         self.assertEqual(["file1.txt"], d1)
378
379
380 def fuseCreateFileTestHelper(mounttmp):
381     class Test(unittest.TestCase):
382         def runTest(self):
383             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
384                 pass
385     Test().runTest()
386
387 class FuseCreateFileTest(MountTestBase):
388     def runTest(self):
389         collection = arvados.collection.Collection(api_client=self.api)
390         collection.save_new()
391
392         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
393         self.assertEqual(collection2["manifest_text"], "")
394
395         collection.save_new()
396
397         m = self.make_mount(fuse.CollectionDirectory)
398         with llfuse.lock:
399             m.new_collection(collection.api_response(), collection)
400         self.assertTrue(m.writable())
401
402         self.assertNotIn("file1.txt", collection)
403
404         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
405
406         self.assertIn("file1.txt", collection)
407
408         d1 = llfuse.listdir(self.mounttmp)
409         self.assertEqual(["file1.txt"], d1)
410
411         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
412         self.assertRegexpMatches(collection2["manifest_text"],
413             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
414
415
416 def fuseWriteFileTestHelperWriteFile(mounttmp):
417     class Test(unittest.TestCase):
418         def runTest(self):
419             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
420                 f.write("Hello world!")
421     Test().runTest()
422
423 def fuseWriteFileTestHelperReadFile(mounttmp):
424     class Test(unittest.TestCase):
425         def runTest(self):
426             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
427                 self.assertEqual(f.read(), "Hello world!")
428     Test().runTest()
429
430 class FuseWriteFileTest(MountTestBase):
431     def runTest(self):
432         collection = arvados.collection.Collection(api_client=self.api)
433         collection.save_new()
434
435         m = self.make_mount(fuse.CollectionDirectory)
436         with llfuse.lock:
437             m.new_collection(collection.api_response(), collection)
438         self.assertTrue(m.writable())
439
440         self.assertNotIn("file1.txt", collection)
441
442         self.assertEqual(0, self.operations.write_counter.get())
443         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
444         self.assertEqual(12, self.operations.write_counter.get())
445
446         with collection.open("file1.txt") as f:
447             self.assertEqual(f.read(), "Hello world!")
448
449         self.assertEqual(0, self.operations.read_counter.get())
450         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
451         self.assertEqual(12, self.operations.read_counter.get())
452
453         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
454         self.assertRegexpMatches(collection2["manifest_text"],
455             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
456
457
458 def fuseUpdateFileTestHelper(mounttmp):
459     class Test(unittest.TestCase):
460         def runTest(self):
461             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
462                 f.write("Hello world!")
463
464             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
465                 fr = f.read()
466                 self.assertEqual(fr, "Hello world!")
467                 f.seek(0)
468                 f.write("Hola mundo!")
469                 f.seek(0)
470                 fr = f.read()
471                 self.assertEqual(fr, "Hola mundo!!")
472
473             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
474                 self.assertEqual(f.read(), "Hola mundo!!")
475
476     Test().runTest()
477
478 class FuseUpdateFileTest(MountTestBase):
479     def runTest(self):
480         collection = arvados.collection.Collection(api_client=self.api)
481         collection.save_new()
482
483         m = self.make_mount(fuse.CollectionDirectory)
484         with llfuse.lock:
485             m.new_collection(collection.api_response(), collection)
486         self.assertTrue(m.writable())
487
488         # See note in MountTestBase.setUp
489         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
490
491         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
492         self.assertRegexpMatches(collection2["manifest_text"],
493             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
494
495
496 def fuseMkdirTestHelper(mounttmp):
497     class Test(unittest.TestCase):
498         def runTest(self):
499             with self.assertRaises(IOError):
500                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
501                     f.write("Hello world!")
502
503             os.mkdir(os.path.join(mounttmp, "testdir"))
504
505             with self.assertRaises(OSError):
506                 os.mkdir(os.path.join(mounttmp, "testdir"))
507
508             d1 = llfuse.listdir(mounttmp)
509             self.assertEqual(["testdir"], d1)
510
511             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
512                 f.write("Hello world!")
513
514             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
515             self.assertEqual(["file1.txt"], d1)
516
517     Test().runTest()
518
519 class FuseMkdirTest(MountTestBase):
520     def runTest(self):
521         collection = arvados.collection.Collection(api_client=self.api)
522         collection.save_new()
523
524         m = self.make_mount(fuse.CollectionDirectory)
525         with llfuse.lock:
526             m.new_collection(collection.api_response(), collection)
527         self.assertTrue(m.writable())
528
529         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
530
531         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
532         self.assertRegexpMatches(collection2["manifest_text"],
533             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
534
535
536 def fuseRmTestHelperWriteFile(mounttmp):
537     class Test(unittest.TestCase):
538         def runTest(self):
539             os.mkdir(os.path.join(mounttmp, "testdir"))
540
541             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
542                 f.write("Hello world!")
543
544     Test().runTest()
545
546 def fuseRmTestHelperDeleteFile(mounttmp):
547     class Test(unittest.TestCase):
548         def runTest(self):
549             # Can't delete because it's not empty
550             with self.assertRaises(OSError):
551                 os.rmdir(os.path.join(mounttmp, "testdir"))
552
553             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
554             self.assertEqual(["file1.txt"], d1)
555
556             # Delete file
557             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
558
559             # Make sure it's empty
560             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
561             self.assertEqual([], d1)
562
563             # Try to delete it again
564             with self.assertRaises(OSError):
565                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
566
567     Test().runTest()
568
569 def fuseRmTestHelperRmdir(mounttmp):
570     class Test(unittest.TestCase):
571         def runTest(self):
572             # Should be able to delete now that it is empty
573             os.rmdir(os.path.join(mounttmp, "testdir"))
574
575             # Make sure it's empty
576             d1 = llfuse.listdir(os.path.join(mounttmp))
577             self.assertEqual([], d1)
578
579             # Try to delete it again
580             with self.assertRaises(OSError):
581                 os.rmdir(os.path.join(mounttmp, "testdir"))
582
583     Test().runTest()
584
585 class FuseRmTest(MountTestBase):
586     def runTest(self):
587         collection = arvados.collection.Collection(api_client=self.api)
588         collection.save_new()
589
590         m = self.make_mount(fuse.CollectionDirectory)
591         with llfuse.lock:
592             m.new_collection(collection.api_response(), collection)
593         self.assertTrue(m.writable())
594
595         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
596
597         # Starting manifest
598         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
599         self.assertRegexpMatches(collection2["manifest_text"],
600             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
601         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
602
603         # Can't have empty directories :-( so manifest will be empty.
604         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
605         self.assertEqual(collection2["manifest_text"], "")
606
607         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
608
609         # manifest should be empty now.
610         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
611         self.assertEqual(collection2["manifest_text"], "")
612
613
614 def fuseMvFileTestHelperWriteFile(mounttmp):
615     class Test(unittest.TestCase):
616         def runTest(self):
617             os.mkdir(os.path.join(mounttmp, "testdir"))
618
619             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
620                 f.write("Hello world!")
621
622     Test().runTest()
623
624 def fuseMvFileTestHelperMoveFile(mounttmp):
625     class Test(unittest.TestCase):
626         def runTest(self):
627             d1 = llfuse.listdir(os.path.join(mounttmp))
628             self.assertEqual(["testdir"], d1)
629             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
630             self.assertEqual(["file1.txt"], d1)
631
632             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
633
634             d1 = llfuse.listdir(os.path.join(mounttmp))
635             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
636             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
637             self.assertEqual([], d1)
638
639     Test().runTest()
640
641 class FuseMvFileTest(MountTestBase):
642     def runTest(self):
643         collection = arvados.collection.Collection(api_client=self.api)
644         collection.save_new()
645
646         m = self.make_mount(fuse.CollectionDirectory)
647         with llfuse.lock:
648             m.new_collection(collection.api_response(), collection)
649         self.assertTrue(m.writable())
650
651         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
652
653         # Starting manifest
654         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
655         self.assertRegexpMatches(collection2["manifest_text"],
656             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
657
658         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
659
660         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
661         self.assertRegexpMatches(collection2["manifest_text"],
662             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
663
664
665 def fuseRenameTestHelper(mounttmp):
666     class Test(unittest.TestCase):
667         def runTest(self):
668             os.mkdir(os.path.join(mounttmp, "testdir"))
669
670             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
671                 f.write("Hello world!")
672
673     Test().runTest()
674
675 class FuseRenameTest(MountTestBase):
676     def runTest(self):
677         collection = arvados.collection.Collection(api_client=self.api)
678         collection.save_new()
679
680         m = self.make_mount(fuse.CollectionDirectory)
681         with llfuse.lock:
682             m.new_collection(collection.api_response(), collection)
683         self.assertTrue(m.writable())
684
685         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
686
687         # Starting manifest
688         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
689         self.assertRegexpMatches(collection2["manifest_text"],
690             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
691
692         d1 = llfuse.listdir(os.path.join(self.mounttmp))
693         self.assertEqual(["testdir"], d1)
694         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
695         self.assertEqual(["file1.txt"], d1)
696
697         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
698
699         d1 = llfuse.listdir(os.path.join(self.mounttmp))
700         self.assertEqual(["testdir2"], sorted(d1))
701         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
702         self.assertEqual(["file1.txt"], d1)
703
704         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
705         self.assertRegexpMatches(collection2["manifest_text"],
706             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
707
708
709 class FuseUpdateFromEventTest(MountTestBase):
710     def runTest(self):
711         collection = arvados.collection.Collection(api_client=self.api)
712         collection.save_new()
713
714         m = self.make_mount(fuse.CollectionDirectory)
715         with llfuse.lock:
716             m.new_collection(collection.api_response(), collection)
717
718         self.operations.listen_for_events()
719
720         d1 = llfuse.listdir(os.path.join(self.mounttmp))
721         self.assertEqual([], sorted(d1))
722
723         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
724             with collection2.open("file1.txt", "w") as f:
725                 f.write("foo")
726
727         for attempt in AssertWithTimeout(10):
728             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
729
730
731 class FuseDeleteProjectEventTest(MountTestBase):
732     def runTest(self):
733
734         aproject = self.api.groups().create(body={
735             "name": "aproject",
736             "group_class": "project"
737         }).execute()
738
739         bproject = self.api.groups().create(body={
740             "name": "bproject",
741             "group_class": "project",
742             "owner_uuid": aproject["uuid"]
743         }).execute()
744
745         self.make_mount(fuse.ProjectDirectory,
746                         project_object=self.api.users().current().execute())
747
748         self.operations.listen_for_events()
749
750         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
751         self.assertEqual(["bproject"], sorted(d1))
752
753         self.api.groups().delete(uuid=bproject["uuid"]).execute()
754
755         for attempt in AssertWithTimeout(10):
756             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
757
758
759 def fuseFileConflictTestHelper(mounttmp):
760     class Test(unittest.TestCase):
761         def runTest(self):
762             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
763                 f.write("bar")
764
765             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
766             self.assertEqual(len(d1), 2)
767
768             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
769                 self.assertEqual(f.read(), "bar")
770
771             self.assertRegexpMatches(d1[1],
772                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
773
774             with open(os.path.join(mounttmp, d1[1]), "r") as f:
775                 self.assertEqual(f.read(), "foo")
776
777     Test().runTest()
778
779 class FuseFileConflictTest(MountTestBase):
780     def runTest(self):
781         collection = arvados.collection.Collection(api_client=self.api)
782         collection.save_new()
783
784         m = self.make_mount(fuse.CollectionDirectory)
785         with llfuse.lock:
786             m.new_collection(collection.api_response(), collection)
787
788         d1 = llfuse.listdir(os.path.join(self.mounttmp))
789         self.assertEqual([], sorted(d1))
790
791         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
792             with collection2.open("file1.txt", "w") as f:
793                 f.write("foo")
794
795         # See note in MountTestBase.setUp
796         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
797
798
799 def fuseUnlinkOpenFileTest(mounttmp):
800     class Test(unittest.TestCase):
801         def runTest(self):
802             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
803                 f.write("foo")
804
805                 d1 = llfuse.listdir(os.path.join(mounttmp))
806                 self.assertEqual(["file1.txt"], sorted(d1))
807
808                 os.remove(os.path.join(mounttmp, "file1.txt"))
809
810                 d1 = llfuse.listdir(os.path.join(mounttmp))
811                 self.assertEqual([], sorted(d1))
812
813                 f.seek(0)
814                 self.assertEqual(f.read(), "foo")
815                 f.write("bar")
816
817                 f.seek(0)
818                 self.assertEqual(f.read(), "foobar")
819
820     Test().runTest()
821
822 class FuseUnlinkOpenFileTest(MountTestBase):
823     def runTest(self):
824         collection = arvados.collection.Collection(api_client=self.api)
825         collection.save_new()
826
827         m = self.make_mount(fuse.CollectionDirectory)
828         with llfuse.lock:
829             m.new_collection(collection.api_response(), collection)
830
831         # See note in MountTestBase.setUp
832         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
833
834         self.assertEqual(collection.manifest_text(), "")
835
836
837 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
838     class Test(unittest.TestCase):
839         def runTest(self):
840             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
841                 f.write("Hello world!")
842
843             d1 = os.listdir(os.path.join(mounttmp, uuid1))
844             self.assertEqual(["file1.txt"], sorted(d1))
845             d1 = os.listdir(os.path.join(mounttmp, uuid2))
846             self.assertEqual([], sorted(d1))
847
848     Test().runTest()
849
850 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
851     class Test(unittest.TestCase):
852         def runTest(self):
853             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
854
855             d1 = os.listdir(os.path.join(mounttmp, uuid1))
856             self.assertEqual([], sorted(d1))
857             d1 = os.listdir(os.path.join(mounttmp, uuid2))
858             self.assertEqual(["file2.txt"], sorted(d1))
859
860     Test().runTest()
861
862 class FuseMvFileBetweenCollectionsTest(MountTestBase):
863     def runTest(self):
864         collection1 = arvados.collection.Collection(api_client=self.api)
865         collection1.save_new()
866
867         collection2 = arvados.collection.Collection(api_client=self.api)
868         collection2.save_new()
869
870         m = self.make_mount(fuse.MagicDirectory)
871
872         # See note in MountTestBase.setUp
873         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
874                                                   collection1.manifest_locator(),
875                                                   collection2.manifest_locator()))
876
877         collection1.update()
878         collection2.update()
879
880         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
881         self.assertEqual(collection2.manifest_text(), "")
882
883         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
884                                                   collection1.manifest_locator(),
885                                                   collection2.manifest_locator()))
886
887         collection1.update()
888         collection2.update()
889
890         self.assertEqual(collection1.manifest_text(), "")
891         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
892
893         collection1.stop_threads()
894         collection2.stop_threads()
895
896
897 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
898     class Test(unittest.TestCase):
899         def runTest(self):
900             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
901             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
902                 f.write("Hello world!")
903
904             d1 = os.listdir(os.path.join(mounttmp, uuid1))
905             self.assertEqual(["testdir"], sorted(d1))
906             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
907             self.assertEqual(["file1.txt"], sorted(d1))
908
909             d1 = os.listdir(os.path.join(mounttmp, uuid2))
910             self.assertEqual([], sorted(d1))
911
912     Test().runTest()
913
914
915 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
916     class Test(unittest.TestCase):
917         def runTest(self):
918             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
919
920             d1 = os.listdir(os.path.join(mounttmp, uuid1))
921             self.assertEqual([], sorted(d1))
922
923             d1 = os.listdir(os.path.join(mounttmp, uuid2))
924             self.assertEqual(["testdir2"], sorted(d1))
925             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
926             self.assertEqual(["file1.txt"], sorted(d1))
927
928             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
929                 self.assertEqual(f.read(), "Hello world!")
930
931     Test().runTest()
932
933 class FuseMvDirBetweenCollectionsTest(MountTestBase):
934     def runTest(self):
935         collection1 = arvados.collection.Collection(api_client=self.api)
936         collection1.save_new()
937
938         collection2 = arvados.collection.Collection(api_client=self.api)
939         collection2.save_new()
940
941         m = self.make_mount(fuse.MagicDirectory)
942
943         # See note in MountTestBase.setUp
944         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
945                                                   collection1.manifest_locator(),
946                                                   collection2.manifest_locator()))
947
948         collection1.update()
949         collection2.update()
950
951         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
952         self.assertEqual(collection2.manifest_text(), "")
953
954         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
955                                                   collection1.manifest_locator(),
956                                                   collection2.manifest_locator()))
957
958         collection1.update()
959         collection2.update()
960
961         self.assertEqual(collection1.manifest_text(), "")
962         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
963
964         collection1.stop_threads()
965         collection2.stop_threads()
966
967 def fuseProjectMkdirTestHelper1(mounttmp):
968     class Test(unittest.TestCase):
969         def runTest(self):
970             os.mkdir(os.path.join(mounttmp, "testcollection"))
971             with self.assertRaises(OSError):
972                 os.mkdir(os.path.join(mounttmp, "testcollection"))
973     Test().runTest()
974
975 def fuseProjectMkdirTestHelper2(mounttmp):
976     class Test(unittest.TestCase):
977         def runTest(self):
978             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
979                 f.write("Hello world!")
980             with self.assertRaises(OSError):
981                 os.rmdir(os.path.join(mounttmp, "testcollection"))
982             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
983             with self.assertRaises(OSError):
984                 os.remove(os.path.join(mounttmp, "testcollection"))
985             os.rmdir(os.path.join(mounttmp, "testcollection"))
986     Test().runTest()
987
988 class FuseProjectMkdirRmdirTest(MountTestBase):
989     def runTest(self):
990         self.make_mount(fuse.ProjectDirectory,
991                         project_object=self.api.users().current().execute())
992
993         d1 = llfuse.listdir(self.mounttmp)
994         self.assertNotIn('testcollection', d1)
995
996         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
997
998         d1 = llfuse.listdir(self.mounttmp)
999         self.assertIn('testcollection', d1)
1000
1001         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1002
1003         d1 = llfuse.listdir(self.mounttmp)
1004         self.assertNotIn('testcollection', d1)
1005
1006
1007 def fuseProjectMvTestHelper1(mounttmp):
1008     class Test(unittest.TestCase):
1009         def runTest(self):
1010             d1 = llfuse.listdir(mounttmp)
1011             self.assertNotIn('testcollection', d1)
1012
1013             os.mkdir(os.path.join(mounttmp, "testcollection"))
1014
1015             d1 = llfuse.listdir(mounttmp)
1016             self.assertIn('testcollection', d1)
1017
1018             with self.assertRaises(OSError):
1019                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1020
1021             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1022
1023             d1 = llfuse.listdir(mounttmp)
1024             self.assertNotIn('testcollection', d1)
1025
1026             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1027             self.assertIn('testcollection', d1)
1028
1029     Test().runTest()
1030
1031 class FuseProjectMvTest(MountTestBase):
1032     def runTest(self):
1033         self.make_mount(fuse.ProjectDirectory,
1034                         project_object=self.api.users().current().execute())
1035
1036         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1037
1038
1039 def fuseFsyncTestHelper(mounttmp, k):
1040     class Test(unittest.TestCase):
1041         def runTest(self):
1042             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1043             os.fsync(fd)
1044             os.close(fd)
1045
1046     Test().runTest()
1047
1048 class FuseFsyncTest(FuseMagicTest):
1049     def runTest(self):
1050         self.make_mount(fuse.MagicDirectory)
1051         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1052
1053
1054 class MagicDirApiError(FuseMagicTest):
1055     def setUp(self):
1056         api = mock.MagicMock()
1057         super(MagicDirApiError, self).setUp(api=api)
1058         api.collections().get().execute.side_effect = iter([
1059             Exception('API fail'),
1060             {
1061                 "manifest_text": self.test_manifest,
1062                 "portable_data_hash": self.test_manifest_pdh,
1063             },
1064         ])
1065         api.keep.get.side_effect = Exception('Keep fail')
1066
1067     def runTest(self):
1068         self.make_mount(fuse.MagicDirectory)
1069
1070         self.operations.inodes.inode_cache.cap = 1
1071         self.operations.inodes.inode_cache.min_entries = 2
1072
1073         with self.assertRaises(OSError):
1074             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1075
1076         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1077
1078
1079 class FuseUnitTest(unittest.TestCase):
1080     def test_sanitize_filename(self):
1081         acceptable = [
1082             "foo.txt",
1083             ".foo",
1084             "..foo",
1085             "...",
1086             "foo...",
1087             "foo..",
1088             "foo.",
1089             "-",
1090             "\x01\x02\x03",
1091             ]
1092         unacceptable = [
1093             "f\00",
1094             "\00\00",
1095             "/foo",
1096             "foo/",
1097             "//",
1098             ]
1099         for f in acceptable:
1100             self.assertEqual(f, fuse.sanitize_filename(f))
1101         for f in unacceptable:
1102             self.assertNotEqual(f, fuse.sanitize_filename(f))
1103             # The sanitized filename should be the same length, though.
1104             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1105         # Special cases
1106         self.assertEqual("_", fuse.sanitize_filename(""))
1107         self.assertEqual("_", fuse.sanitize_filename("."))
1108         self.assertEqual("__", fuse.sanitize_filename(".."))
1109
1110
1111 class FuseMagicTestPDHOnly(MountTestBase):
1112     def setUp(self, api=None):
1113         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1114
1115         cw = arvados.CollectionWriter()
1116
1117         cw.start_new_file('thing1.txt')
1118         cw.write("data 1")
1119
1120         self.testcollection = cw.finish()
1121         self.test_manifest = cw.manifest_text()
1122         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1123         self.testcollectionuuid = str(created['uuid'])
1124
1125     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1126         if skip_pdh_only is True:
1127             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1128         else:
1129             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1130
1131         mount_ls = llfuse.listdir(self.mounttmp)
1132         self.assertIn('README', mount_ls)
1133         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1134                              arvados.util.uuid_pattern.match(fn)
1135                              for fn in mount_ls),
1136                          "new FUSE MagicDirectory lists Collection")
1137
1138         # look up using pdh should succeed in all cases
1139         self.assertDirContents(self.testcollection, ['thing1.txt'])
1140         self.assertDirContents(os.path.join('by_id', self.testcollection),
1141                                ['thing1.txt'])
1142         mount_ls = llfuse.listdir(self.mounttmp)
1143         self.assertIn('README', mount_ls)
1144         self.assertIn(self.testcollection, mount_ls)
1145         self.assertIn(self.testcollection,
1146                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1147
1148         files = {}
1149         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1150
1151         for k, v in files.items():
1152             with open(os.path.join(self.mounttmp, k)) as f:
1153                 self.assertEqual(v, f.read())
1154
1155         # look up using uuid should fail when pdh_only is set
1156         if pdh_only is True:
1157             with self.assertRaises(OSError):
1158                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1159                                ['thing1.txt'])
1160         else:
1161             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1162                                ['thing1.txt'])
1163
1164     def test_with_pdh_only_true(self):
1165         self.verify_pdh_only(pdh_only=True)
1166
1167     def test_with_pdh_only_false(self):
1168         self.verify_pdh_only(pdh_only=False)
1169
1170     def test_with_default_by_id(self):
1171         self.verify_pdh_only(skip_pdh_only=True)