20862: Add 'sdk/ruby-google-api-client/' from commit '2f4be67955e48bb65d008ecd9ff6da9...
[arvados.git] / services / fuse / tests / test_mount.py
1 # Copyright (C) The Arvados Authors. All rights reserved.
2 #
3 # SPDX-License-Identifier: AGPL-3.0
4
5 from __future__ import absolute_import
6 from future.utils import viewitems
7 from builtins import str
8 from builtins import object
9 from pathlib import Path
10 from six import assertRegex
11 import errno
12 import json
13 import llfuse
14 import logging
15 import mock
16 import os
17 import subprocess
18 import time
19 import unittest
20 import tempfile
21 import parameterized
22
23 import arvados
24 import arvados_fuse as fuse
25 from arvados_fuse import fusedir
26 from . import run_test_server
27
28 from .integration_test import IntegrationTest
29 from .mount_test_base import MountTestBase
30 from .test_tmp_collection import storage_classes_desired
31
32 logger = logging.getLogger('arvados.arv-mount')
33
34
35 class AssertWithTimeout(object):
36     """Allow some time for an assertion to pass."""
37
38     def __init__(self, timeout=0):
39         self.timeout = timeout
40
41     def __iter__(self):
42         self.deadline = time.time() + self.timeout
43         self.done = False
44         return self
45
46     def __next__(self):
47         if self.done:
48             raise StopIteration
49         return self.attempt
50
51     def attempt(self, fn, *args, **kwargs):
52         try:
53             fn(*args, **kwargs)
54         except AssertionError:
55             if time.time() > self.deadline:
56                 raise
57             time.sleep(0.1)
58         else:
59             self.done = True
60
61 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
62 class FuseMountTest(MountTestBase):
63     def setUp(self):
64         super(FuseMountTest, self).setUp()
65
66         cw = arvados.CollectionWriter()
67
68         cw.start_new_file('thing1.txt')
69         cw.write("data 1")
70         cw.start_new_file('thing2.txt')
71         cw.write("data 2")
72
73         cw.start_new_stream('dir1')
74         cw.start_new_file('thing3.txt')
75         cw.write("data 3")
76         cw.start_new_file('thing4.txt')
77         cw.write("data 4")
78
79         cw.start_new_stream('dir2')
80         cw.start_new_file('thing5.txt')
81         cw.write("data 5")
82         cw.start_new_file('thing6.txt')
83         cw.write("data 6")
84
85         cw.start_new_stream('dir2/dir3')
86         cw.start_new_file('thing7.txt')
87         cw.write("data 7")
88
89         cw.start_new_file('thing8.txt')
90         cw.write("data 8")
91
92         cw.start_new_stream('edgecases')
93         for f in ":/.../-/*/ ".split("/"):
94             cw.start_new_file(f)
95             cw.write('x')
96
97         for f in ":/.../-/*/ ".split("/"):
98             cw.start_new_stream('edgecases/dirs/' + f)
99             cw.start_new_file('x/x')
100             cw.write('x')
101
102         self.testcollection = cw.finish()
103         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
104
105     def runTest(self):
106         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
107
108         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
109                                       'edgecases', 'dir1', 'dir2'])
110         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
111         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
112         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
113         self.assertDirContents('edgecases',
114                                "dirs/:/.../-/*/ ".split("/"))
115         self.assertDirContents('edgecases/dirs',
116                                ":/.../-/*/ ".split("/"))
117
118         files = {'thing1.txt': 'data 1',
119                  'thing2.txt': 'data 2',
120                  'dir1/thing3.txt': 'data 3',
121                  'dir1/thing4.txt': 'data 4',
122                  'dir2/thing5.txt': 'data 5',
123                  'dir2/thing6.txt': 'data 6',
124                  'dir2/dir3/thing7.txt': 'data 7',
125                  'dir2/dir3/thing8.txt': 'data 8'}
126
127         for k, v in viewitems(files):
128             with open(os.path.join(self.mounttmp, k), 'rb') as f:
129                 self.assertEqual(v, f.read().decode())
130
131
132 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
133 class FuseMagicTest(MountTestBase):
134     def setUp(self, api=None):
135         super(FuseMagicTest, self).setUp(api=api)
136
137         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
138         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
139         self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
140         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
141         self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
142
143         cw = arvados.CollectionWriter()
144
145         cw.start_new_file('thing1.txt')
146         cw.write("data 1")
147
148         self.testcollection = cw.finish()
149         self.test_manifest = cw.manifest_text()
150         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
151         self.test_manifest_pdh = coll['portable_data_hash']
152
153     def runTest(self):
154         self.make_mount(fuse.MagicDirectory)
155
156         mount_ls = llfuse.listdir(self.mounttmp)
157         self.assertIn('README', mount_ls)
158         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
159                              arvados.util.uuid_pattern.match(fn)
160                              for fn in mount_ls),
161                          "new FUSE MagicDirectory has no collections or projects")
162         self.assertDirContents(self.testcollection, ['thing1.txt'])
163         self.assertDirContents(os.path.join('by_id', self.testcollection),
164                                ['thing1.txt'])
165         self.assertIn(self.collection_in_test_project,
166                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
167         self.assertIn(self.collection_in_test_project,
168                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
169         self.assertIn(self.collection_in_filter_group,
170                       llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
171         self.assertIn(self.collection_in_filter_group,
172                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
173
174
175         mount_ls = llfuse.listdir(self.mounttmp)
176         self.assertIn('README', mount_ls)
177         self.assertIn(self.testcollection, mount_ls)
178         self.assertIn(self.testcollection,
179                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
180         self.assertIn(self.test_project, mount_ls)
181         self.assertIn(self.test_project,
182                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
183         self.assertIn(self.filter_group,
184                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
185
186         with self.assertRaises(OSError):
187             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
188
189         files = {}
190         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
191
192         for k, v in viewitems(files):
193             with open(os.path.join(self.mounttmp, k), 'rb') as f:
194                 self.assertEqual(v, f.read().decode())
195
196
197 class FuseTagsTest(MountTestBase):
198     def runTest(self):
199         self.make_mount(fuse.TagsDirectory)
200
201         d1 = llfuse.listdir(self.mounttmp)
202         d1.sort()
203         self.assertEqual(['foo_tag'], d1)
204
205         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
206         d2.sort()
207         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
208
209         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
210         d3.sort()
211         self.assertEqual(['foo'], d3)
212
213
214 class FuseTagsUpdateTest(MountTestBase):
215     def tag_collection(self, coll_uuid, tag_name):
216         return self.api.links().create(
217             body={'link': {'head_uuid': coll_uuid,
218                            'link_class': 'tag',
219                            'name': tag_name,
220         }}).execute()
221
222     def runTest(self):
223         self.make_mount(fuse.TagsDirectory, poll_time=1)
224
225         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
226
227         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
228         self.tag_collection(bar_uuid, 'fuse_test_tag')
229         for attempt in AssertWithTimeout(10):
230             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
231         self.assertDirContents('fuse_test_tag', [bar_uuid])
232
233         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
234         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
235         for attempt in AssertWithTimeout(10):
236             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
237
238         self.api.links().delete(uuid=l['uuid']).execute()
239         for attempt in AssertWithTimeout(10):
240             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
241
242
243 def fuseSharedTestHelper(mounttmp):
244     class Test(unittest.TestCase):
245         def runTest(self):
246             # Double check that we can open and read objects in this folder as a file,
247             # and that its contents are what we expect.
248             baz_path = os.path.join(
249                 mounttmp,
250                 'FUSE User',
251                 'FUSE Test Project',
252                 'collection in FUSE project',
253                 'baz')
254             with open(baz_path) as f:
255                 self.assertEqual("baz", f.read())
256
257             # check mtime on collection
258             st = os.stat(baz_path)
259             try:
260                 mtime = st.st_mtime_ns // 1000000000
261             except AttributeError:
262                 mtime = st.st_mtime
263             self.assertEqual(mtime, 1391448174)
264
265             # shared_dirs is a list of the directories exposed
266             # by fuse.SharedDirectory (i.e. any object visible
267             # to the current user)
268             shared_dirs = llfuse.listdir(mounttmp)
269             shared_dirs.sort()
270             self.assertIn('FUSE User', shared_dirs)
271
272             # fuse_user_objs is a list of the objects owned by the FUSE
273             # test user (which present as files in the 'FUSE User'
274             # directory)
275             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
276             fuse_user_objs.sort()
277             self.assertEqual(['FUSE Test Project',                    # project owned by user
278                               'collection #1 owned by FUSE',          # collection owned by user
279                               'collection #2 owned by FUSE'          # collection owned by user
280                           ], fuse_user_objs)
281
282             # test_proj_files is a list of the files in the FUSE Test Project.
283             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
284             test_proj_files.sort()
285             self.assertEqual(['collection in FUSE project'
286                           ], test_proj_files)
287
288
289     Test().runTest()
290
291 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
292 class FuseSharedTest(MountTestBase):
293     def runTest(self):
294         self.make_mount(fuse.SharedDirectory,
295                         exclude=self.api.users().current().execute()['uuid'])
296         keep = arvados.keep.KeepClient()
297         keep.put("baz".encode())
298
299         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
300
301
302 class FuseHomeTest(MountTestBase):
303     def runTest(self):
304         self.make_mount(fuse.ProjectDirectory,
305                         project_object=self.api.users().current().execute())
306
307         d1 = llfuse.listdir(self.mounttmp)
308         self.assertIn('Unrestricted public data', d1)
309
310         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
311         public_project = run_test_server.fixture('groups')[
312             'anonymously_accessible_project']
313         found_in = 0
314         found_not_in = 0
315         for name, item in viewitems(run_test_server.fixture('collections')):
316             if 'name' not in item:
317                 pass
318             elif item['owner_uuid'] == public_project['uuid']:
319                 self.assertIn(item['name'], d2)
320                 found_in += 1
321             else:
322                 # Artificial assumption here: there is no public
323                 # collection fixture with the same name as a
324                 # non-public collection.
325                 self.assertNotIn(item['name'], d2)
326                 found_not_in += 1
327         self.assertNotEqual(0, found_in)
328         self.assertNotEqual(0, found_not_in)
329
330         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
331         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
332
333
334 def fuseModifyFileTestHelperReadStartContents(mounttmp):
335     class Test(unittest.TestCase):
336         def runTest(self):
337             d1 = llfuse.listdir(mounttmp)
338             self.assertEqual(["file1.txt"], d1)
339             with open(os.path.join(mounttmp, "file1.txt")) as f:
340                 self.assertEqual("blub", f.read())
341     Test().runTest()
342
343 def fuseModifyFileTestHelperReadEndContents(mounttmp):
344     class Test(unittest.TestCase):
345         def runTest(self):
346             d1 = llfuse.listdir(mounttmp)
347             self.assertEqual(["file1.txt"], d1)
348             with open(os.path.join(mounttmp, "file1.txt")) as f:
349                 self.assertEqual("plnp", f.read())
350     Test().runTest()
351
352 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
353 class FuseModifyFileTest(MountTestBase):
354     def runTest(self):
355         collection = arvados.collection.Collection(api_client=self.api)
356         with collection.open("file1.txt", "w") as f:
357             f.write("blub")
358
359         collection.save_new()
360
361         m = self.make_mount(fuse.CollectionDirectory)
362         with llfuse.lock:
363             m.new_collection(collection.api_response(), collection)
364
365         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
366
367         with collection.open("file1.txt", "w") as f:
368             f.write("plnp")
369
370         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
371
372
373 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
374 class FuseAddFileToCollectionTest(MountTestBase):
375     def runTest(self):
376         collection = arvados.collection.Collection(api_client=self.api)
377         with collection.open("file1.txt", "w") as f:
378             f.write("blub")
379
380         collection.save_new()
381
382         m = self.make_mount(fuse.CollectionDirectory)
383         with llfuse.lock:
384             m.new_collection(collection.api_response(), collection)
385
386         d1 = llfuse.listdir(self.mounttmp)
387         self.assertEqual(["file1.txt"], d1)
388
389         with collection.open("file2.txt", "w") as f:
390             f.write("plnp")
391
392         d1 = llfuse.listdir(self.mounttmp)
393         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
394
395
396 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
397 class FuseRemoveFileFromCollectionTest(MountTestBase):
398     def runTest(self):
399         collection = arvados.collection.Collection(api_client=self.api)
400         with collection.open("file1.txt", "w") as f:
401             f.write("blub")
402
403         with collection.open("file2.txt", "w") as f:
404             f.write("plnp")
405
406         collection.save_new()
407
408         m = self.make_mount(fuse.CollectionDirectory)
409         with llfuse.lock:
410             m.new_collection(collection.api_response(), collection)
411
412         d1 = llfuse.listdir(self.mounttmp)
413         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
414
415         collection.remove("file2.txt")
416
417         d1 = llfuse.listdir(self.mounttmp)
418         self.assertEqual(["file1.txt"], d1)
419
420
421 def fuseCreateFileTestHelper(mounttmp):
422     class Test(unittest.TestCase):
423         def runTest(self):
424             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
425                 pass
426     Test().runTest()
427
428 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
429 class FuseCreateFileTest(MountTestBase):
430     def runTest(self):
431         collection = arvados.collection.Collection(api_client=self.api)
432         collection.save_new()
433
434         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
435         self.assertEqual(collection2["manifest_text"], "")
436
437         collection.save_new()
438
439         m = self.make_mount(fuse.CollectionDirectory)
440         with llfuse.lock:
441             m.new_collection(collection.api_response(), collection)
442         self.assertTrue(m.writable())
443
444         self.assertNotIn("file1.txt", collection)
445
446         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
447
448         self.assertIn("file1.txt", collection)
449
450         d1 = llfuse.listdir(self.mounttmp)
451         self.assertEqual(["file1.txt"], d1)
452
453         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
454         assertRegex(self, collection2["manifest_text"],
455             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
456
457
458 def fuseWriteFileTestHelperWriteFile(mounttmp):
459     class Test(unittest.TestCase):
460         def runTest(self):
461             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
462                 f.write("Hello world!")
463     Test().runTest()
464
465 def fuseWriteFileTestHelperReadFile(mounttmp):
466     class Test(unittest.TestCase):
467         def runTest(self):
468             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
469                 self.assertEqual(f.read(), "Hello world!")
470     Test().runTest()
471
472 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
473 class FuseWriteFileTest(MountTestBase):
474     def runTest(self):
475         collection = arvados.collection.Collection(api_client=self.api)
476         collection.save_new()
477
478         m = self.make_mount(fuse.CollectionDirectory)
479         with llfuse.lock:
480             m.new_collection(collection.api_response(), collection)
481         self.assertTrue(m.writable())
482
483         self.assertNotIn("file1.txt", collection)
484
485         self.assertEqual(0, self.operations.write_counter.get())
486         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
487         self.assertEqual(12, self.operations.write_counter.get())
488
489         with collection.open("file1.txt") as f:
490             self.assertEqual(f.read(), "Hello world!")
491
492         self.assertEqual(0, self.operations.read_counter.get())
493         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
494         self.assertEqual(12, self.operations.read_counter.get())
495
496         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
497         assertRegex(self, collection2["manifest_text"],
498             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
499
500
501 def fuseUpdateFileTestHelper(mounttmp):
502     class Test(unittest.TestCase):
503         def runTest(self):
504             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
505                 f.write("Hello world!")
506
507             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
508                 fr = f.read()
509                 self.assertEqual(fr, "Hello world!")
510                 f.seek(0)
511                 f.write("Hola mundo!")
512                 f.seek(0)
513                 fr = f.read()
514                 self.assertEqual(fr, "Hola mundo!!")
515
516             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
517                 self.assertEqual(f.read(), "Hola mundo!!")
518
519     Test().runTest()
520
521 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
522 class FuseUpdateFileTest(MountTestBase):
523     def runTest(self):
524         collection = arvados.collection.Collection(api_client=self.api)
525         collection.save_new()
526
527         m = self.make_mount(fuse.CollectionDirectory)
528         with llfuse.lock:
529             m.new_collection(collection.api_response(), collection)
530         self.assertTrue(m.writable())
531
532         # See note in MountTestBase.setUp
533         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
534
535         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
536         assertRegex(self, collection2["manifest_text"],
537             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
538
539
540 def fuseMkdirTestHelper(mounttmp):
541     class Test(unittest.TestCase):
542         def runTest(self):
543             with self.assertRaises(IOError):
544                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
545                     f.write("Hello world!")
546
547             os.mkdir(os.path.join(mounttmp, "testdir"))
548
549             with self.assertRaises(OSError):
550                 os.mkdir(os.path.join(mounttmp, "testdir"))
551
552             d1 = llfuse.listdir(mounttmp)
553             self.assertEqual(["testdir"], d1)
554
555             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
556                 f.write("Hello world!")
557
558             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
559             self.assertEqual(["file1.txt"], d1)
560
561     Test().runTest()
562
563 class FuseMkdirTest(MountTestBase):
564     def runTest(self):
565         collection = arvados.collection.Collection(api_client=self.api)
566         collection.save_new()
567
568         m = self.make_mount(fuse.CollectionDirectory)
569         with llfuse.lock:
570             m.new_collection(collection.api_response(), collection)
571         self.assertTrue(m.writable())
572
573         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
574
575         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
576         assertRegex(self, collection2["manifest_text"],
577             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
578
579
580 def fuseRmTestHelperWriteFile(mounttmp):
581     class Test(unittest.TestCase):
582         def runTest(self):
583             os.mkdir(os.path.join(mounttmp, "testdir"))
584
585             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
586                 f.write("Hello world!")
587
588     Test().runTest()
589
590 def fuseRmTestHelperDeleteFile(mounttmp):
591     class Test(unittest.TestCase):
592         def runTest(self):
593             # Can't delete because it's not empty
594             with self.assertRaises(OSError):
595                 os.rmdir(os.path.join(mounttmp, "testdir"))
596
597             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
598             self.assertEqual(["file1.txt"], d1)
599
600             # Delete file
601             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
602
603             # Make sure it's empty
604             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
605             self.assertEqual([], d1)
606
607             # Try to delete it again
608             with self.assertRaises(OSError):
609                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
610
611     Test().runTest()
612
613 def fuseRmTestHelperRmdir(mounttmp):
614     class Test(unittest.TestCase):
615         def runTest(self):
616             # Should be able to delete now that it is empty
617             os.rmdir(os.path.join(mounttmp, "testdir"))
618
619             # Make sure it's empty
620             d1 = llfuse.listdir(os.path.join(mounttmp))
621             self.assertEqual([], d1)
622
623             # Try to delete it again
624             with self.assertRaises(OSError):
625                 os.rmdir(os.path.join(mounttmp, "testdir"))
626
627     Test().runTest()
628
629 class FuseRmTest(MountTestBase):
630     def runTest(self):
631         collection = arvados.collection.Collection(api_client=self.api)
632         collection.save_new()
633
634         m = self.make_mount(fuse.CollectionDirectory)
635         with llfuse.lock:
636             m.new_collection(collection.api_response(), collection)
637         self.assertTrue(m.writable())
638
639         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
640
641         # Starting manifest
642         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
643         assertRegex(self, collection2["manifest_text"],
644             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
645         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
646
647         # Empty directories are represented by an empty file named "."
648         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
649         assertRegex(self, collection2["manifest_text"],
650                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
651
652         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
653
654         # manifest should be empty now.
655         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
656         self.assertEqual(collection2["manifest_text"], "")
657
658
659 def fuseMvFileTestHelperWriteFile(mounttmp):
660     class Test(unittest.TestCase):
661         def runTest(self):
662             os.mkdir(os.path.join(mounttmp, "testdir"))
663
664             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
665                 f.write("Hello world!")
666
667     Test().runTest()
668
669 def fuseMvFileTestHelperMoveFile(mounttmp):
670     class Test(unittest.TestCase):
671         def runTest(self):
672             d1 = llfuse.listdir(os.path.join(mounttmp))
673             self.assertEqual(["testdir"], d1)
674             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
675             self.assertEqual(["file1.txt"], d1)
676
677             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
678
679             d1 = llfuse.listdir(os.path.join(mounttmp))
680             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
681             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
682             self.assertEqual([], d1)
683
684     Test().runTest()
685
686 class FuseMvFileTest(MountTestBase):
687     def runTest(self):
688         collection = arvados.collection.Collection(api_client=self.api)
689         collection.save_new()
690
691         m = self.make_mount(fuse.CollectionDirectory)
692         with llfuse.lock:
693             m.new_collection(collection.api_response(), collection)
694         self.assertTrue(m.writable())
695
696         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
697
698         # Starting manifest
699         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
700         assertRegex(self, collection2["manifest_text"],
701             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
702
703         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
704
705         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
706         assertRegex(self, collection2["manifest_text"],
707             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
708
709
710 def fuseRenameTestHelper(mounttmp):
711     class Test(unittest.TestCase):
712         def runTest(self):
713             os.mkdir(os.path.join(mounttmp, "testdir"))
714
715             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
716                 f.write("Hello world!")
717
718     Test().runTest()
719
720 class FuseRenameTest(MountTestBase):
721     def runTest(self):
722         collection = arvados.collection.Collection(api_client=self.api)
723         collection.save_new()
724
725         m = self.make_mount(fuse.CollectionDirectory)
726         with llfuse.lock:
727             m.new_collection(collection.api_response(), collection)
728         self.assertTrue(m.writable())
729
730         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
731
732         # Starting manifest
733         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
734         assertRegex(self, collection2["manifest_text"],
735             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
736
737         d1 = llfuse.listdir(os.path.join(self.mounttmp))
738         self.assertEqual(["testdir"], d1)
739         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
740         self.assertEqual(["file1.txt"], d1)
741
742         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
743
744         d1 = llfuse.listdir(os.path.join(self.mounttmp))
745         self.assertEqual(["testdir2"], sorted(d1))
746         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
747         self.assertEqual(["file1.txt"], d1)
748
749         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
750         assertRegex(self, collection2["manifest_text"],
751             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
752
753
754 class FuseUpdateFromEventTest(MountTestBase):
755     def runTest(self):
756         collection = arvados.collection.Collection(api_client=self.api)
757         collection.save_new()
758
759         m = self.make_mount(fuse.CollectionDirectory)
760         with llfuse.lock:
761             m.new_collection(collection.api_response(), collection)
762
763         self.operations.listen_for_events()
764
765         d1 = llfuse.listdir(os.path.join(self.mounttmp))
766         self.assertEqual([], sorted(d1))
767
768         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
769             with collection2.open("file1.txt", "w") as f:
770                 f.write("foo")
771
772         for attempt in AssertWithTimeout(10):
773             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
774
775
776 class FuseDeleteProjectEventTest(MountTestBase):
777     def runTest(self):
778
779         aproject = self.api.groups().create(body={
780             "name": "aproject",
781             "group_class": "project"
782         }).execute()
783
784         bproject = self.api.groups().create(body={
785             "name": "bproject",
786             "group_class": "project",
787             "owner_uuid": aproject["uuid"]
788         }).execute()
789
790         self.make_mount(fuse.ProjectDirectory,
791                         project_object=self.api.users().current().execute())
792
793         self.operations.listen_for_events()
794
795         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
796         self.assertEqual(["bproject"], sorted(d1))
797
798         self.api.groups().delete(uuid=bproject["uuid"]).execute()
799
800         for attempt in AssertWithTimeout(10):
801             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
802
803
804 def fuseFileConflictTestHelper(mounttmp, uuid, keeptmp, settings):
805     class Test(unittest.TestCase):
806         def runTest(self):
807             os.environ['KEEP_LOCAL_STORE'] = keeptmp
808
809             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
810                 with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
811                     with collection2.open("file1.txt", "w") as f2:
812                         f2.write("foo")
813                 f.write("bar")
814
815             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
816             self.assertEqual(len(d1), 2)
817
818             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
819                 self.assertEqual(f.read(), "bar")
820
821             assertRegex(self, d1[1],
822                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
823
824             with open(os.path.join(mounttmp, d1[1]), "r") as f:
825                 self.assertEqual(f.read(), "foo")
826
827     Test().runTest()
828
829 class FuseFileConflictTest(MountTestBase):
830     def runTest(self):
831         collection = arvados.collection.Collection(api_client=self.api)
832         collection.save_new()
833
834         m = self.make_mount(fuse.CollectionDirectory)
835         with llfuse.lock:
836             m.new_collection(collection.api_response(), collection)
837
838         d1 = llfuse.listdir(os.path.join(self.mounttmp))
839         self.assertEqual([], sorted(d1))
840
841         # See note in MountTestBase.setUp
842         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
843
844
845 def fuseUnlinkOpenFileTest(mounttmp):
846     class Test(unittest.TestCase):
847         def runTest(self):
848             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
849                 f.write("foo")
850
851                 d1 = llfuse.listdir(os.path.join(mounttmp))
852                 self.assertEqual(["file1.txt"], sorted(d1))
853
854                 os.remove(os.path.join(mounttmp, "file1.txt"))
855
856                 d1 = llfuse.listdir(os.path.join(mounttmp))
857                 self.assertEqual([], sorted(d1))
858
859                 f.seek(0)
860                 self.assertEqual(f.read(), "foo")
861                 f.write("bar")
862
863                 f.seek(0)
864                 self.assertEqual(f.read(), "foobar")
865
866     Test().runTest()
867
868 class FuseUnlinkOpenFileTest(MountTestBase):
869     def runTest(self):
870         collection = arvados.collection.Collection(api_client=self.api)
871         collection.save_new()
872
873         m = self.make_mount(fuse.CollectionDirectory)
874         with llfuse.lock:
875             m.new_collection(collection.api_response(), collection)
876
877         # See note in MountTestBase.setUp
878         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
879
880         self.assertEqual(collection.manifest_text(), "")
881
882
883 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
884     class Test(unittest.TestCase):
885         def runTest(self):
886             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
887                 f.write("Hello world!")
888
889             d1 = os.listdir(os.path.join(mounttmp, uuid1))
890             self.assertEqual(["file1.txt"], sorted(d1))
891             d1 = os.listdir(os.path.join(mounttmp, uuid2))
892             self.assertEqual([], sorted(d1))
893
894     Test().runTest()
895
896 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
897     class Test(unittest.TestCase):
898         def runTest(self):
899             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
900
901             d1 = os.listdir(os.path.join(mounttmp, uuid1))
902             self.assertEqual([], sorted(d1))
903             d1 = os.listdir(os.path.join(mounttmp, uuid2))
904             self.assertEqual(["file2.txt"], sorted(d1))
905
906     Test().runTest()
907
908 class FuseMvFileBetweenCollectionsTest(MountTestBase):
909     def runTest(self):
910         collection1 = arvados.collection.Collection(api_client=self.api)
911         collection1.save_new()
912
913         collection2 = arvados.collection.Collection(api_client=self.api)
914         collection2.save_new()
915
916         m = self.make_mount(fuse.MagicDirectory)
917
918         # See note in MountTestBase.setUp
919         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
920                                                   collection1.manifest_locator(),
921                                                   collection2.manifest_locator()))
922
923         collection1.update()
924         collection2.update()
925
926         assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
927         self.assertEqual(collection2.manifest_text(), "")
928
929         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
930                                                   collection1.manifest_locator(),
931                                                   collection2.manifest_locator()))
932
933         collection1.update()
934         collection2.update()
935
936         self.assertEqual(collection1.manifest_text(), "")
937         assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
938
939         collection1.stop_threads()
940         collection2.stop_threads()
941
942
943 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
944     class Test(unittest.TestCase):
945         def runTest(self):
946             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
947             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
948                 f.write("Hello world!")
949
950             d1 = os.listdir(os.path.join(mounttmp, uuid1))
951             self.assertEqual(["testdir"], sorted(d1))
952             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
953             self.assertEqual(["file1.txt"], sorted(d1))
954
955             d1 = os.listdir(os.path.join(mounttmp, uuid2))
956             self.assertEqual([], sorted(d1))
957
958     Test().runTest()
959
960
961 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
962     class Test(unittest.TestCase):
963         def runTest(self):
964             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
965
966             d1 = os.listdir(os.path.join(mounttmp, uuid1))
967             self.assertEqual([], sorted(d1))
968
969             d1 = os.listdir(os.path.join(mounttmp, uuid2))
970             self.assertEqual(["testdir2"], sorted(d1))
971             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
972             self.assertEqual(["file1.txt"], sorted(d1))
973
974             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
975                 self.assertEqual(f.read(), "Hello world!")
976
977     Test().runTest()
978
979 class FuseMvDirBetweenCollectionsTest(MountTestBase):
980     def runTest(self):
981         collection1 = arvados.collection.Collection(api_client=self.api)
982         collection1.save_new()
983
984         collection2 = arvados.collection.Collection(api_client=self.api)
985         collection2.save_new()
986
987         m = self.make_mount(fuse.MagicDirectory)
988
989         # See note in MountTestBase.setUp
990         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
991                                                   collection1.manifest_locator(),
992                                                   collection2.manifest_locator()))
993
994         collection1.update()
995         collection2.update()
996
997         assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
998         self.assertEqual(collection2.manifest_text(), "")
999
1000         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
1001                                                   collection1.manifest_locator(),
1002                                                   collection2.manifest_locator()))
1003
1004         collection1.update()
1005         collection2.update()
1006
1007         self.assertEqual(collection1.manifest_text(), "")
1008         assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
1009
1010         collection1.stop_threads()
1011         collection2.stop_threads()
1012
1013 def fuseProjectMkdirTestHelper1(mounttmp):
1014     class Test(unittest.TestCase):
1015         def runTest(self):
1016             os.mkdir(os.path.join(mounttmp, "testcollection"))
1017             with self.assertRaises(OSError):
1018                 os.mkdir(os.path.join(mounttmp, "testcollection"))
1019     Test().runTest()
1020
1021 def fuseProjectMkdirTestHelper2(mounttmp):
1022     class Test(unittest.TestCase):
1023         def runTest(self):
1024             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1025                 f.write("Hello world!")
1026             with self.assertRaises(OSError):
1027                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1028             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1029             with self.assertRaises(OSError):
1030                 os.remove(os.path.join(mounttmp, "testcollection"))
1031             os.rmdir(os.path.join(mounttmp, "testcollection"))
1032     Test().runTest()
1033
1034 class FuseProjectMkdirRmdirTest(MountTestBase):
1035     def runTest(self):
1036         self.make_mount(fuse.ProjectDirectory,
1037                         project_object=self.api.users().current().execute())
1038
1039         d1 = llfuse.listdir(self.mounttmp)
1040         self.assertNotIn('testcollection', d1)
1041
1042         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1043
1044         d1 = llfuse.listdir(self.mounttmp)
1045         self.assertIn('testcollection', d1)
1046
1047         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1048
1049         d1 = llfuse.listdir(self.mounttmp)
1050         self.assertNotIn('testcollection', d1)
1051
1052
1053 def fuseProjectMvTestHelper1(mounttmp):
1054     class Test(unittest.TestCase):
1055         def runTest(self):
1056             d1 = llfuse.listdir(mounttmp)
1057             self.assertNotIn('testcollection', d1)
1058
1059             os.mkdir(os.path.join(mounttmp, "testcollection"))
1060
1061             d1 = llfuse.listdir(mounttmp)
1062             self.assertIn('testcollection', d1)
1063
1064             with self.assertRaises(OSError):
1065                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1066
1067             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1068
1069             d1 = llfuse.listdir(mounttmp)
1070             self.assertNotIn('testcollection', d1)
1071
1072             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1073             self.assertIn('testcollection', d1)
1074
1075     Test().runTest()
1076
1077 class FuseProjectMvTest(MountTestBase):
1078     def runTest(self):
1079         self.make_mount(fuse.ProjectDirectory,
1080                         project_object=self.api.users().current().execute())
1081
1082         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1083
1084
1085 def fuseFsyncTestHelper(mounttmp, k):
1086     class Test(unittest.TestCase):
1087         def runTest(self):
1088             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1089             os.fsync(fd)
1090             os.close(fd)
1091
1092     Test().runTest()
1093
1094 class FuseFsyncTest(FuseMagicTest):
1095     def runTest(self):
1096         self.make_mount(fuse.MagicDirectory)
1097         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1098
1099
1100 class MagicDirApiError(FuseMagicTest):
1101     def setUp(self):
1102         api = mock.MagicMock()
1103         api.keep.block_cache = mock.MagicMock(cache_max=1)
1104         super(MagicDirApiError, self).setUp(api=api)
1105         api.collections().get().execute.side_effect = iter([
1106             Exception('API fail'),
1107             {
1108                 "manifest_text": self.test_manifest,
1109                 "portable_data_hash": self.test_manifest_pdh,
1110             },
1111         ])
1112         api.keep.get.side_effect = Exception('Keep fail')
1113
1114     def runTest(self):
1115         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1116             self.make_mount(fuse.MagicDirectory)
1117
1118             self.operations.inodes.inode_cache.cap = 1
1119             self.operations.inodes.inode_cache.min_entries = 2
1120
1121             with self.assertRaises(OSError):
1122                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1123
1124             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1125
1126
1127 class SanitizeFilenameTest(MountTestBase):
1128     def test_sanitize_filename(self):
1129         pdir = fuse.ProjectDirectory(1, {}, self.api, 0, False, project_object=self.api.users().current().execute())
1130         acceptable = [
1131             "foo.txt",
1132             ".foo",
1133             "..foo",
1134             "...",
1135             "foo...",
1136             "foo..",
1137             "foo.",
1138             "-",
1139             "\x01\x02\x03",
1140             ]
1141         unacceptable = [
1142             "f\00",
1143             "\00\00",
1144             "/foo",
1145             "foo/",
1146             "//",
1147             ]
1148         for f in acceptable:
1149             self.assertEqual(f, pdir.sanitize_filename(f))
1150         for f in unacceptable:
1151             self.assertNotEqual(f, pdir.sanitize_filename(f))
1152             # The sanitized filename should be the same length, though.
1153             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1154         # Special cases
1155         self.assertEqual("_", pdir.sanitize_filename(""))
1156         self.assertEqual("_", pdir.sanitize_filename("."))
1157         self.assertEqual("__", pdir.sanitize_filename(".."))
1158
1159
1160 class FuseMagicTestPDHOnly(MountTestBase):
1161     def setUp(self, api=None):
1162         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1163
1164         cw = arvados.CollectionWriter()
1165
1166         cw.start_new_file('thing1.txt')
1167         cw.write("data 1")
1168
1169         self.testcollection = cw.finish()
1170         self.test_manifest = cw.manifest_text()
1171         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1172         self.testcollectionuuid = str(created['uuid'])
1173
1174     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1175         if skip_pdh_only is True:
1176             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1177         else:
1178             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1179
1180         mount_ls = llfuse.listdir(self.mounttmp)
1181         self.assertIn('README', mount_ls)
1182         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1183                              arvados.util.uuid_pattern.match(fn)
1184                              for fn in mount_ls),
1185                          "new FUSE MagicDirectory lists Collection")
1186
1187         # look up using pdh should succeed in all cases
1188         self.assertDirContents(self.testcollection, ['thing1.txt'])
1189         self.assertDirContents(os.path.join('by_id', self.testcollection),
1190                                ['thing1.txt'])
1191         mount_ls = llfuse.listdir(self.mounttmp)
1192         self.assertIn('README', mount_ls)
1193         self.assertIn(self.testcollection, mount_ls)
1194         self.assertIn(self.testcollection,
1195                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1196
1197         files = {}
1198         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1199
1200         for k, v in viewitems(files):
1201             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1202                 self.assertEqual(v, f.read().decode())
1203
1204         # look up using uuid should fail when pdh_only is set
1205         if pdh_only is True:
1206             with self.assertRaises(OSError):
1207                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1208                                ['thing1.txt'])
1209         else:
1210             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1211                                ['thing1.txt'])
1212
1213     def test_with_pdh_only_true(self):
1214         self.verify_pdh_only(pdh_only=True)
1215
1216     def test_with_pdh_only_false(self):
1217         self.verify_pdh_only(pdh_only=False)
1218
1219     def test_with_default_by_id(self):
1220         self.verify_pdh_only(skip_pdh_only=True)
1221
1222
1223 class SlashSubstitutionTest(IntegrationTest):
1224     mnt_args = [
1225         '--read-write',
1226         '--mount-home', 'zzz',
1227     ]
1228
1229     def setUp(self):
1230         super(SlashSubstitutionTest, self).setUp()
1231         self.api = arvados.safeapi.ThreadSafeApiCache(
1232             arvados.config.settings(),
1233             version='v1',
1234         )
1235         self.api.config = lambda: {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1236         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1237         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1238         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1239
1240     @IntegrationTest.mount(argv=mnt_args)
1241     @mock.patch('arvados.util.get_config_once')
1242     def test_slash_substitution_before_listing(self, get_config_once):
1243         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1244         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1245         self.checkContents()
1246     @staticmethod
1247     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1248         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1249             f.write('xxx')
1250         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1251             f.write('foo')
1252
1253     @IntegrationTest.mount(argv=mnt_args)
1254     @mock.patch('arvados.util.get_config_once')
1255     def test_slash_substitution_after_listing(self, get_config_once):
1256         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1257         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1258         self.checkContents()
1259     @staticmethod
1260     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1261         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1262             f.write('xxx')
1263         os.listdir(tmpdir)
1264         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1265             f.write('foo')
1266
1267     def checkContents(self):
1268         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1269         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1270
1271     @IntegrationTest.mount(argv=mnt_args)
1272     @mock.patch('arvados.util.get_config_once')
1273     def test_slash_substitution_conflict(self, get_config_once):
1274         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1275         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1276         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1277         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1278         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1279         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1280     @staticmethod
1281     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1282         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1283             f.write('foo')
1284
1285 class StorageClassesTest(IntegrationTest):
1286     mnt_args = [
1287         '--read-write',
1288         '--mount-home', 'homedir',
1289     ]
1290
1291     def setUp(self):
1292         super(StorageClassesTest, self).setUp()
1293         self.api = arvados.safeapi.ThreadSafeApiCache(
1294             arvados.config.settings(),
1295             version='v1',
1296         )
1297
1298     @IntegrationTest.mount(argv=mnt_args)
1299     def test_collection_default_storage_classes(self):
1300         coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
1301         self.api.collections().create(body={'name':'a_collection'}).execute()
1302         self.pool_test(coll_path)
1303     @staticmethod
1304     def _test_collection_default_storage_classes(self, coll):
1305         self.assertEqual(storage_classes_desired(coll), ['default'])
1306
1307     @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
1308     def test_collection_custom_storage_classes(self):
1309         coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
1310         os.mkdir(coll_path)
1311         self.pool_test(coll_path)
1312     @staticmethod
1313     def _test_collection_custom_storage_classes(self, coll):
1314         self.assertEqual(storage_classes_desired(coll), ['foo'])
1315
1316 def _readonlyCollectionTestHelper(mounttmp):
1317     f = open(os.path.join(mounttmp, 'thing1.txt'), 'rt')
1318     # Testing that close() doesn't raise an error.
1319     f.close()
1320
1321 class ReadonlyCollectionTest(MountTestBase):
1322     def setUp(self):
1323         super(ReadonlyCollectionTest, self).setUp()
1324         cw = arvados.collection.Collection()
1325         with cw.open('thing1.txt', 'wt') as f:
1326             f.write("data 1")
1327         cw.save_new(owner_uuid=run_test_server.fixture("groups")["aproject"]["uuid"])
1328         self.testcollection = cw.api_response()
1329
1330     def runTest(self):
1331         settings = arvados.config.settings().copy()
1332         settings["ARVADOS_API_TOKEN"] = run_test_server.fixture("api_client_authorizations")["project_viewer"]["api_token"]
1333         self.api = arvados.safeapi.ThreadSafeApiCache(settings, version='v1')
1334         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection, enable_write=False)
1335
1336         self.pool.apply(_readonlyCollectionTestHelper, (self.mounttmp,))
1337
1338
1339 @parameterized.parameterized_class([
1340     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1341         'project_object': run_test_server.fixture('users')['admin'],
1342     }},
1343     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1344         'project_object': run_test_server.fixture('groups')['public'],
1345     }},
1346 ])
1347 class UnsupportedCreateTest(MountTestBase):
1348     root_class = None
1349     root_kwargs = {}
1350
1351     def setUp(self):
1352         super().setUp()
1353         if 'prefs' in self.root_kwargs.get('project_object', ()):
1354             self.root_kwargs['project_object']['prefs'] = {}
1355         self.make_mount(self.root_class, **self.root_kwargs)
1356         # Make sure the directory knows about its top-level ents.
1357         os.listdir(self.mounttmp)
1358
1359     def test_create(self):
1360         test_path = Path(self.mounttmp, 'test_create')
1361         with self.assertRaises(OSError) as exc_check:
1362             with test_path.open('w'):
1363                 pass
1364         self.assertEqual(exc_check.exception.errno, errno.ENOTSUP)
1365
1366
1367 # FIXME: IMO, for consistency with the "create inside a project" case,
1368 # these operations should also return ENOTSUP instead of EPERM.
1369 # Right now they're returning EPERM because the clasess' writable() method
1370 # usually returns False, and the Operations class transforms that accordingly.
1371 # However, for cases where the mount will never be writable, I think ENOTSUP
1372 # is a clearer error: it lets the user know they can't fix the problem by
1373 # adding permissions in Arvados, etc.
1374 @parameterized.parameterized_class([
1375     {'root_class': fusedir.MagicDirectory,
1376      'preset_dir': 'by_id',
1377      'preset_file': 'README',
1378      },
1379
1380     {'root_class': fusedir.SharedDirectory,
1381      'root_kwargs': {
1382          'exclude': run_test_server.fixture('users')['admin']['uuid'],
1383      },
1384      'preset_dir': 'Active User',
1385      },
1386
1387     {'root_class': fusedir.TagDirectory,
1388      'root_kwargs': {
1389          'tag': run_test_server.fixture('links')['foo_collection_tag']['name'],
1390      },
1391      'preset_dir': run_test_server.fixture('collections')['foo_collection_in_aproject']['uuid'],
1392      },
1393
1394     {'root_class': fusedir.TagsDirectory,
1395      'preset_dir': run_test_server.fixture('links')['foo_collection_tag']['name'],
1396      },
1397 ])
1398 class UnsupportedOperationsTest(UnsupportedCreateTest):
1399     preset_dir = None
1400     preset_file = None
1401
1402     def test_create(self):
1403         test_path = Path(self.mounttmp, 'test_create')
1404         with self.assertRaises(OSError) as exc_check:
1405             with test_path.open('w'):
1406                 pass
1407         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1408
1409     def test_mkdir(self):
1410         test_path = Path(self.mounttmp, 'test_mkdir')
1411         with self.assertRaises(OSError) as exc_check:
1412             test_path.mkdir()
1413         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1414
1415     def test_rename(self):
1416         src_name = self.preset_dir or self.preset_file
1417         if src_name is None:
1418             return
1419         test_src = Path(self.mounttmp, src_name)
1420         test_dst = test_src.with_name('test_dst')
1421         with self.assertRaises(OSError) as exc_check:
1422             test_src.rename(test_dst)
1423         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1424
1425     def test_rmdir(self):
1426         if self.preset_dir is None:
1427             return
1428         test_path = Path(self.mounttmp, self.preset_dir)
1429         with self.assertRaises(OSError) as exc_check:
1430             test_path.rmdir()
1431         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1432
1433     def test_unlink(self):
1434         if self.preset_file is None:
1435             return
1436         test_path = Path(self.mounttmp, self.preset_file)
1437         with self.assertRaises(OSError) as exc_check:
1438             test_path.unlink()
1439         self.assertEqual(exc_check.exception.errno, errno.EPERM)