21356: Remove all Python 2/3 compatibility imports
[arvados.git] / services / fuse / tests / test_mount.py
1 # Copyright (C) The Arvados Authors. All rights reserved.
2 #
3 # SPDX-License-Identifier: AGPL-3.0
4
5 import errno
6 import json
7 import llfuse
8 import logging
9 import os
10 import subprocess
11 import time
12 import unittest
13 import tempfile
14
15 from pathlib import Path
16 from unittest import mock
17
18 import arvados
19 import arvados_fuse as fuse
20 import parameterized
21
22 from arvados_fuse import fusedir
23
24 from . import run_test_server
25 from .integration_test import IntegrationTest
26 from .mount_test_base import MountTestBase
27 from .test_tmp_collection import storage_classes_desired
28
29 logger = logging.getLogger('arvados.arv-mount')
30
31 class AssertWithTimeout(object):
32     """Allow some time for an assertion to pass."""
33
34     def __init__(self, timeout=0):
35         self.timeout = timeout
36
37     def __iter__(self):
38         self.deadline = time.time() + self.timeout
39         self.done = False
40         return self
41
42     def __next__(self):
43         if self.done:
44             raise StopIteration
45         return self.attempt
46
47     def attempt(self, fn, *args, **kwargs):
48         try:
49             fn(*args, **kwargs)
50         except AssertionError:
51             if time.time() > self.deadline:
52                 raise
53             time.sleep(0.1)
54         else:
55             self.done = True
56
57 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
58 class FuseMountTest(MountTestBase):
59     def setUp(self):
60         super(FuseMountTest, self).setUp()
61
62         cw = arvados.CollectionWriter()
63
64         cw.start_new_file('thing1.txt')
65         cw.write("data 1")
66         cw.start_new_file('thing2.txt')
67         cw.write("data 2")
68
69         cw.start_new_stream('dir1')
70         cw.start_new_file('thing3.txt')
71         cw.write("data 3")
72         cw.start_new_file('thing4.txt')
73         cw.write("data 4")
74
75         cw.start_new_stream('dir2')
76         cw.start_new_file('thing5.txt')
77         cw.write("data 5")
78         cw.start_new_file('thing6.txt')
79         cw.write("data 6")
80
81         cw.start_new_stream('dir2/dir3')
82         cw.start_new_file('thing7.txt')
83         cw.write("data 7")
84
85         cw.start_new_file('thing8.txt')
86         cw.write("data 8")
87
88         cw.start_new_stream('edgecases')
89         for f in ":/.../-/*/ ".split("/"):
90             cw.start_new_file(f)
91             cw.write('x')
92
93         for f in ":/.../-/*/ ".split("/"):
94             cw.start_new_stream('edgecases/dirs/' + f)
95             cw.start_new_file('x/x')
96             cw.write('x')
97
98         self.testcollection = cw.finish()
99         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
100
101     def runTest(self):
102         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
103
104         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
105                                       'edgecases', 'dir1', 'dir2'])
106         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
107         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
108         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
109         self.assertDirContents('edgecases',
110                                "dirs/:/.../-/*/ ".split("/"))
111         self.assertDirContents('edgecases/dirs',
112                                ":/.../-/*/ ".split("/"))
113
114         files = {'thing1.txt': 'data 1',
115                  'thing2.txt': 'data 2',
116                  'dir1/thing3.txt': 'data 3',
117                  'dir1/thing4.txt': 'data 4',
118                  'dir2/thing5.txt': 'data 5',
119                  'dir2/thing6.txt': 'data 6',
120                  'dir2/dir3/thing7.txt': 'data 7',
121                  'dir2/dir3/thing8.txt': 'data 8'}
122
123         for k, v in files.items():
124             with open(os.path.join(self.mounttmp, k), 'rb') as f:
125                 self.assertEqual(v, f.read().decode())
126
127
128 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
129 class FuseMagicTest(MountTestBase):
130     def setUp(self, api=None):
131         super(FuseMagicTest, self).setUp(api=api)
132
133         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
134         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
135         self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
136         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
137         self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
138
139         cw = arvados.CollectionWriter()
140
141         cw.start_new_file('thing1.txt')
142         cw.write("data 1")
143
144         self.testcollection = cw.finish()
145         self.test_manifest = cw.manifest_text()
146         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
147         self.test_manifest_pdh = coll['portable_data_hash']
148
149     def runTest(self):
150         self.make_mount(fuse.MagicDirectory)
151
152         mount_ls = llfuse.listdir(self.mounttmp)
153         self.assertIn('README', mount_ls)
154         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
155                              arvados.util.uuid_pattern.match(fn)
156                              for fn in mount_ls),
157                          "new FUSE MagicDirectory has no collections or projects")
158         self.assertDirContents(self.testcollection, ['thing1.txt'])
159         self.assertDirContents(os.path.join('by_id', self.testcollection),
160                                ['thing1.txt'])
161         self.assertIn(self.collection_in_test_project,
162                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
163         self.assertIn(self.collection_in_test_project,
164                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
165         self.assertIn(self.collection_in_filter_group,
166                       llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
167         self.assertIn(self.collection_in_filter_group,
168                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
169
170
171         mount_ls = llfuse.listdir(self.mounttmp)
172         self.assertIn('README', mount_ls)
173         self.assertIn(self.testcollection, mount_ls)
174         self.assertIn(self.testcollection,
175                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
176         self.assertIn(self.test_project, mount_ls)
177         self.assertIn(self.test_project,
178                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
179         self.assertIn(self.filter_group,
180                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
181
182         with self.assertRaises(OSError):
183             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
184
185         files = {}
186         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
187
188         for k, v in files.items():
189             with open(os.path.join(self.mounttmp, k), 'rb') as f:
190                 self.assertEqual(v, f.read().decode())
191
192
193 class FuseTagsTest(MountTestBase):
194     def runTest(self):
195         self.make_mount(fuse.TagsDirectory)
196
197         d1 = llfuse.listdir(self.mounttmp)
198         d1.sort()
199         self.assertEqual(['foo_tag'], d1)
200
201         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
202         d2.sort()
203         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
204
205         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
206         d3.sort()
207         self.assertEqual(['foo'], d3)
208
209
210 class FuseTagsUpdateTest(MountTestBase):
211     def tag_collection(self, coll_uuid, tag_name):
212         return self.api.links().create(
213             body={'link': {'head_uuid': coll_uuid,
214                            'link_class': 'tag',
215                            'name': tag_name,
216         }}).execute()
217
218     def runTest(self):
219         self.make_mount(fuse.TagsDirectory, poll_time=1)
220
221         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
222
223         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
224         self.tag_collection(bar_uuid, 'fuse_test_tag')
225         for attempt in AssertWithTimeout(10):
226             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
227         self.assertDirContents('fuse_test_tag', [bar_uuid])
228
229         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
230         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
231         for attempt in AssertWithTimeout(10):
232             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
233
234         self.api.links().delete(uuid=l['uuid']).execute()
235         for attempt in AssertWithTimeout(10):
236             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
237
238
239 def fuseSharedTestHelper(mounttmp):
240     class Test(unittest.TestCase):
241         def runTest(self):
242             # Double check that we can open and read objects in this folder as a file,
243             # and that its contents are what we expect.
244             baz_path = os.path.join(
245                 mounttmp,
246                 'FUSE User',
247                 'FUSE Test Project',
248                 'collection in FUSE project',
249                 'baz')
250             with open(baz_path) as f:
251                 self.assertEqual("baz", f.read())
252
253             # check mtime on collection
254             st = os.stat(baz_path)
255             try:
256                 mtime = st.st_mtime_ns // 1000000000
257             except AttributeError:
258                 mtime = st.st_mtime
259             self.assertEqual(mtime, 1391448174)
260
261             # shared_dirs is a list of the directories exposed
262             # by fuse.SharedDirectory (i.e. any object visible
263             # to the current user)
264             shared_dirs = llfuse.listdir(mounttmp)
265             shared_dirs.sort()
266             self.assertIn('FUSE User', shared_dirs)
267
268             # fuse_user_objs is a list of the objects owned by the FUSE
269             # test user (which present as files in the 'FUSE User'
270             # directory)
271             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
272             fuse_user_objs.sort()
273             self.assertEqual(['FUSE Test Project',                    # project owned by user
274                               'collection #1 owned by FUSE',          # collection owned by user
275                               'collection #2 owned by FUSE'          # collection owned by user
276                           ], fuse_user_objs)
277
278             # test_proj_files is a list of the files in the FUSE Test Project.
279             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
280             test_proj_files.sort()
281             self.assertEqual(['collection in FUSE project'
282                           ], test_proj_files)
283
284
285     Test().runTest()
286
287 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
288 class FuseSharedTest(MountTestBase):
289     def runTest(self):
290         self.make_mount(fuse.SharedDirectory,
291                         exclude=self.api.users().current().execute()['uuid'])
292         keep = arvados.keep.KeepClient()
293         keep.put("baz".encode())
294
295         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
296
297
298 class FuseHomeTest(MountTestBase):
299     def runTest(self):
300         self.make_mount(fuse.ProjectDirectory,
301                         project_object=self.api.users().current().execute())
302
303         d1 = llfuse.listdir(self.mounttmp)
304         self.assertIn('Unrestricted public data', d1)
305
306         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
307         public_project = run_test_server.fixture('groups')[
308             'anonymously_accessible_project']
309         found_in = 0
310         found_not_in = 0
311         for name, item in run_test_server.fixture('collections').items():
312             if 'name' not in item:
313                 pass
314             elif item['owner_uuid'] == public_project['uuid']:
315                 self.assertIn(item['name'], d2)
316                 found_in += 1
317             else:
318                 # Artificial assumption here: there is no public
319                 # collection fixture with the same name as a
320                 # non-public collection.
321                 self.assertNotIn(item['name'], d2)
322                 found_not_in += 1
323         self.assertNotEqual(0, found_in)
324         self.assertNotEqual(0, found_not_in)
325
326         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
327         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
328
329
330 def fuseModifyFileTestHelperReadStartContents(mounttmp):
331     class Test(unittest.TestCase):
332         def runTest(self):
333             d1 = llfuse.listdir(mounttmp)
334             self.assertEqual(["file1.txt"], d1)
335             with open(os.path.join(mounttmp, "file1.txt")) as f:
336                 self.assertEqual("blub", f.read())
337     Test().runTest()
338
339 def fuseModifyFileTestHelperReadEndContents(mounttmp):
340     class Test(unittest.TestCase):
341         def runTest(self):
342             d1 = llfuse.listdir(mounttmp)
343             self.assertEqual(["file1.txt"], d1)
344             with open(os.path.join(mounttmp, "file1.txt")) as f:
345                 self.assertEqual("plnp", f.read())
346     Test().runTest()
347
348 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
349 class FuseModifyFileTest(MountTestBase):
350     def runTest(self):
351         collection = arvados.collection.Collection(api_client=self.api)
352         with collection.open("file1.txt", "w") as f:
353             f.write("blub")
354
355         collection.save_new()
356
357         m = self.make_mount(fuse.CollectionDirectory)
358         with llfuse.lock:
359             m.new_collection(collection.api_response(), collection)
360
361         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
362
363         with collection.open("file1.txt", "w") as f:
364             f.write("plnp")
365
366         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
367
368
369 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
370 class FuseAddFileToCollectionTest(MountTestBase):
371     def runTest(self):
372         collection = arvados.collection.Collection(api_client=self.api)
373         with collection.open("file1.txt", "w") as f:
374             f.write("blub")
375
376         collection.save_new()
377
378         m = self.make_mount(fuse.CollectionDirectory)
379         with llfuse.lock:
380             m.new_collection(collection.api_response(), collection)
381
382         d1 = llfuse.listdir(self.mounttmp)
383         self.assertEqual(["file1.txt"], d1)
384
385         with collection.open("file2.txt", "w") as f:
386             f.write("plnp")
387
388         d1 = llfuse.listdir(self.mounttmp)
389         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
390
391
392 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
393 class FuseRemoveFileFromCollectionTest(MountTestBase):
394     def runTest(self):
395         collection = arvados.collection.Collection(api_client=self.api)
396         with collection.open("file1.txt", "w") as f:
397             f.write("blub")
398
399         with collection.open("file2.txt", "w") as f:
400             f.write("plnp")
401
402         collection.save_new()
403
404         m = self.make_mount(fuse.CollectionDirectory)
405         with llfuse.lock:
406             m.new_collection(collection.api_response(), collection)
407
408         d1 = llfuse.listdir(self.mounttmp)
409         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
410
411         collection.remove("file2.txt")
412
413         d1 = llfuse.listdir(self.mounttmp)
414         self.assertEqual(["file1.txt"], d1)
415
416
417 def fuseCreateFileTestHelper(mounttmp):
418     class Test(unittest.TestCase):
419         def runTest(self):
420             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
421                 pass
422     Test().runTest()
423
424 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
425 class FuseCreateFileTest(MountTestBase):
426     def runTest(self):
427         collection = arvados.collection.Collection(api_client=self.api)
428         collection.save_new()
429
430         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
431         self.assertEqual(collection2["manifest_text"], "")
432
433         collection.save_new()
434
435         m = self.make_mount(fuse.CollectionDirectory)
436         with llfuse.lock:
437             m.new_collection(collection.api_response(), collection)
438         self.assertTrue(m.writable())
439
440         self.assertNotIn("file1.txt", collection)
441
442         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
443
444         self.assertIn("file1.txt", collection)
445
446         d1 = llfuse.listdir(self.mounttmp)
447         self.assertEqual(["file1.txt"], d1)
448
449         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
450         self.assertRegex(collection2["manifest_text"],
451             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
452
453
454 def fuseWriteFileTestHelperWriteFile(mounttmp):
455     class Test(unittest.TestCase):
456         def runTest(self):
457             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
458                 f.write("Hello world!")
459     Test().runTest()
460
461 def fuseWriteFileTestHelperReadFile(mounttmp):
462     class Test(unittest.TestCase):
463         def runTest(self):
464             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
465                 self.assertEqual(f.read(), "Hello world!")
466     Test().runTest()
467
468 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
469 class FuseWriteFileTest(MountTestBase):
470     def runTest(self):
471         collection = arvados.collection.Collection(api_client=self.api)
472         collection.save_new()
473
474         m = self.make_mount(fuse.CollectionDirectory)
475         with llfuse.lock:
476             m.new_collection(collection.api_response(), collection)
477         self.assertTrue(m.writable())
478
479         self.assertNotIn("file1.txt", collection)
480
481         self.assertEqual(0, self.operations.write_counter.get())
482         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
483         self.assertEqual(12, self.operations.write_counter.get())
484
485         with collection.open("file1.txt") as f:
486             self.assertEqual(f.read(), "Hello world!")
487
488         self.assertEqual(0, self.operations.read_counter.get())
489         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
490         self.assertEqual(12, self.operations.read_counter.get())
491
492         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
493         self.assertRegex(collection2["manifest_text"],
494             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
495
496
497 def fuseUpdateFileTestHelper(mounttmp):
498     class Test(unittest.TestCase):
499         def runTest(self):
500             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
501                 f.write("Hello world!")
502
503             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
504                 fr = f.read()
505                 self.assertEqual(fr, "Hello world!")
506                 f.seek(0)
507                 f.write("Hola mundo!")
508                 f.seek(0)
509                 fr = f.read()
510                 self.assertEqual(fr, "Hola mundo!!")
511
512             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
513                 self.assertEqual(f.read(), "Hola mundo!!")
514
515     Test().runTest()
516
517 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
518 class FuseUpdateFileTest(MountTestBase):
519     def runTest(self):
520         collection = arvados.collection.Collection(api_client=self.api)
521         collection.save_new()
522
523         m = self.make_mount(fuse.CollectionDirectory)
524         with llfuse.lock:
525             m.new_collection(collection.api_response(), collection)
526         self.assertTrue(m.writable())
527
528         # See note in MountTestBase.setUp
529         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
530
531         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
532         self.assertRegex(collection2["manifest_text"],
533             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
534
535
536 def fuseMkdirTestHelper(mounttmp):
537     class Test(unittest.TestCase):
538         def runTest(self):
539             with self.assertRaises(IOError):
540                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
541                     f.write("Hello world!")
542
543             os.mkdir(os.path.join(mounttmp, "testdir"))
544
545             with self.assertRaises(OSError):
546                 os.mkdir(os.path.join(mounttmp, "testdir"))
547
548             d1 = llfuse.listdir(mounttmp)
549             self.assertEqual(["testdir"], d1)
550
551             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
552                 f.write("Hello world!")
553
554             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
555             self.assertEqual(["file1.txt"], d1)
556
557     Test().runTest()
558
559 class FuseMkdirTest(MountTestBase):
560     def runTest(self):
561         collection = arvados.collection.Collection(api_client=self.api)
562         collection.save_new()
563
564         m = self.make_mount(fuse.CollectionDirectory)
565         with llfuse.lock:
566             m.new_collection(collection.api_response(), collection)
567         self.assertTrue(m.writable())
568
569         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
570
571         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
572         self.assertRegex(collection2["manifest_text"],
573             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
574
575
576 def fuseRmTestHelperWriteFile(mounttmp):
577     class Test(unittest.TestCase):
578         def runTest(self):
579             os.mkdir(os.path.join(mounttmp, "testdir"))
580
581             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
582                 f.write("Hello world!")
583
584     Test().runTest()
585
586 def fuseRmTestHelperDeleteFile(mounttmp):
587     class Test(unittest.TestCase):
588         def runTest(self):
589             # Can't delete because it's not empty
590             with self.assertRaises(OSError):
591                 os.rmdir(os.path.join(mounttmp, "testdir"))
592
593             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
594             self.assertEqual(["file1.txt"], d1)
595
596             # Delete file
597             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
598
599             # Make sure it's empty
600             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
601             self.assertEqual([], d1)
602
603             # Try to delete it again
604             with self.assertRaises(OSError):
605                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
606
607     Test().runTest()
608
609 def fuseRmTestHelperRmdir(mounttmp):
610     class Test(unittest.TestCase):
611         def runTest(self):
612             # Should be able to delete now that it is empty
613             os.rmdir(os.path.join(mounttmp, "testdir"))
614
615             # Make sure it's empty
616             d1 = llfuse.listdir(os.path.join(mounttmp))
617             self.assertEqual([], d1)
618
619             # Try to delete it again
620             with self.assertRaises(OSError):
621                 os.rmdir(os.path.join(mounttmp, "testdir"))
622
623     Test().runTest()
624
625 class FuseRmTest(MountTestBase):
626     def runTest(self):
627         collection = arvados.collection.Collection(api_client=self.api)
628         collection.save_new()
629
630         m = self.make_mount(fuse.CollectionDirectory)
631         with llfuse.lock:
632             m.new_collection(collection.api_response(), collection)
633         self.assertTrue(m.writable())
634
635         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
636
637         # Starting manifest
638         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
639         self.assertRegex(collection2["manifest_text"],
640             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
641         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
642
643         # Empty directories are represented by an empty file named "."
644         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
645         self.assertRegex(collection2["manifest_text"],
646                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
647
648         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
649
650         # manifest should be empty now.
651         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
652         self.assertEqual(collection2["manifest_text"], "")
653
654
655 def fuseMvFileTestHelperWriteFile(mounttmp):
656     class Test(unittest.TestCase):
657         def runTest(self):
658             os.mkdir(os.path.join(mounttmp, "testdir"))
659
660             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
661                 f.write("Hello world!")
662
663     Test().runTest()
664
665 def fuseMvFileTestHelperMoveFile(mounttmp):
666     class Test(unittest.TestCase):
667         def runTest(self):
668             d1 = llfuse.listdir(os.path.join(mounttmp))
669             self.assertEqual(["testdir"], d1)
670             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
671             self.assertEqual(["file1.txt"], d1)
672
673             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
674
675             d1 = llfuse.listdir(os.path.join(mounttmp))
676             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
677             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
678             self.assertEqual([], d1)
679
680     Test().runTest()
681
682 class FuseMvFileTest(MountTestBase):
683     def runTest(self):
684         collection = arvados.collection.Collection(api_client=self.api)
685         collection.save_new()
686
687         m = self.make_mount(fuse.CollectionDirectory)
688         with llfuse.lock:
689             m.new_collection(collection.api_response(), collection)
690         self.assertTrue(m.writable())
691
692         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
693
694         # Starting manifest
695         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
696         self.assertRegex(collection2["manifest_text"],
697             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
698
699         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
700
701         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
702         self.assertRegex(collection2["manifest_text"],
703             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
704
705
706 def fuseRenameTestHelper(mounttmp):
707     class Test(unittest.TestCase):
708         def runTest(self):
709             os.mkdir(os.path.join(mounttmp, "testdir"))
710
711             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
712                 f.write("Hello world!")
713
714     Test().runTest()
715
716 class FuseRenameTest(MountTestBase):
717     def runTest(self):
718         collection = arvados.collection.Collection(api_client=self.api)
719         collection.save_new()
720
721         m = self.make_mount(fuse.CollectionDirectory)
722         with llfuse.lock:
723             m.new_collection(collection.api_response(), collection)
724         self.assertTrue(m.writable())
725
726         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
727
728         # Starting manifest
729         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
730         self.assertRegex(collection2["manifest_text"],
731             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
732
733         d1 = llfuse.listdir(os.path.join(self.mounttmp))
734         self.assertEqual(["testdir"], d1)
735         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
736         self.assertEqual(["file1.txt"], d1)
737
738         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
739
740         d1 = llfuse.listdir(os.path.join(self.mounttmp))
741         self.assertEqual(["testdir2"], sorted(d1))
742         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
743         self.assertEqual(["file1.txt"], d1)
744
745         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
746         self.assertRegex(collection2["manifest_text"],
747             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
748
749
750 class FuseUpdateFromEventTest(MountTestBase):
751     def runTest(self):
752         collection = arvados.collection.Collection(api_client=self.api)
753         collection.save_new()
754
755         m = self.make_mount(fuse.CollectionDirectory)
756         with llfuse.lock:
757             m.new_collection(collection.api_response(), collection)
758
759         self.operations.listen_for_events()
760
761         d1 = llfuse.listdir(os.path.join(self.mounttmp))
762         self.assertEqual([], sorted(d1))
763
764         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
765             with collection2.open("file1.txt", "w") as f:
766                 f.write("foo")
767
768         for attempt in AssertWithTimeout(10):
769             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
770
771
772 class FuseDeleteProjectEventTest(MountTestBase):
773     def runTest(self):
774
775         aproject = self.api.groups().create(body={
776             "name": "aproject",
777             "group_class": "project"
778         }).execute()
779
780         bproject = self.api.groups().create(body={
781             "name": "bproject",
782             "group_class": "project",
783             "owner_uuid": aproject["uuid"]
784         }).execute()
785
786         self.make_mount(fuse.ProjectDirectory,
787                         project_object=self.api.users().current().execute())
788
789         self.operations.listen_for_events()
790
791         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
792         self.assertEqual(["bproject"], sorted(d1))
793
794         self.api.groups().delete(uuid=bproject["uuid"]).execute()
795
796         for attempt in AssertWithTimeout(10):
797             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
798
799
800 def fuseFileConflictTestHelper(mounttmp, uuid, keeptmp, settings):
801     class Test(unittest.TestCase):
802         def runTest(self):
803             os.environ['KEEP_LOCAL_STORE'] = keeptmp
804
805             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
806                 with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
807                     with collection2.open("file1.txt", "w") as f2:
808                         f2.write("foo")
809                 f.write("bar")
810
811             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
812             self.assertEqual(len(d1), 2)
813
814             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
815                 self.assertEqual(f.read(), "bar")
816
817             self.assertRegex(d1[1],
818                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
819
820             with open(os.path.join(mounttmp, d1[1]), "r") as f:
821                 self.assertEqual(f.read(), "foo")
822
823     Test().runTest()
824
825 class FuseFileConflictTest(MountTestBase):
826     def runTest(self):
827         collection = arvados.collection.Collection(api_client=self.api)
828         collection.save_new()
829
830         m = self.make_mount(fuse.CollectionDirectory)
831         with llfuse.lock:
832             m.new_collection(collection.api_response(), collection)
833
834         d1 = llfuse.listdir(os.path.join(self.mounttmp))
835         self.assertEqual([], sorted(d1))
836
837         # See note in MountTestBase.setUp
838         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
839
840
841 def fuseUnlinkOpenFileTest(mounttmp):
842     class Test(unittest.TestCase):
843         def runTest(self):
844             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
845                 f.write("foo")
846
847                 d1 = llfuse.listdir(os.path.join(mounttmp))
848                 self.assertEqual(["file1.txt"], sorted(d1))
849
850                 os.remove(os.path.join(mounttmp, "file1.txt"))
851
852                 d1 = llfuse.listdir(os.path.join(mounttmp))
853                 self.assertEqual([], sorted(d1))
854
855                 f.seek(0)
856                 self.assertEqual(f.read(), "foo")
857                 f.write("bar")
858
859                 f.seek(0)
860                 self.assertEqual(f.read(), "foobar")
861
862     Test().runTest()
863
864 class FuseUnlinkOpenFileTest(MountTestBase):
865     def runTest(self):
866         collection = arvados.collection.Collection(api_client=self.api)
867         collection.save_new()
868
869         m = self.make_mount(fuse.CollectionDirectory)
870         with llfuse.lock:
871             m.new_collection(collection.api_response(), collection)
872
873         # See note in MountTestBase.setUp
874         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
875
876         self.assertEqual(collection.manifest_text(), "")
877
878
879 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
880     class Test(unittest.TestCase):
881         def runTest(self):
882             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
883                 f.write("Hello world!")
884
885             d1 = os.listdir(os.path.join(mounttmp, uuid1))
886             self.assertEqual(["file1.txt"], sorted(d1))
887             d1 = os.listdir(os.path.join(mounttmp, uuid2))
888             self.assertEqual([], sorted(d1))
889
890     Test().runTest()
891
892 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
893     class Test(unittest.TestCase):
894         def runTest(self):
895             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
896
897             d1 = os.listdir(os.path.join(mounttmp, uuid1))
898             self.assertEqual([], sorted(d1))
899             d1 = os.listdir(os.path.join(mounttmp, uuid2))
900             self.assertEqual(["file2.txt"], sorted(d1))
901
902     Test().runTest()
903
904 class FuseMvFileBetweenCollectionsTest(MountTestBase):
905     def runTest(self):
906         collection1 = arvados.collection.Collection(api_client=self.api)
907         collection1.save_new()
908
909         collection2 = arvados.collection.Collection(api_client=self.api)
910         collection2.save_new()
911
912         m = self.make_mount(fuse.MagicDirectory)
913
914         # See note in MountTestBase.setUp
915         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
916                                                   collection1.manifest_locator(),
917                                                   collection2.manifest_locator()))
918
919         collection1.update()
920         collection2.update()
921
922         self.assertRegex(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
923         self.assertEqual(collection2.manifest_text(), "")
924
925         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
926                                                   collection1.manifest_locator(),
927                                                   collection2.manifest_locator()))
928
929         collection1.update()
930         collection2.update()
931
932         self.assertEqual(collection1.manifest_text(), "")
933         self.assertRegex(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
934
935         collection1.stop_threads()
936         collection2.stop_threads()
937
938
939 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
940     class Test(unittest.TestCase):
941         def runTest(self):
942             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
943             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
944                 f.write("Hello world!")
945
946             d1 = os.listdir(os.path.join(mounttmp, uuid1))
947             self.assertEqual(["testdir"], sorted(d1))
948             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
949             self.assertEqual(["file1.txt"], sorted(d1))
950
951             d1 = os.listdir(os.path.join(mounttmp, uuid2))
952             self.assertEqual([], sorted(d1))
953
954     Test().runTest()
955
956
957 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
958     class Test(unittest.TestCase):
959         def runTest(self):
960             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
961
962             d1 = os.listdir(os.path.join(mounttmp, uuid1))
963             self.assertEqual([], sorted(d1))
964
965             d1 = os.listdir(os.path.join(mounttmp, uuid2))
966             self.assertEqual(["testdir2"], sorted(d1))
967             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
968             self.assertEqual(["file1.txt"], sorted(d1))
969
970             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
971                 self.assertEqual(f.read(), "Hello world!")
972
973     Test().runTest()
974
975 class FuseMvDirBetweenCollectionsTest(MountTestBase):
976     def runTest(self):
977         collection1 = arvados.collection.Collection(api_client=self.api)
978         collection1.save_new()
979
980         collection2 = arvados.collection.Collection(api_client=self.api)
981         collection2.save_new()
982
983         m = self.make_mount(fuse.MagicDirectory)
984
985         # See note in MountTestBase.setUp
986         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
987                                                   collection1.manifest_locator(),
988                                                   collection2.manifest_locator()))
989
990         collection1.update()
991         collection2.update()
992
993         self.assertRegex(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
994         self.assertEqual(collection2.manifest_text(), "")
995
996         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
997                                                   collection1.manifest_locator(),
998                                                   collection2.manifest_locator()))
999
1000         collection1.update()
1001         collection2.update()
1002
1003         self.assertEqual(collection1.manifest_text(), "")
1004         self.assertRegex(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
1005
1006         collection1.stop_threads()
1007         collection2.stop_threads()
1008
1009 def fuseProjectMkdirTestHelper1(mounttmp):
1010     class Test(unittest.TestCase):
1011         def runTest(self):
1012             os.mkdir(os.path.join(mounttmp, "testcollection"))
1013             with self.assertRaises(OSError):
1014                 os.mkdir(os.path.join(mounttmp, "testcollection"))
1015     Test().runTest()
1016
1017 def fuseProjectMkdirTestHelper2(mounttmp):
1018     class Test(unittest.TestCase):
1019         def runTest(self):
1020             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1021                 f.write("Hello world!")
1022             with self.assertRaises(OSError):
1023                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1024             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1025             with self.assertRaises(OSError):
1026                 os.remove(os.path.join(mounttmp, "testcollection"))
1027             os.rmdir(os.path.join(mounttmp, "testcollection"))
1028     Test().runTest()
1029
1030 class FuseProjectMkdirRmdirTest(MountTestBase):
1031     def runTest(self):
1032         self.make_mount(fuse.ProjectDirectory,
1033                         project_object=self.api.users().current().execute())
1034
1035         d1 = llfuse.listdir(self.mounttmp)
1036         self.assertNotIn('testcollection', d1)
1037
1038         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1039
1040         d1 = llfuse.listdir(self.mounttmp)
1041         self.assertIn('testcollection', d1)
1042
1043         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1044
1045         d1 = llfuse.listdir(self.mounttmp)
1046         self.assertNotIn('testcollection', d1)
1047
1048
1049 def fuseProjectMvTestHelper1(mounttmp):
1050     class Test(unittest.TestCase):
1051         def runTest(self):
1052             d1 = llfuse.listdir(mounttmp)
1053             self.assertNotIn('testcollection', d1)
1054
1055             os.mkdir(os.path.join(mounttmp, "testcollection"))
1056
1057             d1 = llfuse.listdir(mounttmp)
1058             self.assertIn('testcollection', d1)
1059
1060             with self.assertRaises(OSError):
1061                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1062
1063             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1064
1065             d1 = llfuse.listdir(mounttmp)
1066             self.assertNotIn('testcollection', d1)
1067
1068             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1069             self.assertIn('testcollection', d1)
1070
1071     Test().runTest()
1072
1073 class FuseProjectMvTest(MountTestBase):
1074     def runTest(self):
1075         self.make_mount(fuse.ProjectDirectory,
1076                         project_object=self.api.users().current().execute())
1077
1078         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1079
1080
1081 def fuseFsyncTestHelper(mounttmp, k):
1082     class Test(unittest.TestCase):
1083         def runTest(self):
1084             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1085             os.fsync(fd)
1086             os.close(fd)
1087
1088     Test().runTest()
1089
1090 class FuseFsyncTest(FuseMagicTest):
1091     def runTest(self):
1092         self.make_mount(fuse.MagicDirectory)
1093         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1094
1095
1096 class MagicDirApiError(FuseMagicTest):
1097     def setUp(self):
1098         api = mock.MagicMock()
1099         api.keep.block_cache = mock.MagicMock(cache_max=1)
1100         super(MagicDirApiError, self).setUp(api=api)
1101         api.collections().get().execute.side_effect = iter([
1102             Exception('API fail'),
1103             {
1104                 "manifest_text": self.test_manifest,
1105                 "portable_data_hash": self.test_manifest_pdh,
1106             },
1107         ])
1108         api.keep.get.side_effect = Exception('Keep fail')
1109
1110     def runTest(self):
1111         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1112             self.make_mount(fuse.MagicDirectory)
1113
1114             self.operations.inodes.inode_cache.cap = 1
1115             self.operations.inodes.inode_cache.min_entries = 2
1116
1117             with self.assertRaises(OSError):
1118                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1119
1120             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1121
1122
1123 class SanitizeFilenameTest(MountTestBase):
1124     def test_sanitize_filename(self):
1125         pdir = fuse.ProjectDirectory(
1126             1, fuse.Inodes(None), self.api, 0, False, None,
1127             project_object=self.api.users().current().execute(),
1128         )
1129         acceptable = [
1130             "foo.txt",
1131             ".foo",
1132             "..foo",
1133             "...",
1134             "foo...",
1135             "foo..",
1136             "foo.",
1137             "-",
1138             "\x01\x02\x03",
1139             ]
1140         unacceptable = [
1141             "f\00",
1142             "\00\00",
1143             "/foo",
1144             "foo/",
1145             "//",
1146             ]
1147         for f in acceptable:
1148             self.assertEqual(f, pdir.sanitize_filename(f))
1149         for f in unacceptable:
1150             self.assertNotEqual(f, pdir.sanitize_filename(f))
1151             # The sanitized filename should be the same length, though.
1152             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1153         # Special cases
1154         self.assertEqual("_", pdir.sanitize_filename(""))
1155         self.assertEqual("_", pdir.sanitize_filename("."))
1156         self.assertEqual("__", pdir.sanitize_filename(".."))
1157
1158
1159 class FuseMagicTestPDHOnly(MountTestBase):
1160     def setUp(self, api=None):
1161         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1162
1163         cw = arvados.CollectionWriter()
1164
1165         cw.start_new_file('thing1.txt')
1166         cw.write("data 1")
1167
1168         self.testcollection = cw.finish()
1169         self.test_manifest = cw.manifest_text()
1170         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1171         self.testcollectionuuid = str(created['uuid'])
1172
1173     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1174         if skip_pdh_only is True:
1175             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1176         else:
1177             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1178
1179         mount_ls = llfuse.listdir(self.mounttmp)
1180         self.assertIn('README', mount_ls)
1181         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1182                              arvados.util.uuid_pattern.match(fn)
1183                              for fn in mount_ls),
1184                          "new FUSE MagicDirectory lists Collection")
1185
1186         # look up using pdh should succeed in all cases
1187         self.assertDirContents(self.testcollection, ['thing1.txt'])
1188         self.assertDirContents(os.path.join('by_id', self.testcollection),
1189                                ['thing1.txt'])
1190         mount_ls = llfuse.listdir(self.mounttmp)
1191         self.assertIn('README', mount_ls)
1192         self.assertIn(self.testcollection, mount_ls)
1193         self.assertIn(self.testcollection,
1194                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1195
1196         files = {}
1197         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1198
1199         for k, v in files.items():
1200             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1201                 self.assertEqual(v, f.read().decode())
1202
1203         # look up using uuid should fail when pdh_only is set
1204         if pdh_only is True:
1205             with self.assertRaises(OSError):
1206                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1207                                ['thing1.txt'])
1208         else:
1209             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1210                                ['thing1.txt'])
1211
1212     def test_with_pdh_only_true(self):
1213         self.verify_pdh_only(pdh_only=True)
1214
1215     def test_with_pdh_only_false(self):
1216         self.verify_pdh_only(pdh_only=False)
1217
1218     def test_with_default_by_id(self):
1219         self.verify_pdh_only(skip_pdh_only=True)
1220
1221
1222 class SlashSubstitutionTest(IntegrationTest):
1223     mnt_args = [
1224         '--read-write',
1225         '--mount-home', 'zzz',
1226         '--fsns', '[SLASH]'
1227     ]
1228
1229     def setUp(self):
1230         super(SlashSubstitutionTest, self).setUp()
1231
1232         self.api = arvados.safeapi.ThreadSafeApiCache(
1233             arvados.config.settings(),
1234             version='v1'
1235         )
1236         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1237         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1238         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1239
1240     @IntegrationTest.mount(argv=mnt_args)
1241     def test_slash_substitution_before_listing(self):
1242         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1243         self.checkContents()
1244     @staticmethod
1245     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1246         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1247             f.write('xxx')
1248         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1249             f.write('foo')
1250
1251     @IntegrationTest.mount(argv=mnt_args)
1252     @mock.patch('arvados.util.get_config_once')
1253     def test_slash_substitution_after_listing(self, get_config_once):
1254         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1255         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1256         self.checkContents()
1257     @staticmethod
1258     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1259         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1260             f.write('xxx')
1261         os.listdir(tmpdir)
1262         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1263             f.write('foo')
1264
1265     def checkContents(self):
1266         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1267         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1268
1269     @IntegrationTest.mount(argv=mnt_args)
1270     @mock.patch('arvados.util.get_config_once')
1271     def test_slash_substitution_conflict(self, get_config_once):
1272         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1273         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1274         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1275         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1276         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1277         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1278     @staticmethod
1279     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1280         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1281             f.write('foo')
1282
1283 class StorageClassesTest(IntegrationTest):
1284     mnt_args = [
1285         '--read-write',
1286         '--mount-home', 'homedir',
1287     ]
1288
1289     def setUp(self):
1290         super(StorageClassesTest, self).setUp()
1291         self.api = arvados.safeapi.ThreadSafeApiCache(
1292             arvados.config.settings(),
1293             version='v1',
1294         )
1295
1296     @IntegrationTest.mount(argv=mnt_args)
1297     def test_collection_default_storage_classes(self):
1298         coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
1299         self.api.collections().create(body={'name':'a_collection'}).execute()
1300         self.pool_test(coll_path)
1301     @staticmethod
1302     def _test_collection_default_storage_classes(self, coll):
1303         self.assertEqual(storage_classes_desired(coll), ['default'])
1304
1305     @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
1306     def test_collection_custom_storage_classes(self):
1307         coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
1308         os.mkdir(coll_path)
1309         self.pool_test(coll_path)
1310     @staticmethod
1311     def _test_collection_custom_storage_classes(self, coll):
1312         self.assertEqual(storage_classes_desired(coll), ['foo'])
1313
1314 def _readonlyCollectionTestHelper(mounttmp):
1315     f = open(os.path.join(mounttmp, 'thing1.txt'), 'rt')
1316     # Testing that close() doesn't raise an error.
1317     f.close()
1318
1319 class ReadonlyCollectionTest(MountTestBase):
1320     def setUp(self):
1321         super(ReadonlyCollectionTest, self).setUp()
1322         cw = arvados.collection.Collection()
1323         with cw.open('thing1.txt', 'wt') as f:
1324             f.write("data 1")
1325         cw.save_new(owner_uuid=run_test_server.fixture("groups")["aproject"]["uuid"])
1326         self.testcollection = cw.api_response()
1327
1328     def runTest(self):
1329         settings = arvados.config.settings().copy()
1330         settings["ARVADOS_API_TOKEN"] = run_test_server.fixture("api_client_authorizations")["project_viewer"]["api_token"]
1331         self.api = arvados.safeapi.ThreadSafeApiCache(settings, version='v1')
1332         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection, enable_write=False)
1333
1334         self.pool.apply(_readonlyCollectionTestHelper, (self.mounttmp,))
1335
1336
1337 @parameterized.parameterized_class([
1338     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1339         'project_object': run_test_server.fixture('users')['admin'],
1340     }},
1341     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1342         'project_object': run_test_server.fixture('groups')['public'],
1343     }},
1344 ])
1345 class UnsupportedCreateTest(MountTestBase):
1346     root_class = None
1347     root_kwargs = {}
1348
1349     def setUp(self):
1350         super().setUp()
1351         if 'prefs' in self.root_kwargs.get('project_object', ()):
1352             self.root_kwargs['project_object']['prefs'] = {}
1353         self.make_mount(self.root_class, **self.root_kwargs)
1354         # Make sure the directory knows about its top-level ents.
1355         os.listdir(self.mounttmp)
1356
1357     def test_create(self):
1358         test_path = Path(self.mounttmp, 'test_create')
1359         with self.assertRaises(OSError) as exc_check:
1360             with test_path.open('w'):
1361                 pass
1362         self.assertEqual(exc_check.exception.errno, errno.ENOTSUP)
1363
1364
1365 # FIXME: IMO, for consistency with the "create inside a project" case,
1366 # these operations should also return ENOTSUP instead of EPERM.
1367 # Right now they're returning EPERM because the clasess' writable() method
1368 # usually returns False, and the Operations class transforms that accordingly.
1369 # However, for cases where the mount will never be writable, I think ENOTSUP
1370 # is a clearer error: it lets the user know they can't fix the problem by
1371 # adding permissions in Arvados, etc.
1372 @parameterized.parameterized_class([
1373     {'root_class': fusedir.MagicDirectory,
1374      'preset_dir': 'by_id',
1375      'preset_file': 'README',
1376      },
1377
1378     {'root_class': fusedir.SharedDirectory,
1379      'root_kwargs': {
1380          'exclude': run_test_server.fixture('users')['admin']['uuid'],
1381      },
1382      'preset_dir': 'Active User',
1383      },
1384
1385     {'root_class': fusedir.TagDirectory,
1386      'root_kwargs': {
1387          'tag': run_test_server.fixture('links')['foo_collection_tag']['name'],
1388      },
1389      'preset_dir': run_test_server.fixture('collections')['foo_collection_in_aproject']['uuid'],
1390      },
1391
1392     {'root_class': fusedir.TagsDirectory,
1393      'preset_dir': run_test_server.fixture('links')['foo_collection_tag']['name'],
1394      },
1395 ])
1396 class UnsupportedOperationsTest(UnsupportedCreateTest):
1397     preset_dir = None
1398     preset_file = None
1399
1400     def test_create(self):
1401         test_path = Path(self.mounttmp, 'test_create')
1402         with self.assertRaises(OSError) as exc_check:
1403             with test_path.open('w'):
1404                 pass
1405         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1406
1407     def test_mkdir(self):
1408         test_path = Path(self.mounttmp, 'test_mkdir')
1409         with self.assertRaises(OSError) as exc_check:
1410             test_path.mkdir()
1411         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1412
1413     def test_rename(self):
1414         src_name = self.preset_dir or self.preset_file
1415         if src_name is None:
1416             return
1417         test_src = Path(self.mounttmp, src_name)
1418         test_dst = test_src.with_name('test_dst')
1419         with self.assertRaises(OSError) as exc_check:
1420             test_src.rename(test_dst)
1421         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1422
1423     def test_rmdir(self):
1424         if self.preset_dir is None:
1425             return
1426         test_path = Path(self.mounttmp, self.preset_dir)
1427         with self.assertRaises(OSError) as exc_check:
1428             test_path.rmdir()
1429         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1430
1431     def test_unlink(self):
1432         if self.preset_file is None:
1433             return
1434         test_path = Path(self.mounttmp, self.preset_file)
1435         with self.assertRaises(OSError) as exc_check:
1436             test_path.unlink()
1437         self.assertEqual(exc_check.exception.errno, errno.EPERM)