17778: Merge branch 'master' into 17778-doc-update
[arvados.git] / services / fuse / tests / test_mount.py
1 # Copyright (C) The Arvados Authors. All rights reserved.
2 #
3 # SPDX-License-Identifier: AGPL-3.0
4
5 from __future__ import absolute_import
6 from future.utils import viewitems
7 from builtins import str
8 from builtins import object
9 from six import assertRegex
10 import json
11 import llfuse
12 import logging
13 import mock
14 import os
15 import subprocess
16 import time
17 import unittest
18
19 import arvados
20 import arvados_fuse as fuse
21 from . import run_test_server
22
23 from .integration_test import IntegrationTest
24 from .mount_test_base import MountTestBase
25 from .test_tmp_collection import storage_classes_desired
26
27 logger = logging.getLogger('arvados.arv-mount')
28
29
30 class AssertWithTimeout(object):
31     """Allow some time for an assertion to pass."""
32
33     def __init__(self, timeout=0):
34         self.timeout = timeout
35
36     def __iter__(self):
37         self.deadline = time.time() + self.timeout
38         self.done = False
39         return self
40
41     def __next__(self):
42         if self.done:
43             raise StopIteration
44         return self.attempt
45
46     def attempt(self, fn, *args, **kwargs):
47         try:
48             fn(*args, **kwargs)
49         except AssertionError:
50             if time.time() > self.deadline:
51                 raise
52             time.sleep(0.1)
53         else:
54             self.done = True
55
56
57 class FuseMountTest(MountTestBase):
58     def setUp(self):
59         super(FuseMountTest, self).setUp()
60
61         cw = arvados.CollectionWriter()
62
63         cw.start_new_file('thing1.txt')
64         cw.write("data 1")
65         cw.start_new_file('thing2.txt')
66         cw.write("data 2")
67
68         cw.start_new_stream('dir1')
69         cw.start_new_file('thing3.txt')
70         cw.write("data 3")
71         cw.start_new_file('thing4.txt')
72         cw.write("data 4")
73
74         cw.start_new_stream('dir2')
75         cw.start_new_file('thing5.txt')
76         cw.write("data 5")
77         cw.start_new_file('thing6.txt')
78         cw.write("data 6")
79
80         cw.start_new_stream('dir2/dir3')
81         cw.start_new_file('thing7.txt')
82         cw.write("data 7")
83
84         cw.start_new_file('thing8.txt')
85         cw.write("data 8")
86
87         cw.start_new_stream('edgecases')
88         for f in ":/.../-/*/ ".split("/"):
89             cw.start_new_file(f)
90             cw.write('x')
91
92         for f in ":/.../-/*/ ".split("/"):
93             cw.start_new_stream('edgecases/dirs/' + f)
94             cw.start_new_file('x/x')
95             cw.write('x')
96
97         self.testcollection = cw.finish()
98         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
99
100     def runTest(self):
101         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
102
103         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
104                                       'edgecases', 'dir1', 'dir2'])
105         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
106         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
107         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
108         self.assertDirContents('edgecases',
109                                "dirs/:/.../-/*/ ".split("/"))
110         self.assertDirContents('edgecases/dirs',
111                                ":/.../-/*/ ".split("/"))
112
113         files = {'thing1.txt': 'data 1',
114                  'thing2.txt': 'data 2',
115                  'dir1/thing3.txt': 'data 3',
116                  'dir1/thing4.txt': 'data 4',
117                  'dir2/thing5.txt': 'data 5',
118                  'dir2/thing6.txt': 'data 6',
119                  'dir2/dir3/thing7.txt': 'data 7',
120                  'dir2/dir3/thing8.txt': 'data 8'}
121
122         for k, v in viewitems(files):
123             with open(os.path.join(self.mounttmp, k), 'rb') as f:
124                 self.assertEqual(v, f.read().decode())
125
126
127 class FuseMagicTest(MountTestBase):
128     def setUp(self, api=None):
129         super(FuseMagicTest, self).setUp(api=api)
130
131         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
132         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
133         self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
134         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
135         self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
136
137         cw = arvados.CollectionWriter()
138
139         cw.start_new_file('thing1.txt')
140         cw.write("data 1")
141
142         self.testcollection = cw.finish()
143         self.test_manifest = cw.manifest_text()
144         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
145         self.test_manifest_pdh = coll['portable_data_hash']
146
147     def runTest(self):
148         self.make_mount(fuse.MagicDirectory)
149
150         mount_ls = llfuse.listdir(self.mounttmp)
151         self.assertIn('README', mount_ls)
152         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
153                              arvados.util.uuid_pattern.match(fn)
154                              for fn in mount_ls),
155                          "new FUSE MagicDirectory has no collections or projects")
156         self.assertDirContents(self.testcollection, ['thing1.txt'])
157         self.assertDirContents(os.path.join('by_id', self.testcollection),
158                                ['thing1.txt'])
159         self.assertIn(self.collection_in_test_project,
160                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
161         self.assertIn(self.collection_in_test_project,
162                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
163         self.assertIn(self.collection_in_filter_group,
164                       llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
165         self.assertIn(self.collection_in_filter_group,
166                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
167
168
169         mount_ls = llfuse.listdir(self.mounttmp)
170         self.assertIn('README', mount_ls)
171         self.assertIn(self.testcollection, mount_ls)
172         self.assertIn(self.testcollection,
173                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
174         self.assertIn(self.test_project, mount_ls)
175         self.assertIn(self.test_project,
176                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
177         self.assertIn(self.filter_group,
178                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
179
180         with self.assertRaises(OSError):
181             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
182
183         files = {}
184         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
185
186         for k, v in viewitems(files):
187             with open(os.path.join(self.mounttmp, k), 'rb') as f:
188                 self.assertEqual(v, f.read().decode())
189
190
191 class FuseTagsTest(MountTestBase):
192     def runTest(self):
193         self.make_mount(fuse.TagsDirectory)
194
195         d1 = llfuse.listdir(self.mounttmp)
196         d1.sort()
197         self.assertEqual(['foo_tag'], d1)
198
199         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
200         d2.sort()
201         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
202
203         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
204         d3.sort()
205         self.assertEqual(['foo'], d3)
206
207
208 class FuseTagsUpdateTest(MountTestBase):
209     def tag_collection(self, coll_uuid, tag_name):
210         return self.api.links().create(
211             body={'link': {'head_uuid': coll_uuid,
212                            'link_class': 'tag',
213                            'name': tag_name,
214         }}).execute()
215
216     def runTest(self):
217         self.make_mount(fuse.TagsDirectory, poll_time=1)
218
219         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
220
221         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
222         self.tag_collection(bar_uuid, 'fuse_test_tag')
223         for attempt in AssertWithTimeout(10):
224             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
225         self.assertDirContents('fuse_test_tag', [bar_uuid])
226
227         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
228         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
229         for attempt in AssertWithTimeout(10):
230             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
231
232         self.api.links().delete(uuid=l['uuid']).execute()
233         for attempt in AssertWithTimeout(10):
234             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
235
236
237 def fuseSharedTestHelper(mounttmp):
238     class Test(unittest.TestCase):
239         def runTest(self):
240             # Double check that we can open and read objects in this folder as a file,
241             # and that its contents are what we expect.
242             baz_path = os.path.join(
243                 mounttmp,
244                 'FUSE User',
245                 'FUSE Test Project',
246                 'collection in FUSE project',
247                 'baz')
248             with open(baz_path) as f:
249                 self.assertEqual("baz", f.read())
250
251             # check mtime on collection
252             st = os.stat(baz_path)
253             try:
254                 mtime = st.st_mtime_ns // 1000000000
255             except AttributeError:
256                 mtime = st.st_mtime
257             self.assertEqual(mtime, 1391448174)
258
259             # shared_dirs is a list of the directories exposed
260             # by fuse.SharedDirectory (i.e. any object visible
261             # to the current user)
262             shared_dirs = llfuse.listdir(mounttmp)
263             shared_dirs.sort()
264             self.assertIn('FUSE User', shared_dirs)
265
266             # fuse_user_objs is a list of the objects owned by the FUSE
267             # test user (which present as files in the 'FUSE User'
268             # directory)
269             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
270             fuse_user_objs.sort()
271             self.assertEqual(['FUSE Test Project',                    # project owned by user
272                               'collection #1 owned by FUSE',          # collection owned by user
273                               'collection #2 owned by FUSE'          # collection owned by user
274                           ], fuse_user_objs)
275
276             # test_proj_files is a list of the files in the FUSE Test Project.
277             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
278             test_proj_files.sort()
279             self.assertEqual(['collection in FUSE project'
280                           ], test_proj_files)
281
282
283     Test().runTest()
284
285 class FuseSharedTest(MountTestBase):
286     def runTest(self):
287         self.make_mount(fuse.SharedDirectory,
288                         exclude=self.api.users().current().execute()['uuid'])
289         keep = arvados.keep.KeepClient()
290         keep.put("baz".encode())
291
292         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
293
294
295 class FuseHomeTest(MountTestBase):
296     def runTest(self):
297         self.make_mount(fuse.ProjectDirectory,
298                         project_object=self.api.users().current().execute())
299
300         d1 = llfuse.listdir(self.mounttmp)
301         self.assertIn('Unrestricted public data', d1)
302
303         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
304         public_project = run_test_server.fixture('groups')[
305             'anonymously_accessible_project']
306         found_in = 0
307         found_not_in = 0
308         for name, item in viewitems(run_test_server.fixture('collections')):
309             if 'name' not in item:
310                 pass
311             elif item['owner_uuid'] == public_project['uuid']:
312                 self.assertIn(item['name'], d2)
313                 found_in += 1
314             else:
315                 # Artificial assumption here: there is no public
316                 # collection fixture with the same name as a
317                 # non-public collection.
318                 self.assertNotIn(item['name'], d2)
319                 found_not_in += 1
320         self.assertNotEqual(0, found_in)
321         self.assertNotEqual(0, found_not_in)
322
323         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
324         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
325
326
327 def fuseModifyFileTestHelperReadStartContents(mounttmp):
328     class Test(unittest.TestCase):
329         def runTest(self):
330             d1 = llfuse.listdir(mounttmp)
331             self.assertEqual(["file1.txt"], d1)
332             with open(os.path.join(mounttmp, "file1.txt")) as f:
333                 self.assertEqual("blub", f.read())
334     Test().runTest()
335
336 def fuseModifyFileTestHelperReadEndContents(mounttmp):
337     class Test(unittest.TestCase):
338         def runTest(self):
339             d1 = llfuse.listdir(mounttmp)
340             self.assertEqual(["file1.txt"], d1)
341             with open(os.path.join(mounttmp, "file1.txt")) as f:
342                 self.assertEqual("plnp", f.read())
343     Test().runTest()
344
345 class FuseModifyFileTest(MountTestBase):
346     def runTest(self):
347         collection = arvados.collection.Collection(api_client=self.api)
348         with collection.open("file1.txt", "w") as f:
349             f.write("blub")
350
351         collection.save_new()
352
353         m = self.make_mount(fuse.CollectionDirectory)
354         with llfuse.lock:
355             m.new_collection(collection.api_response(), collection)
356
357         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
358
359         with collection.open("file1.txt", "w") as f:
360             f.write("plnp")
361
362         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
363
364
365 class FuseAddFileToCollectionTest(MountTestBase):
366     def runTest(self):
367         collection = arvados.collection.Collection(api_client=self.api)
368         with collection.open("file1.txt", "w") as f:
369             f.write("blub")
370
371         collection.save_new()
372
373         m = self.make_mount(fuse.CollectionDirectory)
374         with llfuse.lock:
375             m.new_collection(collection.api_response(), collection)
376
377         d1 = llfuse.listdir(self.mounttmp)
378         self.assertEqual(["file1.txt"], d1)
379
380         with collection.open("file2.txt", "w") as f:
381             f.write("plnp")
382
383         d1 = llfuse.listdir(self.mounttmp)
384         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
385
386
387 class FuseRemoveFileFromCollectionTest(MountTestBase):
388     def runTest(self):
389         collection = arvados.collection.Collection(api_client=self.api)
390         with collection.open("file1.txt", "w") as f:
391             f.write("blub")
392
393         with collection.open("file2.txt", "w") as f:
394             f.write("plnp")
395
396         collection.save_new()
397
398         m = self.make_mount(fuse.CollectionDirectory)
399         with llfuse.lock:
400             m.new_collection(collection.api_response(), collection)
401
402         d1 = llfuse.listdir(self.mounttmp)
403         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
404
405         collection.remove("file2.txt")
406
407         d1 = llfuse.listdir(self.mounttmp)
408         self.assertEqual(["file1.txt"], d1)
409
410
411 def fuseCreateFileTestHelper(mounttmp):
412     class Test(unittest.TestCase):
413         def runTest(self):
414             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
415                 pass
416     Test().runTest()
417
418 class FuseCreateFileTest(MountTestBase):
419     def runTest(self):
420         collection = arvados.collection.Collection(api_client=self.api)
421         collection.save_new()
422
423         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
424         self.assertEqual(collection2["manifest_text"], "")
425
426         collection.save_new()
427
428         m = self.make_mount(fuse.CollectionDirectory)
429         with llfuse.lock:
430             m.new_collection(collection.api_response(), collection)
431         self.assertTrue(m.writable())
432
433         self.assertNotIn("file1.txt", collection)
434
435         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
436
437         self.assertIn("file1.txt", collection)
438
439         d1 = llfuse.listdir(self.mounttmp)
440         self.assertEqual(["file1.txt"], d1)
441
442         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
443         assertRegex(self, collection2["manifest_text"],
444             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
445
446
447 def fuseWriteFileTestHelperWriteFile(mounttmp):
448     class Test(unittest.TestCase):
449         def runTest(self):
450             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
451                 f.write("Hello world!")
452     Test().runTest()
453
454 def fuseWriteFileTestHelperReadFile(mounttmp):
455     class Test(unittest.TestCase):
456         def runTest(self):
457             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
458                 self.assertEqual(f.read(), "Hello world!")
459     Test().runTest()
460
461 class FuseWriteFileTest(MountTestBase):
462     def runTest(self):
463         collection = arvados.collection.Collection(api_client=self.api)
464         collection.save_new()
465
466         m = self.make_mount(fuse.CollectionDirectory)
467         with llfuse.lock:
468             m.new_collection(collection.api_response(), collection)
469         self.assertTrue(m.writable())
470
471         self.assertNotIn("file1.txt", collection)
472
473         self.assertEqual(0, self.operations.write_counter.get())
474         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
475         self.assertEqual(12, self.operations.write_counter.get())
476
477         with collection.open("file1.txt") as f:
478             self.assertEqual(f.read(), "Hello world!")
479
480         self.assertEqual(0, self.operations.read_counter.get())
481         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
482         self.assertEqual(12, self.operations.read_counter.get())
483
484         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
485         assertRegex(self, collection2["manifest_text"],
486             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
487
488
489 def fuseUpdateFileTestHelper(mounttmp):
490     class Test(unittest.TestCase):
491         def runTest(self):
492             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
493                 f.write("Hello world!")
494
495             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
496                 fr = f.read()
497                 self.assertEqual(fr, "Hello world!")
498                 f.seek(0)
499                 f.write("Hola mundo!")
500                 f.seek(0)
501                 fr = f.read()
502                 self.assertEqual(fr, "Hola mundo!!")
503
504             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
505                 self.assertEqual(f.read(), "Hola mundo!!")
506
507     Test().runTest()
508
509 class FuseUpdateFileTest(MountTestBase):
510     def runTest(self):
511         collection = arvados.collection.Collection(api_client=self.api)
512         collection.save_new()
513
514         m = self.make_mount(fuse.CollectionDirectory)
515         with llfuse.lock:
516             m.new_collection(collection.api_response(), collection)
517         self.assertTrue(m.writable())
518
519         # See note in MountTestBase.setUp
520         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
521
522         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
523         assertRegex(self, collection2["manifest_text"],
524             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
525
526
527 def fuseMkdirTestHelper(mounttmp):
528     class Test(unittest.TestCase):
529         def runTest(self):
530             with self.assertRaises(IOError):
531                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
532                     f.write("Hello world!")
533
534             os.mkdir(os.path.join(mounttmp, "testdir"))
535
536             with self.assertRaises(OSError):
537                 os.mkdir(os.path.join(mounttmp, "testdir"))
538
539             d1 = llfuse.listdir(mounttmp)
540             self.assertEqual(["testdir"], d1)
541
542             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
543                 f.write("Hello world!")
544
545             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
546             self.assertEqual(["file1.txt"], d1)
547
548     Test().runTest()
549
550 class FuseMkdirTest(MountTestBase):
551     def runTest(self):
552         collection = arvados.collection.Collection(api_client=self.api)
553         collection.save_new()
554
555         m = self.make_mount(fuse.CollectionDirectory)
556         with llfuse.lock:
557             m.new_collection(collection.api_response(), collection)
558         self.assertTrue(m.writable())
559
560         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
561
562         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
563         assertRegex(self, collection2["manifest_text"],
564             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
565
566
567 def fuseRmTestHelperWriteFile(mounttmp):
568     class Test(unittest.TestCase):
569         def runTest(self):
570             os.mkdir(os.path.join(mounttmp, "testdir"))
571
572             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
573                 f.write("Hello world!")
574
575     Test().runTest()
576
577 def fuseRmTestHelperDeleteFile(mounttmp):
578     class Test(unittest.TestCase):
579         def runTest(self):
580             # Can't delete because it's not empty
581             with self.assertRaises(OSError):
582                 os.rmdir(os.path.join(mounttmp, "testdir"))
583
584             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
585             self.assertEqual(["file1.txt"], d1)
586
587             # Delete file
588             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
589
590             # Make sure it's empty
591             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
592             self.assertEqual([], d1)
593
594             # Try to delete it again
595             with self.assertRaises(OSError):
596                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
597
598     Test().runTest()
599
600 def fuseRmTestHelperRmdir(mounttmp):
601     class Test(unittest.TestCase):
602         def runTest(self):
603             # Should be able to delete now that it is empty
604             os.rmdir(os.path.join(mounttmp, "testdir"))
605
606             # Make sure it's empty
607             d1 = llfuse.listdir(os.path.join(mounttmp))
608             self.assertEqual([], d1)
609
610             # Try to delete it again
611             with self.assertRaises(OSError):
612                 os.rmdir(os.path.join(mounttmp, "testdir"))
613
614     Test().runTest()
615
616 class FuseRmTest(MountTestBase):
617     def runTest(self):
618         collection = arvados.collection.Collection(api_client=self.api)
619         collection.save_new()
620
621         m = self.make_mount(fuse.CollectionDirectory)
622         with llfuse.lock:
623             m.new_collection(collection.api_response(), collection)
624         self.assertTrue(m.writable())
625
626         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
627
628         # Starting manifest
629         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
630         assertRegex(self, collection2["manifest_text"],
631             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
632         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
633
634         # Empty directories are represented by an empty file named "."
635         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
636         assertRegex(self, collection2["manifest_text"],
637                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
638
639         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
640
641         # manifest should be empty now.
642         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
643         self.assertEqual(collection2["manifest_text"], "")
644
645
646 def fuseMvFileTestHelperWriteFile(mounttmp):
647     class Test(unittest.TestCase):
648         def runTest(self):
649             os.mkdir(os.path.join(mounttmp, "testdir"))
650
651             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
652                 f.write("Hello world!")
653
654     Test().runTest()
655
656 def fuseMvFileTestHelperMoveFile(mounttmp):
657     class Test(unittest.TestCase):
658         def runTest(self):
659             d1 = llfuse.listdir(os.path.join(mounttmp))
660             self.assertEqual(["testdir"], d1)
661             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
662             self.assertEqual(["file1.txt"], d1)
663
664             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
665
666             d1 = llfuse.listdir(os.path.join(mounttmp))
667             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
668             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
669             self.assertEqual([], d1)
670
671     Test().runTest()
672
673 class FuseMvFileTest(MountTestBase):
674     def runTest(self):
675         collection = arvados.collection.Collection(api_client=self.api)
676         collection.save_new()
677
678         m = self.make_mount(fuse.CollectionDirectory)
679         with llfuse.lock:
680             m.new_collection(collection.api_response(), collection)
681         self.assertTrue(m.writable())
682
683         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
684
685         # Starting manifest
686         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
687         assertRegex(self, collection2["manifest_text"],
688             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
689
690         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
691
692         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
693         assertRegex(self, collection2["manifest_text"],
694             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
695
696
697 def fuseRenameTestHelper(mounttmp):
698     class Test(unittest.TestCase):
699         def runTest(self):
700             os.mkdir(os.path.join(mounttmp, "testdir"))
701
702             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
703                 f.write("Hello world!")
704
705     Test().runTest()
706
707 class FuseRenameTest(MountTestBase):
708     def runTest(self):
709         collection = arvados.collection.Collection(api_client=self.api)
710         collection.save_new()
711
712         m = self.make_mount(fuse.CollectionDirectory)
713         with llfuse.lock:
714             m.new_collection(collection.api_response(), collection)
715         self.assertTrue(m.writable())
716
717         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
718
719         # Starting manifest
720         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
721         assertRegex(self, collection2["manifest_text"],
722             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
723
724         d1 = llfuse.listdir(os.path.join(self.mounttmp))
725         self.assertEqual(["testdir"], d1)
726         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
727         self.assertEqual(["file1.txt"], d1)
728
729         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
730
731         d1 = llfuse.listdir(os.path.join(self.mounttmp))
732         self.assertEqual(["testdir2"], sorted(d1))
733         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
734         self.assertEqual(["file1.txt"], d1)
735
736         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
737         assertRegex(self, collection2["manifest_text"],
738             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
739
740
741 class FuseUpdateFromEventTest(MountTestBase):
742     def runTest(self):
743         collection = arvados.collection.Collection(api_client=self.api)
744         collection.save_new()
745
746         m = self.make_mount(fuse.CollectionDirectory)
747         with llfuse.lock:
748             m.new_collection(collection.api_response(), collection)
749
750         self.operations.listen_for_events()
751
752         d1 = llfuse.listdir(os.path.join(self.mounttmp))
753         self.assertEqual([], sorted(d1))
754
755         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
756             with collection2.open("file1.txt", "w") as f:
757                 f.write("foo")
758
759         for attempt in AssertWithTimeout(10):
760             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
761
762
763 class FuseDeleteProjectEventTest(MountTestBase):
764     def runTest(self):
765
766         aproject = self.api.groups().create(body={
767             "name": "aproject",
768             "group_class": "project"
769         }).execute()
770
771         bproject = self.api.groups().create(body={
772             "name": "bproject",
773             "group_class": "project",
774             "owner_uuid": aproject["uuid"]
775         }).execute()
776
777         self.make_mount(fuse.ProjectDirectory,
778                         project_object=self.api.users().current().execute())
779
780         self.operations.listen_for_events()
781
782         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
783         self.assertEqual(["bproject"], sorted(d1))
784
785         self.api.groups().delete(uuid=bproject["uuid"]).execute()
786
787         for attempt in AssertWithTimeout(10):
788             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
789
790
791 def fuseFileConflictTestHelper(mounttmp):
792     class Test(unittest.TestCase):
793         def runTest(self):
794             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
795                 f.write("bar")
796
797             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
798             self.assertEqual(len(d1), 2)
799
800             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
801                 self.assertEqual(f.read(), "bar")
802
803             assertRegex(self, d1[1],
804                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
805
806             with open(os.path.join(mounttmp, d1[1]), "r") as f:
807                 self.assertEqual(f.read(), "foo")
808
809     Test().runTest()
810
811 class FuseFileConflictTest(MountTestBase):
812     def runTest(self):
813         collection = arvados.collection.Collection(api_client=self.api)
814         collection.save_new()
815
816         m = self.make_mount(fuse.CollectionDirectory)
817         with llfuse.lock:
818             m.new_collection(collection.api_response(), collection)
819
820         d1 = llfuse.listdir(os.path.join(self.mounttmp))
821         self.assertEqual([], sorted(d1))
822
823         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
824             with collection2.open("file1.txt", "w") as f:
825                 f.write("foo")
826
827         # See note in MountTestBase.setUp
828         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
829
830
831 def fuseUnlinkOpenFileTest(mounttmp):
832     class Test(unittest.TestCase):
833         def runTest(self):
834             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
835                 f.write("foo")
836
837                 d1 = llfuse.listdir(os.path.join(mounttmp))
838                 self.assertEqual(["file1.txt"], sorted(d1))
839
840                 os.remove(os.path.join(mounttmp, "file1.txt"))
841
842                 d1 = llfuse.listdir(os.path.join(mounttmp))
843                 self.assertEqual([], sorted(d1))
844
845                 f.seek(0)
846                 self.assertEqual(f.read(), "foo")
847                 f.write("bar")
848
849                 f.seek(0)
850                 self.assertEqual(f.read(), "foobar")
851
852     Test().runTest()
853
854 class FuseUnlinkOpenFileTest(MountTestBase):
855     def runTest(self):
856         collection = arvados.collection.Collection(api_client=self.api)
857         collection.save_new()
858
859         m = self.make_mount(fuse.CollectionDirectory)
860         with llfuse.lock:
861             m.new_collection(collection.api_response(), collection)
862
863         # See note in MountTestBase.setUp
864         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
865
866         self.assertEqual(collection.manifest_text(), "")
867
868
869 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
870     class Test(unittest.TestCase):
871         def runTest(self):
872             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
873                 f.write("Hello world!")
874
875             d1 = os.listdir(os.path.join(mounttmp, uuid1))
876             self.assertEqual(["file1.txt"], sorted(d1))
877             d1 = os.listdir(os.path.join(mounttmp, uuid2))
878             self.assertEqual([], sorted(d1))
879
880     Test().runTest()
881
882 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
883     class Test(unittest.TestCase):
884         def runTest(self):
885             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
886
887             d1 = os.listdir(os.path.join(mounttmp, uuid1))
888             self.assertEqual([], sorted(d1))
889             d1 = os.listdir(os.path.join(mounttmp, uuid2))
890             self.assertEqual(["file2.txt"], sorted(d1))
891
892     Test().runTest()
893
894 class FuseMvFileBetweenCollectionsTest(MountTestBase):
895     def runTest(self):
896         collection1 = arvados.collection.Collection(api_client=self.api)
897         collection1.save_new()
898
899         collection2 = arvados.collection.Collection(api_client=self.api)
900         collection2.save_new()
901
902         m = self.make_mount(fuse.MagicDirectory)
903
904         # See note in MountTestBase.setUp
905         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
906                                                   collection1.manifest_locator(),
907                                                   collection2.manifest_locator()))
908
909         collection1.update()
910         collection2.update()
911
912         assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
913         self.assertEqual(collection2.manifest_text(), "")
914
915         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
916                                                   collection1.manifest_locator(),
917                                                   collection2.manifest_locator()))
918
919         collection1.update()
920         collection2.update()
921
922         self.assertEqual(collection1.manifest_text(), "")
923         assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
924
925         collection1.stop_threads()
926         collection2.stop_threads()
927
928
929 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
930     class Test(unittest.TestCase):
931         def runTest(self):
932             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
933             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
934                 f.write("Hello world!")
935
936             d1 = os.listdir(os.path.join(mounttmp, uuid1))
937             self.assertEqual(["testdir"], sorted(d1))
938             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
939             self.assertEqual(["file1.txt"], sorted(d1))
940
941             d1 = os.listdir(os.path.join(mounttmp, uuid2))
942             self.assertEqual([], sorted(d1))
943
944     Test().runTest()
945
946
947 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
948     class Test(unittest.TestCase):
949         def runTest(self):
950             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
951
952             d1 = os.listdir(os.path.join(mounttmp, uuid1))
953             self.assertEqual([], sorted(d1))
954
955             d1 = os.listdir(os.path.join(mounttmp, uuid2))
956             self.assertEqual(["testdir2"], sorted(d1))
957             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
958             self.assertEqual(["file1.txt"], sorted(d1))
959
960             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
961                 self.assertEqual(f.read(), "Hello world!")
962
963     Test().runTest()
964
965 class FuseMvDirBetweenCollectionsTest(MountTestBase):
966     def runTest(self):
967         collection1 = arvados.collection.Collection(api_client=self.api)
968         collection1.save_new()
969
970         collection2 = arvados.collection.Collection(api_client=self.api)
971         collection2.save_new()
972
973         m = self.make_mount(fuse.MagicDirectory)
974
975         # See note in MountTestBase.setUp
976         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
977                                                   collection1.manifest_locator(),
978                                                   collection2.manifest_locator()))
979
980         collection1.update()
981         collection2.update()
982
983         assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
984         self.assertEqual(collection2.manifest_text(), "")
985
986         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
987                                                   collection1.manifest_locator(),
988                                                   collection2.manifest_locator()))
989
990         collection1.update()
991         collection2.update()
992
993         self.assertEqual(collection1.manifest_text(), "")
994         assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
995
996         collection1.stop_threads()
997         collection2.stop_threads()
998
999 def fuseProjectMkdirTestHelper1(mounttmp):
1000     class Test(unittest.TestCase):
1001         def runTest(self):
1002             os.mkdir(os.path.join(mounttmp, "testcollection"))
1003             with self.assertRaises(OSError):
1004                 os.mkdir(os.path.join(mounttmp, "testcollection"))
1005     Test().runTest()
1006
1007 def fuseProjectMkdirTestHelper2(mounttmp):
1008     class Test(unittest.TestCase):
1009         def runTest(self):
1010             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1011                 f.write("Hello world!")
1012             with self.assertRaises(OSError):
1013                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1014             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1015             with self.assertRaises(OSError):
1016                 os.remove(os.path.join(mounttmp, "testcollection"))
1017             os.rmdir(os.path.join(mounttmp, "testcollection"))
1018     Test().runTest()
1019
1020 class FuseProjectMkdirRmdirTest(MountTestBase):
1021     def runTest(self):
1022         self.make_mount(fuse.ProjectDirectory,
1023                         project_object=self.api.users().current().execute())
1024
1025         d1 = llfuse.listdir(self.mounttmp)
1026         self.assertNotIn('testcollection', d1)
1027
1028         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1029
1030         d1 = llfuse.listdir(self.mounttmp)
1031         self.assertIn('testcollection', d1)
1032
1033         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1034
1035         d1 = llfuse.listdir(self.mounttmp)
1036         self.assertNotIn('testcollection', d1)
1037
1038
1039 def fuseProjectMvTestHelper1(mounttmp):
1040     class Test(unittest.TestCase):
1041         def runTest(self):
1042             d1 = llfuse.listdir(mounttmp)
1043             self.assertNotIn('testcollection', d1)
1044
1045             os.mkdir(os.path.join(mounttmp, "testcollection"))
1046
1047             d1 = llfuse.listdir(mounttmp)
1048             self.assertIn('testcollection', d1)
1049
1050             with self.assertRaises(OSError):
1051                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1052
1053             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1054
1055             d1 = llfuse.listdir(mounttmp)
1056             self.assertNotIn('testcollection', d1)
1057
1058             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1059             self.assertIn('testcollection', d1)
1060
1061     Test().runTest()
1062
1063 class FuseProjectMvTest(MountTestBase):
1064     def runTest(self):
1065         self.make_mount(fuse.ProjectDirectory,
1066                         project_object=self.api.users().current().execute())
1067
1068         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1069
1070
1071 def fuseFsyncTestHelper(mounttmp, k):
1072     class Test(unittest.TestCase):
1073         def runTest(self):
1074             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1075             os.fsync(fd)
1076             os.close(fd)
1077
1078     Test().runTest()
1079
1080 class FuseFsyncTest(FuseMagicTest):
1081     def runTest(self):
1082         self.make_mount(fuse.MagicDirectory)
1083         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1084
1085
1086 class MagicDirApiError(FuseMagicTest):
1087     def setUp(self):
1088         api = mock.MagicMock()
1089         super(MagicDirApiError, self).setUp(api=api)
1090         api.collections().get().execute.side_effect = iter([
1091             Exception('API fail'),
1092             {
1093                 "manifest_text": self.test_manifest,
1094                 "portable_data_hash": self.test_manifest_pdh,
1095             },
1096         ])
1097         api.keep.get.side_effect = Exception('Keep fail')
1098
1099     def runTest(self):
1100         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1101             self.make_mount(fuse.MagicDirectory)
1102
1103             self.operations.inodes.inode_cache.cap = 1
1104             self.operations.inodes.inode_cache.min_entries = 2
1105
1106             with self.assertRaises(OSError):
1107                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1108
1109             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1110
1111
1112 class SanitizeFilenameTest(MountTestBase):
1113     def test_sanitize_filename(self):
1114         pdir = fuse.ProjectDirectory(1, {}, self.api, 0, project_object=self.api.users().current().execute())
1115         acceptable = [
1116             "foo.txt",
1117             ".foo",
1118             "..foo",
1119             "...",
1120             "foo...",
1121             "foo..",
1122             "foo.",
1123             "-",
1124             "\x01\x02\x03",
1125             ]
1126         unacceptable = [
1127             "f\00",
1128             "\00\00",
1129             "/foo",
1130             "foo/",
1131             "//",
1132             ]
1133         for f in acceptable:
1134             self.assertEqual(f, pdir.sanitize_filename(f))
1135         for f in unacceptable:
1136             self.assertNotEqual(f, pdir.sanitize_filename(f))
1137             # The sanitized filename should be the same length, though.
1138             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1139         # Special cases
1140         self.assertEqual("_", pdir.sanitize_filename(""))
1141         self.assertEqual("_", pdir.sanitize_filename("."))
1142         self.assertEqual("__", pdir.sanitize_filename(".."))
1143
1144
1145 class FuseMagicTestPDHOnly(MountTestBase):
1146     def setUp(self, api=None):
1147         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1148
1149         cw = arvados.CollectionWriter()
1150
1151         cw.start_new_file('thing1.txt')
1152         cw.write("data 1")
1153
1154         self.testcollection = cw.finish()
1155         self.test_manifest = cw.manifest_text()
1156         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1157         self.testcollectionuuid = str(created['uuid'])
1158
1159     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1160         if skip_pdh_only is True:
1161             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1162         else:
1163             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1164
1165         mount_ls = llfuse.listdir(self.mounttmp)
1166         self.assertIn('README', mount_ls)
1167         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1168                              arvados.util.uuid_pattern.match(fn)
1169                              for fn in mount_ls),
1170                          "new FUSE MagicDirectory lists Collection")
1171
1172         # look up using pdh should succeed in all cases
1173         self.assertDirContents(self.testcollection, ['thing1.txt'])
1174         self.assertDirContents(os.path.join('by_id', self.testcollection),
1175                                ['thing1.txt'])
1176         mount_ls = llfuse.listdir(self.mounttmp)
1177         self.assertIn('README', mount_ls)
1178         self.assertIn(self.testcollection, mount_ls)
1179         self.assertIn(self.testcollection,
1180                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1181
1182         files = {}
1183         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1184
1185         for k, v in viewitems(files):
1186             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1187                 self.assertEqual(v, f.read().decode())
1188
1189         # look up using uuid should fail when pdh_only is set
1190         if pdh_only is True:
1191             with self.assertRaises(OSError):
1192                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1193                                ['thing1.txt'])
1194         else:
1195             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1196                                ['thing1.txt'])
1197
1198     def test_with_pdh_only_true(self):
1199         self.verify_pdh_only(pdh_only=True)
1200
1201     def test_with_pdh_only_false(self):
1202         self.verify_pdh_only(pdh_only=False)
1203
1204     def test_with_default_by_id(self):
1205         self.verify_pdh_only(skip_pdh_only=True)
1206
1207
1208 class SlashSubstitutionTest(IntegrationTest):
1209     mnt_args = [
1210         '--read-write',
1211         '--mount-home', 'zzz',
1212     ]
1213
1214     def setUp(self):
1215         super(SlashSubstitutionTest, self).setUp()
1216         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1217         self.api.config = lambda: {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1218         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1219         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1220         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1221
1222     @IntegrationTest.mount(argv=mnt_args)
1223     @mock.patch('arvados.util.get_config_once')
1224     def test_slash_substitution_before_listing(self, get_config_once):
1225         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1226         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1227         self.checkContents()
1228     @staticmethod
1229     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1230         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1231             f.write('xxx')
1232         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1233             f.write('foo')
1234
1235     @IntegrationTest.mount(argv=mnt_args)
1236     @mock.patch('arvados.util.get_config_once')
1237     def test_slash_substitution_after_listing(self, get_config_once):
1238         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1239         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1240         self.checkContents()
1241     @staticmethod
1242     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1243         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1244             f.write('xxx')
1245         os.listdir(tmpdir)
1246         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1247             f.write('foo')
1248
1249     def checkContents(self):
1250         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1251         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1252
1253     @IntegrationTest.mount(argv=mnt_args)
1254     @mock.patch('arvados.util.get_config_once')
1255     def test_slash_substitution_conflict(self, get_config_once):
1256         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1257         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1258         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1259         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1260         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1261         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1262     @staticmethod
1263     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1264         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1265             f.write('foo')
1266
1267 class StorageClassesTest(IntegrationTest):
1268     mnt_args = [
1269         '--read-write',
1270         '--mount-home', 'homedir',
1271     ]
1272
1273     def setUp(self):
1274         super(StorageClassesTest, self).setUp()
1275         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1276
1277     @IntegrationTest.mount(argv=mnt_args)
1278     def test_collection_default_storage_classes(self):
1279         coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
1280         self.api.collections().create(body={'name':'a_collection'}).execute()
1281         self.pool_test(coll_path)
1282     @staticmethod
1283     def _test_collection_default_storage_classes(self, coll):
1284         self.assertEqual(storage_classes_desired(coll), ['default'])
1285
1286     @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
1287     def test_collection_custom_storage_classes(self):
1288         coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
1289         os.mkdir(coll_path)
1290         self.pool_test(coll_path)
1291     @staticmethod
1292     def _test_collection_custom_storage_classes(self, coll):
1293         self.assertEqual(storage_classes_desired(coll), ['foo'])