services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 from __future__ import absolute_import
   6 from future.utils import viewitems
   7 from builtins import str
   8 from builtins import object
   9 from six import assertRegex
  10 import json
  11 import llfuse
  12 import logging
  13 import mock
  14 import os
  15 import subprocess
  16 import time
  17 import unittest
  18
  19 import arvados
  20 import arvados_fuse as fuse
  21 from . import run_test_server
  22
  23 from .integration_test import IntegrationTest
  24 from .mount_test_base import MountTestBase
  25
  26 logger = logging.getLogger('arvados.arv-mount')
  27
  28
  29 class AssertWithTimeout(object):
  30     """Allow some time for an assertion to pass."""
  31
  32     def __init__(self, timeout=0):
  33         self.timeout = timeout
  34
  35     def __iter__(self):
  36         self.deadline = time.time() + self.timeout
  37         self.done = False
  38         return self
  39
  40     def __next__(self):
  41         if self.done:
  42             raise StopIteration
  43         return self.attempt
  44
  45     def attempt(self, fn, *args, **kwargs):
  46         try:
  47             fn(*args, **kwargs)
  48         except AssertionError:
  49             if time.time() > self.deadline:
  50                 raise
  51             time.sleep(0.1)
  52         else:
  53             self.done = True
  54
  55
  56 class FuseMountTest(MountTestBase):
  57     def setUp(self):
  58         super(FuseMountTest, self).setUp()
  59
  60         cw = arvados.CollectionWriter()
  61
  62         cw.start_new_file('thing1.txt')
  63         cw.write("data 1")
  64         cw.start_new_file('thing2.txt')
  65         cw.write("data 2")
  66
  67         cw.start_new_stream('dir1')
  68         cw.start_new_file('thing3.txt')
  69         cw.write("data 3")
  70         cw.start_new_file('thing4.txt')
  71         cw.write("data 4")
  72
  73         cw.start_new_stream('dir2')
  74         cw.start_new_file('thing5.txt')
  75         cw.write("data 5")
  76         cw.start_new_file('thing6.txt')
  77         cw.write("data 6")
  78
  79         cw.start_new_stream('dir2/dir3')
  80         cw.start_new_file('thing7.txt')
  81         cw.write("data 7")
  82
  83         cw.start_new_file('thing8.txt')
  84         cw.write("data 8")
  85
  86         cw.start_new_stream('edgecases')
  87         for f in ":/.../-/*/ ".split("/"):
  88             cw.start_new_file(f)
  89             cw.write('x')
  90
  91         for f in ":/.../-/*/ ".split("/"):
  92             cw.start_new_stream('edgecases/dirs/' + f)
  93             cw.start_new_file('x/x')
  94             cw.write('x')
  95
  96         self.testcollection = cw.finish()
  97         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  98
  99     def runTest(self):
 100         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
 101
 102         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 103                                       'edgecases', 'dir1', 'dir2'])
 104         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 105         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 106         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 107         self.assertDirContents('edgecases',
 108                                "dirs/:/.../-/*/ ".split("/"))
 109         self.assertDirContents('edgecases/dirs',
 110                                ":/.../-/*/ ".split("/"))
 111
 112         files = {'thing1.txt': 'data 1',
 113                  'thing2.txt': 'data 2',
 114                  'dir1/thing3.txt': 'data 3',
 115                  'dir1/thing4.txt': 'data 4',
 116                  'dir2/thing5.txt': 'data 5',
 117                  'dir2/thing6.txt': 'data 6',
 118                  'dir2/dir3/thing7.txt': 'data 7',
 119                  'dir2/dir3/thing8.txt': 'data 8'}
 120
 121         for k, v in viewitems(files):
 122             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 123                 self.assertEqual(v, f.read().decode())
 124
 125
 126 class FuseMagicTest(MountTestBase):
 127     def setUp(self, api=None):
 128         super(FuseMagicTest, self).setUp(api=api)
 129
 130         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 131         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
 132         self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
 133         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 134         self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
 135
 136         cw = arvados.CollectionWriter()
 137
 138         cw.start_new_file('thing1.txt')
 139         cw.write("data 1")
 140
 141         self.testcollection = cw.finish()
 142         self.test_manifest = cw.manifest_text()
 143         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 144         self.test_manifest_pdh = coll['portable_data_hash']
 145
 146     def runTest(self):
 147         self.make_mount(fuse.MagicDirectory)
 148
 149         mount_ls = llfuse.listdir(self.mounttmp)
 150         self.assertIn('README', mount_ls)
 151         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 152                              arvados.util.uuid_pattern.match(fn)
 153                              for fn in mount_ls),
 154                          "new FUSE MagicDirectory has no collections or projects")
 155         self.assertDirContents(self.testcollection, ['thing1.txt'])
 156         self.assertDirContents(os.path.join('by_id', self.testcollection),
 157                                ['thing1.txt'])
 158         self.assertIn(self.collection_in_test_project,
 159                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 160         self.assertIn(self.collection_in_test_project,
 161                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 162         self.assertIn(self.collection_in_filter_group,
 163                       llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
 164         self.assertIn(self.collection_in_filter_group,
 165                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
 166
 167
 168         mount_ls = llfuse.listdir(self.mounttmp)
 169         self.assertIn('README', mount_ls)
 170         self.assertIn(self.testcollection, mount_ls)
 171         self.assertIn(self.testcollection,
 172                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 173         self.assertIn(self.test_project, mount_ls)
 174         self.assertIn(self.test_project,
 175                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 176         self.assertIn(self.filter_group,
 177                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 178
 179         with self.assertRaises(OSError):
 180             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 181
 182         files = {}
 183         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 184
 185         for k, v in viewitems(files):
 186             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 187                 self.assertEqual(v, f.read().decode())
 188
 189
 190 class FuseTagsTest(MountTestBase):
 191     def runTest(self):
 192         self.make_mount(fuse.TagsDirectory)
 193
 194         d1 = llfuse.listdir(self.mounttmp)
 195         d1.sort()
 196         self.assertEqual(['foo_tag'], d1)
 197
 198         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 199         d2.sort()
 200         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 201
 202         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 203         d3.sort()
 204         self.assertEqual(['foo'], d3)
 205
 206
 207 class FuseTagsUpdateTest(MountTestBase):
 208     def tag_collection(self, coll_uuid, tag_name):
 209         return self.api.links().create(
 210             body={'link': {'head_uuid': coll_uuid,
 211                            'link_class': 'tag',
 212                            'name': tag_name,
 213         }}).execute()
 214
 215     def runTest(self):
 216         self.make_mount(fuse.TagsDirectory, poll_time=1)
 217
 218         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 219
 220         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 221         self.tag_collection(bar_uuid, 'fuse_test_tag')
 222         for attempt in AssertWithTimeout(10):
 223             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 224         self.assertDirContents('fuse_test_tag', [bar_uuid])
 225
 226         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 227         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 228         for attempt in AssertWithTimeout(10):
 229             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 230
 231         self.api.links().delete(uuid=l['uuid']).execute()
 232         for attempt in AssertWithTimeout(10):
 233             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 234
 235
 236 def fuseSharedTestHelper(mounttmp):
 237     class Test(unittest.TestCase):
 238         def runTest(self):
 239             # Double check that we can open and read objects in this folder as a file,
 240             # and that its contents are what we expect.
 241             baz_path = os.path.join(
 242                 mounttmp,
 243                 'FUSE User',
 244                 'FUSE Test Project',
 245                 'collection in FUSE project',
 246                 'baz')
 247             with open(baz_path) as f:
 248                 self.assertEqual("baz", f.read())
 249
 250             # check mtime on collection
 251             st = os.stat(baz_path)
 252             try:
 253                 mtime = st.st_mtime_ns // 1000000000
 254             except AttributeError:
 255                 mtime = st.st_mtime
 256             self.assertEqual(mtime, 1391448174)
 257
 258             # shared_dirs is a list of the directories exposed
 259             # by fuse.SharedDirectory (i.e. any object visible
 260             # to the current user)
 261             shared_dirs = llfuse.listdir(mounttmp)
 262             shared_dirs.sort()
 263             self.assertIn('FUSE User', shared_dirs)
 264
 265             # fuse_user_objs is a list of the objects owned by the FUSE
 266             # test user (which present as files in the 'FUSE User'
 267             # directory)
 268             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 269             fuse_user_objs.sort()
 270             self.assertEqual(['FUSE Test Project',                    # project owned by user
 271                               'collection #1 owned by FUSE',          # collection owned by user
 272                               'collection #2 owned by FUSE'          # collection owned by user
 273                           ], fuse_user_objs)
 274
 275             # test_proj_files is a list of the files in the FUSE Test Project.
 276             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 277             test_proj_files.sort()
 278             self.assertEqual(['collection in FUSE project'
 279                           ], test_proj_files)
 280
 281
 282     Test().runTest()
 283
 284 class FuseSharedTest(MountTestBase):
 285     def runTest(self):
 286         self.make_mount(fuse.SharedDirectory,
 287                         exclude=self.api.users().current().execute()['uuid'])
 288         keep = arvados.keep.KeepClient()
 289         keep.put("baz".encode())
 290
 291         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 292
 293
 294 class FuseHomeTest(MountTestBase):
 295     def runTest(self):
 296         self.make_mount(fuse.ProjectDirectory,
 297                         project_object=self.api.users().current().execute())
 298
 299         d1 = llfuse.listdir(self.mounttmp)
 300         self.assertIn('Unrestricted public data', d1)
 301
 302         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 303         public_project = run_test_server.fixture('groups')[
 304             'anonymously_accessible_project']
 305         found_in = 0
 306         found_not_in = 0
 307         for name, item in viewitems(run_test_server.fixture('collections')):
 308             if 'name' not in item:
 309                 pass
 310             elif item['owner_uuid'] == public_project['uuid']:
 311                 self.assertIn(item['name'], d2)
 312                 found_in += 1
 313             else:
 314                 # Artificial assumption here: there is no public
 315                 # collection fixture with the same name as a
 316                 # non-public collection.
 317                 self.assertNotIn(item['name'], d2)
 318                 found_not_in += 1
 319         self.assertNotEqual(0, found_in)
 320         self.assertNotEqual(0, found_not_in)
 321
 322         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 323         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 324
 325
 326 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 327     class Test(unittest.TestCase):
 328         def runTest(self):
 329             d1 = llfuse.listdir(mounttmp)
 330             self.assertEqual(["file1.txt"], d1)
 331             with open(os.path.join(mounttmp, "file1.txt")) as f:
 332                 self.assertEqual("blub", f.read())
 333     Test().runTest()
 334
 335 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 336     class Test(unittest.TestCase):
 337         def runTest(self):
 338             d1 = llfuse.listdir(mounttmp)
 339             self.assertEqual(["file1.txt"], d1)
 340             with open(os.path.join(mounttmp, "file1.txt")) as f:
 341                 self.assertEqual("plnp", f.read())
 342     Test().runTest()
 343
 344 class FuseModifyFileTest(MountTestBase):
 345     def runTest(self):
 346         collection = arvados.collection.Collection(api_client=self.api)
 347         with collection.open("file1.txt", "w") as f:
 348             f.write("blub")
 349
 350         collection.save_new()
 351
 352         m = self.make_mount(fuse.CollectionDirectory)
 353         with llfuse.lock:
 354             m.new_collection(collection.api_response(), collection)
 355
 356         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 357
 358         with collection.open("file1.txt", "w") as f:
 359             f.write("plnp")
 360
 361         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 362
 363
 364 class FuseAddFileToCollectionTest(MountTestBase):
 365     def runTest(self):
 366         collection = arvados.collection.Collection(api_client=self.api)
 367         with collection.open("file1.txt", "w") as f:
 368             f.write("blub")
 369
 370         collection.save_new()
 371
 372         m = self.make_mount(fuse.CollectionDirectory)
 373         with llfuse.lock:
 374             m.new_collection(collection.api_response(), collection)
 375
 376         d1 = llfuse.listdir(self.mounttmp)
 377         self.assertEqual(["file1.txt"], d1)
 378
 379         with collection.open("file2.txt", "w") as f:
 380             f.write("plnp")
 381
 382         d1 = llfuse.listdir(self.mounttmp)
 383         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 384
 385
 386 class FuseRemoveFileFromCollectionTest(MountTestBase):
 387     def runTest(self):
 388         collection = arvados.collection.Collection(api_client=self.api)
 389         with collection.open("file1.txt", "w") as f:
 390             f.write("blub")
 391
 392         with collection.open("file2.txt", "w") as f:
 393             f.write("plnp")
 394
 395         collection.save_new()
 396
 397         m = self.make_mount(fuse.CollectionDirectory)
 398         with llfuse.lock:
 399             m.new_collection(collection.api_response(), collection)
 400
 401         d1 = llfuse.listdir(self.mounttmp)
 402         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 403
 404         collection.remove("file2.txt")
 405
 406         d1 = llfuse.listdir(self.mounttmp)
 407         self.assertEqual(["file1.txt"], d1)
 408
 409
 410 def fuseCreateFileTestHelper(mounttmp):
 411     class Test(unittest.TestCase):
 412         def runTest(self):
 413             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 414                 pass
 415     Test().runTest()
 416
 417 class FuseCreateFileTest(MountTestBase):
 418     def runTest(self):
 419         collection = arvados.collection.Collection(api_client=self.api)
 420         collection.save_new()
 421
 422         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 423         self.assertEqual(collection2["manifest_text"], "")
 424
 425         collection.save_new()
 426
 427         m = self.make_mount(fuse.CollectionDirectory)
 428         with llfuse.lock:
 429             m.new_collection(collection.api_response(), collection)
 430         self.assertTrue(m.writable())
 431
 432         self.assertNotIn("file1.txt", collection)
 433
 434         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 435
 436         self.assertIn("file1.txt", collection)
 437
 438         d1 = llfuse.listdir(self.mounttmp)
 439         self.assertEqual(["file1.txt"], d1)
 440
 441         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 442         assertRegex(self, collection2["manifest_text"],
 443             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 444
 445
 446 def fuseWriteFileTestHelperWriteFile(mounttmp):
 447     class Test(unittest.TestCase):
 448         def runTest(self):
 449             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 450                 f.write("Hello world!")
 451     Test().runTest()
 452
 453 def fuseWriteFileTestHelperReadFile(mounttmp):
 454     class Test(unittest.TestCase):
 455         def runTest(self):
 456             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 457                 self.assertEqual(f.read(), "Hello world!")
 458     Test().runTest()
 459
 460 class FuseWriteFileTest(MountTestBase):
 461     def runTest(self):
 462         collection = arvados.collection.Collection(api_client=self.api)
 463         collection.save_new()
 464
 465         m = self.make_mount(fuse.CollectionDirectory)
 466         with llfuse.lock:
 467             m.new_collection(collection.api_response(), collection)
 468         self.assertTrue(m.writable())
 469
 470         self.assertNotIn("file1.txt", collection)
 471
 472         self.assertEqual(0, self.operations.write_counter.get())
 473         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 474         self.assertEqual(12, self.operations.write_counter.get())
 475
 476         with collection.open("file1.txt") as f:
 477             self.assertEqual(f.read(), "Hello world!")
 478
 479         self.assertEqual(0, self.operations.read_counter.get())
 480         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 481         self.assertEqual(12, self.operations.read_counter.get())
 482
 483         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 484         assertRegex(self, collection2["manifest_text"],
 485             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 486
 487
 488 def fuseUpdateFileTestHelper(mounttmp):
 489     class Test(unittest.TestCase):
 490         def runTest(self):
 491             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 492                 f.write("Hello world!")
 493
 494             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 495                 fr = f.read()
 496                 self.assertEqual(fr, "Hello world!")
 497                 f.seek(0)
 498                 f.write("Hola mundo!")
 499                 f.seek(0)
 500                 fr = f.read()
 501                 self.assertEqual(fr, "Hola mundo!!")
 502
 503             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 504                 self.assertEqual(f.read(), "Hola mundo!!")
 505
 506     Test().runTest()
 507
 508 class FuseUpdateFileTest(MountTestBase):
 509     def runTest(self):
 510         collection = arvados.collection.Collection(api_client=self.api)
 511         collection.save_new()
 512
 513         m = self.make_mount(fuse.CollectionDirectory)
 514         with llfuse.lock:
 515             m.new_collection(collection.api_response(), collection)
 516         self.assertTrue(m.writable())
 517
 518         # See note in MountTestBase.setUp
 519         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 520
 521         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 522         assertRegex(self, collection2["manifest_text"],
 523             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 524
 525
 526 def fuseMkdirTestHelper(mounttmp):
 527     class Test(unittest.TestCase):
 528         def runTest(self):
 529             with self.assertRaises(IOError):
 530                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 531                     f.write("Hello world!")
 532
 533             os.mkdir(os.path.join(mounttmp, "testdir"))
 534
 535             with self.assertRaises(OSError):
 536                 os.mkdir(os.path.join(mounttmp, "testdir"))
 537
 538             d1 = llfuse.listdir(mounttmp)
 539             self.assertEqual(["testdir"], d1)
 540
 541             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 542                 f.write("Hello world!")
 543
 544             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 545             self.assertEqual(["file1.txt"], d1)
 546
 547     Test().runTest()
 548
 549 class FuseMkdirTest(MountTestBase):
 550     def runTest(self):
 551         collection = arvados.collection.Collection(api_client=self.api)
 552         collection.save_new()
 553
 554         m = self.make_mount(fuse.CollectionDirectory)
 555         with llfuse.lock:
 556             m.new_collection(collection.api_response(), collection)
 557         self.assertTrue(m.writable())
 558
 559         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 560
 561         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 562         assertRegex(self, collection2["manifest_text"],
 563             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 564
 565
 566 def fuseRmTestHelperWriteFile(mounttmp):
 567     class Test(unittest.TestCase):
 568         def runTest(self):
 569             os.mkdir(os.path.join(mounttmp, "testdir"))
 570
 571             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 572                 f.write("Hello world!")
 573
 574     Test().runTest()
 575
 576 def fuseRmTestHelperDeleteFile(mounttmp):
 577     class Test(unittest.TestCase):
 578         def runTest(self):
 579             # Can't delete because it's not empty
 580             with self.assertRaises(OSError):
 581                 os.rmdir(os.path.join(mounttmp, "testdir"))
 582
 583             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 584             self.assertEqual(["file1.txt"], d1)
 585
 586             # Delete file
 587             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 588
 589             # Make sure it's empty
 590             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 591             self.assertEqual([], d1)
 592
 593             # Try to delete it again
 594             with self.assertRaises(OSError):
 595                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 596
 597     Test().runTest()
 598
 599 def fuseRmTestHelperRmdir(mounttmp):
 600     class Test(unittest.TestCase):
 601         def runTest(self):
 602             # Should be able to delete now that it is empty
 603             os.rmdir(os.path.join(mounttmp, "testdir"))
 604
 605             # Make sure it's empty
 606             d1 = llfuse.listdir(os.path.join(mounttmp))
 607             self.assertEqual([], d1)
 608
 609             # Try to delete it again
 610             with self.assertRaises(OSError):
 611                 os.rmdir(os.path.join(mounttmp, "testdir"))
 612
 613     Test().runTest()
 614
 615 class FuseRmTest(MountTestBase):
 616     def runTest(self):
 617         collection = arvados.collection.Collection(api_client=self.api)
 618         collection.save_new()
 619
 620         m = self.make_mount(fuse.CollectionDirectory)
 621         with llfuse.lock:
 622             m.new_collection(collection.api_response(), collection)
 623         self.assertTrue(m.writable())
 624
 625         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 626
 627         # Starting manifest
 628         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 629         assertRegex(self, collection2["manifest_text"],
 630             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 631         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 632
 633         # Empty directories are represented by an empty file named "."
 634         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 635         assertRegex(self, collection2["manifest_text"],
 636                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 637
 638         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 639
 640         # manifest should be empty now.
 641         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 642         self.assertEqual(collection2["manifest_text"], "")
 643
 644
 645 def fuseMvFileTestHelperWriteFile(mounttmp):
 646     class Test(unittest.TestCase):
 647         def runTest(self):
 648             os.mkdir(os.path.join(mounttmp, "testdir"))
 649
 650             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 651                 f.write("Hello world!")
 652
 653     Test().runTest()
 654
 655 def fuseMvFileTestHelperMoveFile(mounttmp):
 656     class Test(unittest.TestCase):
 657         def runTest(self):
 658             d1 = llfuse.listdir(os.path.join(mounttmp))
 659             self.assertEqual(["testdir"], d1)
 660             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 661             self.assertEqual(["file1.txt"], d1)
 662
 663             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 664
 665             d1 = llfuse.listdir(os.path.join(mounttmp))
 666             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 667             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 668             self.assertEqual([], d1)
 669
 670     Test().runTest()
 671
 672 class FuseMvFileTest(MountTestBase):
 673     def runTest(self):
 674         collection = arvados.collection.Collection(api_client=self.api)
 675         collection.save_new()
 676
 677         m = self.make_mount(fuse.CollectionDirectory)
 678         with llfuse.lock:
 679             m.new_collection(collection.api_response(), collection)
 680         self.assertTrue(m.writable())
 681
 682         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 683
 684         # Starting manifest
 685         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 686         assertRegex(self, collection2["manifest_text"],
 687             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 688
 689         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 690
 691         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 692         assertRegex(self, collection2["manifest_text"],
 693             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 694
 695
 696 def fuseRenameTestHelper(mounttmp):
 697     class Test(unittest.TestCase):
 698         def runTest(self):
 699             os.mkdir(os.path.join(mounttmp, "testdir"))
 700
 701             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 702                 f.write("Hello world!")
 703
 704     Test().runTest()
 705
 706 class FuseRenameTest(MountTestBase):
 707     def runTest(self):
 708         collection = arvados.collection.Collection(api_client=self.api)
 709         collection.save_new()
 710
 711         m = self.make_mount(fuse.CollectionDirectory)
 712         with llfuse.lock:
 713             m.new_collection(collection.api_response(), collection)
 714         self.assertTrue(m.writable())
 715
 716         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 717
 718         # Starting manifest
 719         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 720         assertRegex(self, collection2["manifest_text"],
 721             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 722
 723         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 724         self.assertEqual(["testdir"], d1)
 725         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 726         self.assertEqual(["file1.txt"], d1)
 727
 728         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 729
 730         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 731         self.assertEqual(["testdir2"], sorted(d1))
 732         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 733         self.assertEqual(["file1.txt"], d1)
 734
 735         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 736         assertRegex(self, collection2["manifest_text"],
 737             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 738
 739
 740 class FuseUpdateFromEventTest(MountTestBase):
 741     def runTest(self):
 742         collection = arvados.collection.Collection(api_client=self.api)
 743         collection.save_new()
 744
 745         m = self.make_mount(fuse.CollectionDirectory)
 746         with llfuse.lock:
 747             m.new_collection(collection.api_response(), collection)
 748
 749         self.operations.listen_for_events()
 750
 751         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 752         self.assertEqual([], sorted(d1))
 753
 754         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 755             with collection2.open("file1.txt", "w") as f:
 756                 f.write("foo")
 757
 758         for attempt in AssertWithTimeout(10):
 759             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 760
 761
 762 class FuseDeleteProjectEventTest(MountTestBase):
 763     def runTest(self):
 764
 765         aproject = self.api.groups().create(body={
 766             "name": "aproject",
 767             "group_class": "project"
 768         }).execute()
 769
 770         bproject = self.api.groups().create(body={
 771             "name": "bproject",
 772             "group_class": "project",
 773             "owner_uuid": aproject["uuid"]
 774         }).execute()
 775
 776         self.make_mount(fuse.ProjectDirectory,
 777                         project_object=self.api.users().current().execute())
 778
 779         self.operations.listen_for_events()
 780
 781         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 782         self.assertEqual(["bproject"], sorted(d1))
 783
 784         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 785
 786         for attempt in AssertWithTimeout(10):
 787             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 788
 789
 790 def fuseFileConflictTestHelper(mounttmp):
 791     class Test(unittest.TestCase):
 792         def runTest(self):
 793             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 794                 f.write("bar")
 795
 796             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 797             self.assertEqual(len(d1), 2)
 798
 799             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 800                 self.assertEqual(f.read(), "bar")
 801
 802             assertRegex(self, d1[1],
 803                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 804
 805             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 806                 self.assertEqual(f.read(), "foo")
 807
 808     Test().runTest()
 809
 810 class FuseFileConflictTest(MountTestBase):
 811     def runTest(self):
 812         collection = arvados.collection.Collection(api_client=self.api)
 813         collection.save_new()
 814
 815         m = self.make_mount(fuse.CollectionDirectory)
 816         with llfuse.lock:
 817             m.new_collection(collection.api_response(), collection)
 818
 819         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 820         self.assertEqual([], sorted(d1))
 821
 822         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 823             with collection2.open("file1.txt", "w") as f:
 824                 f.write("foo")
 825
 826         # See note in MountTestBase.setUp
 827         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 828
 829
 830 def fuseUnlinkOpenFileTest(mounttmp):
 831     class Test(unittest.TestCase):
 832         def runTest(self):
 833             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 834                 f.write("foo")
 835
 836                 d1 = llfuse.listdir(os.path.join(mounttmp))
 837                 self.assertEqual(["file1.txt"], sorted(d1))
 838
 839                 os.remove(os.path.join(mounttmp, "file1.txt"))
 840
 841                 d1 = llfuse.listdir(os.path.join(mounttmp))
 842                 self.assertEqual([], sorted(d1))
 843
 844                 f.seek(0)
 845                 self.assertEqual(f.read(), "foo")
 846                 f.write("bar")
 847
 848                 f.seek(0)
 849                 self.assertEqual(f.read(), "foobar")
 850
 851     Test().runTest()
 852
 853 class FuseUnlinkOpenFileTest(MountTestBase):
 854     def runTest(self):
 855         collection = arvados.collection.Collection(api_client=self.api)
 856         collection.save_new()
 857
 858         m = self.make_mount(fuse.CollectionDirectory)
 859         with llfuse.lock:
 860             m.new_collection(collection.api_response(), collection)
 861
 862         # See note in MountTestBase.setUp
 863         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 864
 865         self.assertEqual(collection.manifest_text(), "")
 866
 867
 868 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 869     class Test(unittest.TestCase):
 870         def runTest(self):
 871             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 872                 f.write("Hello world!")
 873
 874             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 875             self.assertEqual(["file1.txt"], sorted(d1))
 876             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 877             self.assertEqual([], sorted(d1))
 878
 879     Test().runTest()
 880
 881 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 882     class Test(unittest.TestCase):
 883         def runTest(self):
 884             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 885
 886             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 887             self.assertEqual([], sorted(d1))
 888             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 889             self.assertEqual(["file2.txt"], sorted(d1))
 890
 891     Test().runTest()
 892
 893 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 894     def runTest(self):
 895         collection1 = arvados.collection.Collection(api_client=self.api)
 896         collection1.save_new()
 897
 898         collection2 = arvados.collection.Collection(api_client=self.api)
 899         collection2.save_new()
 900
 901         m = self.make_mount(fuse.MagicDirectory)
 902
 903         # See note in MountTestBase.setUp
 904         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 905                                                   collection1.manifest_locator(),
 906                                                   collection2.manifest_locator()))
 907
 908         collection1.update()
 909         collection2.update()
 910
 911         assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 912         self.assertEqual(collection2.manifest_text(), "")
 913
 914         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 915                                                   collection1.manifest_locator(),
 916                                                   collection2.manifest_locator()))
 917
 918         collection1.update()
 919         collection2.update()
 920
 921         self.assertEqual(collection1.manifest_text(), "")
 922         assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 923
 924         collection1.stop_threads()
 925         collection2.stop_threads()
 926
 927
 928 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 929     class Test(unittest.TestCase):
 930         def runTest(self):
 931             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 932             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 933                 f.write("Hello world!")
 934
 935             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 936             self.assertEqual(["testdir"], sorted(d1))
 937             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 938             self.assertEqual(["file1.txt"], sorted(d1))
 939
 940             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 941             self.assertEqual([], sorted(d1))
 942
 943     Test().runTest()
 944
 945
 946 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 947     class Test(unittest.TestCase):
 948         def runTest(self):
 949             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 950
 951             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 952             self.assertEqual([], sorted(d1))
 953
 954             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 955             self.assertEqual(["testdir2"], sorted(d1))
 956             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 957             self.assertEqual(["file1.txt"], sorted(d1))
 958
 959             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 960                 self.assertEqual(f.read(), "Hello world!")
 961
 962     Test().runTest()
 963
 964 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 965     def runTest(self):
 966         collection1 = arvados.collection.Collection(api_client=self.api)
 967         collection1.save_new()
 968
 969         collection2 = arvados.collection.Collection(api_client=self.api)
 970         collection2.save_new()
 971
 972         m = self.make_mount(fuse.MagicDirectory)
 973
 974         # See note in MountTestBase.setUp
 975         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 976                                                   collection1.manifest_locator(),
 977                                                   collection2.manifest_locator()))
 978
 979         collection1.update()
 980         collection2.update()
 981
 982         assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 983         self.assertEqual(collection2.manifest_text(), "")
 984
 985         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 986                                                   collection1.manifest_locator(),
 987                                                   collection2.manifest_locator()))
 988
 989         collection1.update()
 990         collection2.update()
 991
 992         self.assertEqual(collection1.manifest_text(), "")
 993         assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 994
 995         collection1.stop_threads()
 996         collection2.stop_threads()
 997
 998 def fuseProjectMkdirTestHelper1(mounttmp):
 999     class Test(unittest.TestCase):
1000         def runTest(self):
1001             os.mkdir(os.path.join(mounttmp, "testcollection"))
1002             with self.assertRaises(OSError):
1003                 os.mkdir(os.path.join(mounttmp, "testcollection"))
1004     Test().runTest()
1005
1006 def fuseProjectMkdirTestHelper2(mounttmp):
1007     class Test(unittest.TestCase):
1008         def runTest(self):
1009             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1010                 f.write("Hello world!")
1011             with self.assertRaises(OSError):
1012                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1013             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1014             with self.assertRaises(OSError):
1015                 os.remove(os.path.join(mounttmp, "testcollection"))
1016             os.rmdir(os.path.join(mounttmp, "testcollection"))
1017     Test().runTest()
1018
1019 class FuseProjectMkdirRmdirTest(MountTestBase):
1020     def runTest(self):
1021         self.make_mount(fuse.ProjectDirectory,
1022                         project_object=self.api.users().current().execute())
1023
1024         d1 = llfuse.listdir(self.mounttmp)
1025         self.assertNotIn('testcollection', d1)
1026
1027         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1028
1029         d1 = llfuse.listdir(self.mounttmp)
1030         self.assertIn('testcollection', d1)
1031
1032         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1033
1034         d1 = llfuse.listdir(self.mounttmp)
1035         self.assertNotIn('testcollection', d1)
1036
1037
1038 def fuseProjectMvTestHelper1(mounttmp):
1039     class Test(unittest.TestCase):
1040         def runTest(self):
1041             d1 = llfuse.listdir(mounttmp)
1042             self.assertNotIn('testcollection', d1)
1043
1044             os.mkdir(os.path.join(mounttmp, "testcollection"))
1045
1046             d1 = llfuse.listdir(mounttmp)
1047             self.assertIn('testcollection', d1)
1048
1049             with self.assertRaises(OSError):
1050                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1051
1052             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1053
1054             d1 = llfuse.listdir(mounttmp)
1055             self.assertNotIn('testcollection', d1)
1056
1057             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1058             self.assertIn('testcollection', d1)
1059
1060     Test().runTest()
1061
1062 class FuseProjectMvTest(MountTestBase):
1063     def runTest(self):
1064         self.make_mount(fuse.ProjectDirectory,
1065                         project_object=self.api.users().current().execute())
1066
1067         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1068
1069
1070 def fuseFsyncTestHelper(mounttmp, k):
1071     class Test(unittest.TestCase):
1072         def runTest(self):
1073             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1074             os.fsync(fd)
1075             os.close(fd)
1076
1077     Test().runTest()
1078
1079 class FuseFsyncTest(FuseMagicTest):
1080     def runTest(self):
1081         self.make_mount(fuse.MagicDirectory)
1082         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1083
1084
1085 class MagicDirApiError(FuseMagicTest):
1086     def setUp(self):
1087         api = mock.MagicMock()
1088         super(MagicDirApiError, self).setUp(api=api)
1089         api.collections().get().execute.side_effect = iter([
1090             Exception('API fail'),
1091             {
1092                 "manifest_text": self.test_manifest,
1093                 "portable_data_hash": self.test_manifest_pdh,
1094             },
1095         ])
1096         api.keep.get.side_effect = Exception('Keep fail')
1097
1098     def runTest(self):
1099         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1100             self.make_mount(fuse.MagicDirectory)
1101
1102             self.operations.inodes.inode_cache.cap = 1
1103             self.operations.inodes.inode_cache.min_entries = 2
1104
1105             with self.assertRaises(OSError):
1106                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1107
1108             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1109
1110
1111 class SanitizeFilenameTest(MountTestBase):
1112     def test_sanitize_filename(self):
1113         pdir = fuse.ProjectDirectory(1, {}, self.api, 0, project_object=self.api.users().current().execute())
1114         acceptable = [
1115             "foo.txt",
1116             ".foo",
1117             "..foo",
1118             "...",
1119             "foo...",
1120             "foo..",
1121             "foo.",
1122             "-",
1123             "\x01\x02\x03",
1124             ]
1125         unacceptable = [
1126             "f\00",
1127             "\00\00",
1128             "/foo",
1129             "foo/",
1130             "//",
1131             ]
1132         for f in acceptable:
1133             self.assertEqual(f, pdir.sanitize_filename(f))
1134         for f in unacceptable:
1135             self.assertNotEqual(f, pdir.sanitize_filename(f))
1136             # The sanitized filename should be the same length, though.
1137             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1138         # Special cases
1139         self.assertEqual("_", pdir.sanitize_filename(""))
1140         self.assertEqual("_", pdir.sanitize_filename("."))
1141         self.assertEqual("__", pdir.sanitize_filename(".."))
1142
1143
1144 class FuseMagicTestPDHOnly(MountTestBase):
1145     def setUp(self, api=None):
1146         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1147
1148         cw = arvados.CollectionWriter()
1149
1150         cw.start_new_file('thing1.txt')
1151         cw.write("data 1")
1152
1153         self.testcollection = cw.finish()
1154         self.test_manifest = cw.manifest_text()
1155         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1156         self.testcollectionuuid = str(created['uuid'])
1157
1158     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1159         if skip_pdh_only is True:
1160             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1161         else:
1162             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1163
1164         mount_ls = llfuse.listdir(self.mounttmp)
1165         self.assertIn('README', mount_ls)
1166         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1167                              arvados.util.uuid_pattern.match(fn)
1168                              for fn in mount_ls),
1169                          "new FUSE MagicDirectory lists Collection")
1170
1171         # look up using pdh should succeed in all cases
1172         self.assertDirContents(self.testcollection, ['thing1.txt'])
1173         self.assertDirContents(os.path.join('by_id', self.testcollection),
1174                                ['thing1.txt'])
1175         mount_ls = llfuse.listdir(self.mounttmp)
1176         self.assertIn('README', mount_ls)
1177         self.assertIn(self.testcollection, mount_ls)
1178         self.assertIn(self.testcollection,
1179                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1180
1181         files = {}
1182         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1183
1184         for k, v in viewitems(files):
1185             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1186                 self.assertEqual(v, f.read().decode())
1187
1188         # look up using uuid should fail when pdh_only is set
1189         if pdh_only is True:
1190             with self.assertRaises(OSError):
1191                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1192                                ['thing1.txt'])
1193         else:
1194             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1195                                ['thing1.txt'])
1196
1197     def test_with_pdh_only_true(self):
1198         self.verify_pdh_only(pdh_only=True)
1199
1200     def test_with_pdh_only_false(self):
1201         self.verify_pdh_only(pdh_only=False)
1202
1203     def test_with_default_by_id(self):
1204         self.verify_pdh_only(skip_pdh_only=True)
1205
1206
1207 class SlashSubstitutionTest(IntegrationTest):
1208     mnt_args = [
1209         '--read-write',
1210         '--mount-home', 'zzz',
1211     ]
1212
1213     def setUp(self):
1214         super(SlashSubstitutionTest, self).setUp()
1215         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1216         self.api.config = lambda: {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1217         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1218         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1219         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1220
1221     @IntegrationTest.mount(argv=mnt_args)
1222     @mock.patch('arvados.util.get_config_once')
1223     def test_slash_substitution_before_listing(self, get_config_once):
1224         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1225         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1226         self.checkContents()
1227     @staticmethod
1228     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1229         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1230             f.write('xxx')
1231         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1232             f.write('foo')
1233
1234     @IntegrationTest.mount(argv=mnt_args)
1235     @mock.patch('arvados.util.get_config_once')
1236     def test_slash_substitution_after_listing(self, get_config_once):
1237         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1238         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1239         self.checkContents()
1240     @staticmethod
1241     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1242         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1243             f.write('xxx')
1244         os.listdir(tmpdir)
1245         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1246             f.write('foo')
1247
1248     def checkContents(self):
1249         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1250         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1251
1252     @IntegrationTest.mount(argv=mnt_args)
1253     @mock.patch('arvados.util.get_config_once')
1254     def test_slash_substitution_conflict(self, get_config_once):
1255         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1256         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1257         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1258         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1259         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1260         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1261     @staticmethod
1262     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1263         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1264             f.write('foo')