services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 from __future__ import absolute_import
   6 from future.utils import viewitems
   7 from builtins import str
   8 from builtins import object
   9 from six import assertRegex
  10 import json
  11 import llfuse
  12 import logging
  13 import mock
  14 import os
  15 import subprocess
  16 import time
  17 import unittest
  18
  19 import arvados
  20 import arvados_fuse as fuse
  21 from . import run_test_server
  22
  23 from .integration_test import IntegrationTest
  24 from .mount_test_base import MountTestBase
  25
  26 logger = logging.getLogger('arvados.arv-mount')
  27
  28
  29 class AssertWithTimeout(object):
  30     """Allow some time for an assertion to pass."""
  31
  32     def __init__(self, timeout=0):
  33         self.timeout = timeout
  34
  35     def __iter__(self):
  36         self.deadline = time.time() + self.timeout
  37         self.done = False
  38         return self
  39
  40     def __next__(self):
  41         if self.done:
  42             raise StopIteration
  43         return self.attempt
  44
  45     def attempt(self, fn, *args, **kwargs):
  46         try:
  47             fn(*args, **kwargs)
  48         except AssertionError:
  49             if time.time() > self.deadline:
  50                 raise
  51             time.sleep(0.1)
  52         else:
  53             self.done = True
  54
  55
  56 class FuseMountTest(MountTestBase):
  57     def setUp(self):
  58         super(FuseMountTest, self).setUp()
  59
  60         cw = arvados.CollectionWriter()
  61
  62         cw.start_new_file('thing1.txt')
  63         cw.write("data 1")
  64         cw.start_new_file('thing2.txt')
  65         cw.write("data 2")
  66
  67         cw.start_new_stream('dir1')
  68         cw.start_new_file('thing3.txt')
  69         cw.write("data 3")
  70         cw.start_new_file('thing4.txt')
  71         cw.write("data 4")
  72
  73         cw.start_new_stream('dir2')
  74         cw.start_new_file('thing5.txt')
  75         cw.write("data 5")
  76         cw.start_new_file('thing6.txt')
  77         cw.write("data 6")
  78
  79         cw.start_new_stream('dir2/dir3')
  80         cw.start_new_file('thing7.txt')
  81         cw.write("data 7")
  82
  83         cw.start_new_file('thing8.txt')
  84         cw.write("data 8")
  85
  86         cw.start_new_stream('edgecases')
  87         for f in ":/.../-/*/ ".split("/"):
  88             cw.start_new_file(f)
  89             cw.write('x')
  90
  91         for f in ":/.../-/*/ ".split("/"):
  92             cw.start_new_stream('edgecases/dirs/' + f)
  93             cw.start_new_file('x/x')
  94             cw.write('x')
  95
  96         self.testcollection = cw.finish()
  97         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  98
  99     def runTest(self):
 100         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
 101
 102         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 103                                       'edgecases', 'dir1', 'dir2'])
 104         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 105         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 106         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 107         self.assertDirContents('edgecases',
 108                                "dirs/:/.../-/*/ ".split("/"))
 109         self.assertDirContents('edgecases/dirs',
 110                                ":/.../-/*/ ".split("/"))
 111
 112         files = {'thing1.txt': 'data 1',
 113                  'thing2.txt': 'data 2',
 114                  'dir1/thing3.txt': 'data 3',
 115                  'dir1/thing4.txt': 'data 4',
 116                  'dir2/thing5.txt': 'data 5',
 117                  'dir2/thing6.txt': 'data 6',
 118                  'dir2/dir3/thing7.txt': 'data 7',
 119                  'dir2/dir3/thing8.txt': 'data 8'}
 120
 121         for k, v in viewitems(files):
 122             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 123                 self.assertEqual(v, f.read().decode())
 124
 125
 126 class FuseMagicTest(MountTestBase):
 127     def setUp(self, api=None):
 128         super(FuseMagicTest, self).setUp(api=api)
 129
 130         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 131         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
 132         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 133
 134         cw = arvados.CollectionWriter()
 135
 136         cw.start_new_file('thing1.txt')
 137         cw.write("data 1")
 138
 139         self.testcollection = cw.finish()
 140         self.test_manifest = cw.manifest_text()
 141         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 142         self.test_manifest_pdh = coll['portable_data_hash']
 143
 144     def runTest(self):
 145         self.make_mount(fuse.MagicDirectory)
 146
 147         mount_ls = llfuse.listdir(self.mounttmp)
 148         self.assertIn('README', mount_ls)
 149         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 150                              arvados.util.uuid_pattern.match(fn)
 151                              for fn in mount_ls),
 152                          "new FUSE MagicDirectory has no collections or projects")
 153         self.assertDirContents(self.testcollection, ['thing1.txt'])
 154         self.assertDirContents(os.path.join('by_id', self.testcollection),
 155                                ['thing1.txt'])
 156         self.assertIn(self.collection_in_test_project,
 157                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 158         self.assertIn(self.collection_in_test_project,
 159                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 160
 161         mount_ls = llfuse.listdir(self.mounttmp)
 162         self.assertIn('README', mount_ls)
 163         self.assertIn(self.testcollection, mount_ls)
 164         self.assertIn(self.testcollection,
 165                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 166         self.assertIn(self.test_project, mount_ls)
 167         self.assertIn(self.test_project,
 168                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 169
 170         with self.assertRaises(OSError):
 171             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 172
 173         files = {}
 174         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 175
 176         for k, v in viewitems(files):
 177             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 178                 self.assertEqual(v, f.read().decode())
 179
 180
 181 class FuseTagsTest(MountTestBase):
 182     def runTest(self):
 183         self.make_mount(fuse.TagsDirectory)
 184
 185         d1 = llfuse.listdir(self.mounttmp)
 186         d1.sort()
 187         self.assertEqual(['foo_tag'], d1)
 188
 189         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 190         d2.sort()
 191         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 192
 193         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 194         d3.sort()
 195         self.assertEqual(['foo'], d3)
 196
 197
 198 class FuseTagsUpdateTest(MountTestBase):
 199     def tag_collection(self, coll_uuid, tag_name):
 200         return self.api.links().create(
 201             body={'link': {'head_uuid': coll_uuid,
 202                            'link_class': 'tag',
 203                            'name': tag_name,
 204         }}).execute()
 205
 206     def runTest(self):
 207         self.make_mount(fuse.TagsDirectory, poll_time=1)
 208
 209         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 210
 211         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 212         self.tag_collection(bar_uuid, 'fuse_test_tag')
 213         for attempt in AssertWithTimeout(10):
 214             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 215         self.assertDirContents('fuse_test_tag', [bar_uuid])
 216
 217         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 218         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 219         for attempt in AssertWithTimeout(10):
 220             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 221
 222         self.api.links().delete(uuid=l['uuid']).execute()
 223         for attempt in AssertWithTimeout(10):
 224             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 225
 226
 227 def fuseSharedTestHelper(mounttmp):
 228     class Test(unittest.TestCase):
 229         def runTest(self):
 230             # Double check that we can open and read objects in this folder as a file,
 231             # and that its contents are what we expect.
 232             baz_path = os.path.join(
 233                 mounttmp,
 234                 'FUSE User',
 235                 'FUSE Test Project',
 236                 'collection in FUSE project',
 237                 'baz')
 238             with open(baz_path) as f:
 239                 self.assertEqual("baz", f.read())
 240
 241             # check mtime on collection
 242             st = os.stat(baz_path)
 243             try:
 244                 mtime = st.st_mtime_ns // 1000000000
 245             except AttributeError:
 246                 mtime = st.st_mtime
 247             self.assertEqual(mtime, 1391448174)
 248
 249             # shared_dirs is a list of the directories exposed
 250             # by fuse.SharedDirectory (i.e. any object visible
 251             # to the current user)
 252             shared_dirs = llfuse.listdir(mounttmp)
 253             shared_dirs.sort()
 254             self.assertIn('FUSE User', shared_dirs)
 255
 256             # fuse_user_objs is a list of the objects owned by the FUSE
 257             # test user (which present as files in the 'FUSE User'
 258             # directory)
 259             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 260             fuse_user_objs.sort()
 261             self.assertEqual(['FUSE Test Project',                    # project owned by user
 262                               'collection #1 owned by FUSE',          # collection owned by user
 263                               'collection #2 owned by FUSE'          # collection owned by user
 264                           ], fuse_user_objs)
 265
 266             # test_proj_files is a list of the files in the FUSE Test Project.
 267             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 268             test_proj_files.sort()
 269             self.assertEqual(['collection in FUSE project'
 270                           ], test_proj_files)
 271
 272
 273     Test().runTest()
 274
 275 class FuseSharedTest(MountTestBase):
 276     def runTest(self):
 277         self.make_mount(fuse.SharedDirectory,
 278                         exclude=self.api.users().current().execute()['uuid'])
 279         keep = arvados.keep.KeepClient()
 280         keep.put("baz".encode())
 281
 282         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 283
 284
 285 class FuseHomeTest(MountTestBase):
 286     def runTest(self):
 287         self.make_mount(fuse.ProjectDirectory,
 288                         project_object=self.api.users().current().execute())
 289
 290         d1 = llfuse.listdir(self.mounttmp)
 291         self.assertIn('Unrestricted public data', d1)
 292
 293         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 294         public_project = run_test_server.fixture('groups')[
 295             'anonymously_accessible_project']
 296         found_in = 0
 297         found_not_in = 0
 298         for name, item in viewitems(run_test_server.fixture('collections')):
 299             if 'name' not in item:
 300                 pass
 301             elif item['owner_uuid'] == public_project['uuid']:
 302                 self.assertIn(item['name'], d2)
 303                 found_in += 1
 304             else:
 305                 # Artificial assumption here: there is no public
 306                 # collection fixture with the same name as a
 307                 # non-public collection.
 308                 self.assertNotIn(item['name'], d2)
 309                 found_not_in += 1
 310         self.assertNotEqual(0, found_in)
 311         self.assertNotEqual(0, found_not_in)
 312
 313         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 314         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 315
 316
 317 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 318     class Test(unittest.TestCase):
 319         def runTest(self):
 320             d1 = llfuse.listdir(mounttmp)
 321             self.assertEqual(["file1.txt"], d1)
 322             with open(os.path.join(mounttmp, "file1.txt")) as f:
 323                 self.assertEqual("blub", f.read())
 324     Test().runTest()
 325
 326 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 327     class Test(unittest.TestCase):
 328         def runTest(self):
 329             d1 = llfuse.listdir(mounttmp)
 330             self.assertEqual(["file1.txt"], d1)
 331             with open(os.path.join(mounttmp, "file1.txt")) as f:
 332                 self.assertEqual("plnp", f.read())
 333     Test().runTest()
 334
 335 class FuseModifyFileTest(MountTestBase):
 336     def runTest(self):
 337         collection = arvados.collection.Collection(api_client=self.api)
 338         with collection.open("file1.txt", "w") as f:
 339             f.write("blub")
 340
 341         collection.save_new()
 342
 343         m = self.make_mount(fuse.CollectionDirectory)
 344         with llfuse.lock:
 345             m.new_collection(collection.api_response(), collection)
 346
 347         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 348
 349         with collection.open("file1.txt", "w") as f:
 350             f.write("plnp")
 351
 352         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 353
 354
 355 class FuseAddFileToCollectionTest(MountTestBase):
 356     def runTest(self):
 357         collection = arvados.collection.Collection(api_client=self.api)
 358         with collection.open("file1.txt", "w") as f:
 359             f.write("blub")
 360
 361         collection.save_new()
 362
 363         m = self.make_mount(fuse.CollectionDirectory)
 364         with llfuse.lock:
 365             m.new_collection(collection.api_response(), collection)
 366
 367         d1 = llfuse.listdir(self.mounttmp)
 368         self.assertEqual(["file1.txt"], d1)
 369
 370         with collection.open("file2.txt", "w") as f:
 371             f.write("plnp")
 372
 373         d1 = llfuse.listdir(self.mounttmp)
 374         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 375
 376
 377 class FuseRemoveFileFromCollectionTest(MountTestBase):
 378     def runTest(self):
 379         collection = arvados.collection.Collection(api_client=self.api)
 380         with collection.open("file1.txt", "w") as f:
 381             f.write("blub")
 382
 383         with collection.open("file2.txt", "w") as f:
 384             f.write("plnp")
 385
 386         collection.save_new()
 387
 388         m = self.make_mount(fuse.CollectionDirectory)
 389         with llfuse.lock:
 390             m.new_collection(collection.api_response(), collection)
 391
 392         d1 = llfuse.listdir(self.mounttmp)
 393         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 394
 395         collection.remove("file2.txt")
 396
 397         d1 = llfuse.listdir(self.mounttmp)
 398         self.assertEqual(["file1.txt"], d1)
 399
 400
 401 def fuseCreateFileTestHelper(mounttmp):
 402     class Test(unittest.TestCase):
 403         def runTest(self):
 404             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 405                 pass
 406     Test().runTest()
 407
 408 class FuseCreateFileTest(MountTestBase):
 409     def runTest(self):
 410         collection = arvados.collection.Collection(api_client=self.api)
 411         collection.save_new()
 412
 413         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 414         self.assertEqual(collection2["manifest_text"], "")
 415
 416         collection.save_new()
 417
 418         m = self.make_mount(fuse.CollectionDirectory)
 419         with llfuse.lock:
 420             m.new_collection(collection.api_response(), collection)
 421         self.assertTrue(m.writable())
 422
 423         self.assertNotIn("file1.txt", collection)
 424
 425         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 426
 427         self.assertIn("file1.txt", collection)
 428
 429         d1 = llfuse.listdir(self.mounttmp)
 430         self.assertEqual(["file1.txt"], d1)
 431
 432         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 433         assertRegex(self, collection2["manifest_text"],
 434             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 435
 436
 437 def fuseWriteFileTestHelperWriteFile(mounttmp):
 438     class Test(unittest.TestCase):
 439         def runTest(self):
 440             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 441                 f.write("Hello world!")
 442     Test().runTest()
 443
 444 def fuseWriteFileTestHelperReadFile(mounttmp):
 445     class Test(unittest.TestCase):
 446         def runTest(self):
 447             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 448                 self.assertEqual(f.read(), "Hello world!")
 449     Test().runTest()
 450
 451 class FuseWriteFileTest(MountTestBase):
 452     def runTest(self):
 453         collection = arvados.collection.Collection(api_client=self.api)
 454         collection.save_new()
 455
 456         m = self.make_mount(fuse.CollectionDirectory)
 457         with llfuse.lock:
 458             m.new_collection(collection.api_response(), collection)
 459         self.assertTrue(m.writable())
 460
 461         self.assertNotIn("file1.txt", collection)
 462
 463         self.assertEqual(0, self.operations.write_counter.get())
 464         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 465         self.assertEqual(12, self.operations.write_counter.get())
 466
 467         with collection.open("file1.txt") as f:
 468             self.assertEqual(f.read(), "Hello world!")
 469
 470         self.assertEqual(0, self.operations.read_counter.get())
 471         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 472         self.assertEqual(12, self.operations.read_counter.get())
 473
 474         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 475         assertRegex(self, collection2["manifest_text"],
 476             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 477
 478
 479 def fuseUpdateFileTestHelper(mounttmp):
 480     class Test(unittest.TestCase):
 481         def runTest(self):
 482             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 483                 f.write("Hello world!")
 484
 485             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 486                 fr = f.read()
 487                 self.assertEqual(fr, "Hello world!")
 488                 f.seek(0)
 489                 f.write("Hola mundo!")
 490                 f.seek(0)
 491                 fr = f.read()
 492                 self.assertEqual(fr, "Hola mundo!!")
 493
 494             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 495                 self.assertEqual(f.read(), "Hola mundo!!")
 496
 497     Test().runTest()
 498
 499 class FuseUpdateFileTest(MountTestBase):
 500     def runTest(self):
 501         collection = arvados.collection.Collection(api_client=self.api)
 502         collection.save_new()
 503
 504         m = self.make_mount(fuse.CollectionDirectory)
 505         with llfuse.lock:
 506             m.new_collection(collection.api_response(), collection)
 507         self.assertTrue(m.writable())
 508
 509         # See note in MountTestBase.setUp
 510         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 511
 512         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 513         assertRegex(self, collection2["manifest_text"],
 514             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 515
 516
 517 def fuseMkdirTestHelper(mounttmp):
 518     class Test(unittest.TestCase):
 519         def runTest(self):
 520             with self.assertRaises(IOError):
 521                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 522                     f.write("Hello world!")
 523
 524             os.mkdir(os.path.join(mounttmp, "testdir"))
 525
 526             with self.assertRaises(OSError):
 527                 os.mkdir(os.path.join(mounttmp, "testdir"))
 528
 529             d1 = llfuse.listdir(mounttmp)
 530             self.assertEqual(["testdir"], d1)
 531
 532             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 533                 f.write("Hello world!")
 534
 535             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 536             self.assertEqual(["file1.txt"], d1)
 537
 538     Test().runTest()
 539
 540 class FuseMkdirTest(MountTestBase):
 541     def runTest(self):
 542         collection = arvados.collection.Collection(api_client=self.api)
 543         collection.save_new()
 544
 545         m = self.make_mount(fuse.CollectionDirectory)
 546         with llfuse.lock:
 547             m.new_collection(collection.api_response(), collection)
 548         self.assertTrue(m.writable())
 549
 550         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 551
 552         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 553         assertRegex(self, collection2["manifest_text"],
 554             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 555
 556
 557 def fuseRmTestHelperWriteFile(mounttmp):
 558     class Test(unittest.TestCase):
 559         def runTest(self):
 560             os.mkdir(os.path.join(mounttmp, "testdir"))
 561
 562             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 563                 f.write("Hello world!")
 564
 565     Test().runTest()
 566
 567 def fuseRmTestHelperDeleteFile(mounttmp):
 568     class Test(unittest.TestCase):
 569         def runTest(self):
 570             # Can't delete because it's not empty
 571             with self.assertRaises(OSError):
 572                 os.rmdir(os.path.join(mounttmp, "testdir"))
 573
 574             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 575             self.assertEqual(["file1.txt"], d1)
 576
 577             # Delete file
 578             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 579
 580             # Make sure it's empty
 581             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 582             self.assertEqual([], d1)
 583
 584             # Try to delete it again
 585             with self.assertRaises(OSError):
 586                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 587
 588     Test().runTest()
 589
 590 def fuseRmTestHelperRmdir(mounttmp):
 591     class Test(unittest.TestCase):
 592         def runTest(self):
 593             # Should be able to delete now that it is empty
 594             os.rmdir(os.path.join(mounttmp, "testdir"))
 595
 596             # Make sure it's empty
 597             d1 = llfuse.listdir(os.path.join(mounttmp))
 598             self.assertEqual([], d1)
 599
 600             # Try to delete it again
 601             with self.assertRaises(OSError):
 602                 os.rmdir(os.path.join(mounttmp, "testdir"))
 603
 604     Test().runTest()
 605
 606 class FuseRmTest(MountTestBase):
 607     def runTest(self):
 608         collection = arvados.collection.Collection(api_client=self.api)
 609         collection.save_new()
 610
 611         m = self.make_mount(fuse.CollectionDirectory)
 612         with llfuse.lock:
 613             m.new_collection(collection.api_response(), collection)
 614         self.assertTrue(m.writable())
 615
 616         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 617
 618         # Starting manifest
 619         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 620         assertRegex(self, collection2["manifest_text"],
 621             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 622         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 623
 624         # Empty directories are represented by an empty file named "."
 625         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 626         assertRegex(self, collection2["manifest_text"],
 627                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 628
 629         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 630
 631         # manifest should be empty now.
 632         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 633         self.assertEqual(collection2["manifest_text"], "")
 634
 635
 636 def fuseMvFileTestHelperWriteFile(mounttmp):
 637     class Test(unittest.TestCase):
 638         def runTest(self):
 639             os.mkdir(os.path.join(mounttmp, "testdir"))
 640
 641             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 642                 f.write("Hello world!")
 643
 644     Test().runTest()
 645
 646 def fuseMvFileTestHelperMoveFile(mounttmp):
 647     class Test(unittest.TestCase):
 648         def runTest(self):
 649             d1 = llfuse.listdir(os.path.join(mounttmp))
 650             self.assertEqual(["testdir"], d1)
 651             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 652             self.assertEqual(["file1.txt"], d1)
 653
 654             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 655
 656             d1 = llfuse.listdir(os.path.join(mounttmp))
 657             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 658             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 659             self.assertEqual([], d1)
 660
 661     Test().runTest()
 662
 663 class FuseMvFileTest(MountTestBase):
 664     def runTest(self):
 665         collection = arvados.collection.Collection(api_client=self.api)
 666         collection.save_new()
 667
 668         m = self.make_mount(fuse.CollectionDirectory)
 669         with llfuse.lock:
 670             m.new_collection(collection.api_response(), collection)
 671         self.assertTrue(m.writable())
 672
 673         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 674
 675         # Starting manifest
 676         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 677         assertRegex(self, collection2["manifest_text"],
 678             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 679
 680         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 681
 682         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 683         assertRegex(self, collection2["manifest_text"],
 684             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 685
 686
 687 def fuseRenameTestHelper(mounttmp):
 688     class Test(unittest.TestCase):
 689         def runTest(self):
 690             os.mkdir(os.path.join(mounttmp, "testdir"))
 691
 692             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 693                 f.write("Hello world!")
 694
 695     Test().runTest()
 696
 697 class FuseRenameTest(MountTestBase):
 698     def runTest(self):
 699         collection = arvados.collection.Collection(api_client=self.api)
 700         collection.save_new()
 701
 702         m = self.make_mount(fuse.CollectionDirectory)
 703         with llfuse.lock:
 704             m.new_collection(collection.api_response(), collection)
 705         self.assertTrue(m.writable())
 706
 707         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 708
 709         # Starting manifest
 710         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 711         assertRegex(self, collection2["manifest_text"],
 712             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 713
 714         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 715         self.assertEqual(["testdir"], d1)
 716         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 717         self.assertEqual(["file1.txt"], d1)
 718
 719         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 720
 721         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 722         self.assertEqual(["testdir2"], sorted(d1))
 723         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 724         self.assertEqual(["file1.txt"], d1)
 725
 726         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 727         assertRegex(self, collection2["manifest_text"],
 728             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 729
 730
 731 class FuseUpdateFromEventTest(MountTestBase):
 732     def runTest(self):
 733         collection = arvados.collection.Collection(api_client=self.api)
 734         collection.save_new()
 735
 736         m = self.make_mount(fuse.CollectionDirectory)
 737         with llfuse.lock:
 738             m.new_collection(collection.api_response(), collection)
 739
 740         self.operations.listen_for_events()
 741
 742         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 743         self.assertEqual([], sorted(d1))
 744
 745         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 746             with collection2.open("file1.txt", "w") as f:
 747                 f.write("foo")
 748
 749         for attempt in AssertWithTimeout(10):
 750             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 751
 752
 753 class FuseDeleteProjectEventTest(MountTestBase):
 754     def runTest(self):
 755
 756         aproject = self.api.groups().create(body={
 757             "name": "aproject",
 758             "group_class": "project"
 759         }).execute()
 760
 761         bproject = self.api.groups().create(body={
 762             "name": "bproject",
 763             "group_class": "project",
 764             "owner_uuid": aproject["uuid"]
 765         }).execute()
 766
 767         self.make_mount(fuse.ProjectDirectory,
 768                         project_object=self.api.users().current().execute())
 769
 770         self.operations.listen_for_events()
 771
 772         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 773         self.assertEqual(["bproject"], sorted(d1))
 774
 775         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 776
 777         for attempt in AssertWithTimeout(10):
 778             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 779
 780
 781 def fuseFileConflictTestHelper(mounttmp):
 782     class Test(unittest.TestCase):
 783         def runTest(self):
 784             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 785                 f.write("bar")
 786
 787             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 788             self.assertEqual(len(d1), 2)
 789
 790             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 791                 self.assertEqual(f.read(), "bar")
 792
 793             assertRegex(self, d1[1],
 794                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 795
 796             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 797                 self.assertEqual(f.read(), "foo")
 798
 799     Test().runTest()
 800
 801 class FuseFileConflictTest(MountTestBase):
 802     def runTest(self):
 803         collection = arvados.collection.Collection(api_client=self.api)
 804         collection.save_new()
 805
 806         m = self.make_mount(fuse.CollectionDirectory)
 807         with llfuse.lock:
 808             m.new_collection(collection.api_response(), collection)
 809
 810         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 811         self.assertEqual([], sorted(d1))
 812
 813         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 814             with collection2.open("file1.txt", "w") as f:
 815                 f.write("foo")
 816
 817         # See note in MountTestBase.setUp
 818         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 819
 820
 821 def fuseUnlinkOpenFileTest(mounttmp):
 822     class Test(unittest.TestCase):
 823         def runTest(self):
 824             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 825                 f.write("foo")
 826
 827                 d1 = llfuse.listdir(os.path.join(mounttmp))
 828                 self.assertEqual(["file1.txt"], sorted(d1))
 829
 830                 os.remove(os.path.join(mounttmp, "file1.txt"))
 831
 832                 d1 = llfuse.listdir(os.path.join(mounttmp))
 833                 self.assertEqual([], sorted(d1))
 834
 835                 f.seek(0)
 836                 self.assertEqual(f.read(), "foo")
 837                 f.write("bar")
 838
 839                 f.seek(0)
 840                 self.assertEqual(f.read(), "foobar")
 841
 842     Test().runTest()
 843
 844 class FuseUnlinkOpenFileTest(MountTestBase):
 845     def runTest(self):
 846         collection = arvados.collection.Collection(api_client=self.api)
 847         collection.save_new()
 848
 849         m = self.make_mount(fuse.CollectionDirectory)
 850         with llfuse.lock:
 851             m.new_collection(collection.api_response(), collection)
 852
 853         # See note in MountTestBase.setUp
 854         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 855
 856         self.assertEqual(collection.manifest_text(), "")
 857
 858
 859 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 860     class Test(unittest.TestCase):
 861         def runTest(self):
 862             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 863                 f.write("Hello world!")
 864
 865             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 866             self.assertEqual(["file1.txt"], sorted(d1))
 867             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 868             self.assertEqual([], sorted(d1))
 869
 870     Test().runTest()
 871
 872 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 873     class Test(unittest.TestCase):
 874         def runTest(self):
 875             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 876
 877             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 878             self.assertEqual([], sorted(d1))
 879             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 880             self.assertEqual(["file2.txt"], sorted(d1))
 881
 882     Test().runTest()
 883
 884 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 885     def runTest(self):
 886         collection1 = arvados.collection.Collection(api_client=self.api)
 887         collection1.save_new()
 888
 889         collection2 = arvados.collection.Collection(api_client=self.api)
 890         collection2.save_new()
 891
 892         m = self.make_mount(fuse.MagicDirectory)
 893
 894         # See note in MountTestBase.setUp
 895         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 896                                                   collection1.manifest_locator(),
 897                                                   collection2.manifest_locator()))
 898
 899         collection1.update()
 900         collection2.update()
 901
 902         assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 903         self.assertEqual(collection2.manifest_text(), "")
 904
 905         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 906                                                   collection1.manifest_locator(),
 907                                                   collection2.manifest_locator()))
 908
 909         collection1.update()
 910         collection2.update()
 911
 912         self.assertEqual(collection1.manifest_text(), "")
 913         assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 914
 915         collection1.stop_threads()
 916         collection2.stop_threads()
 917
 918
 919 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 920     class Test(unittest.TestCase):
 921         def runTest(self):
 922             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 923             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 924                 f.write("Hello world!")
 925
 926             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 927             self.assertEqual(["testdir"], sorted(d1))
 928             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 929             self.assertEqual(["file1.txt"], sorted(d1))
 930
 931             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 932             self.assertEqual([], sorted(d1))
 933
 934     Test().runTest()
 935
 936
 937 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 938     class Test(unittest.TestCase):
 939         def runTest(self):
 940             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 941
 942             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 943             self.assertEqual([], sorted(d1))
 944
 945             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 946             self.assertEqual(["testdir2"], sorted(d1))
 947             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 948             self.assertEqual(["file1.txt"], sorted(d1))
 949
 950             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 951                 self.assertEqual(f.read(), "Hello world!")
 952
 953     Test().runTest()
 954
 955 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 956     def runTest(self):
 957         collection1 = arvados.collection.Collection(api_client=self.api)
 958         collection1.save_new()
 959
 960         collection2 = arvados.collection.Collection(api_client=self.api)
 961         collection2.save_new()
 962
 963         m = self.make_mount(fuse.MagicDirectory)
 964
 965         # See note in MountTestBase.setUp
 966         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 967                                                   collection1.manifest_locator(),
 968                                                   collection2.manifest_locator()))
 969
 970         collection1.update()
 971         collection2.update()
 972
 973         assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 974         self.assertEqual(collection2.manifest_text(), "")
 975
 976         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 977                                                   collection1.manifest_locator(),
 978                                                   collection2.manifest_locator()))
 979
 980         collection1.update()
 981         collection2.update()
 982
 983         self.assertEqual(collection1.manifest_text(), "")
 984         assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 985
 986         collection1.stop_threads()
 987         collection2.stop_threads()
 988
 989 def fuseProjectMkdirTestHelper1(mounttmp):
 990     class Test(unittest.TestCase):
 991         def runTest(self):
 992             os.mkdir(os.path.join(mounttmp, "testcollection"))
 993             with self.assertRaises(OSError):
 994                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 995     Test().runTest()
 996
 997 def fuseProjectMkdirTestHelper2(mounttmp):
 998     class Test(unittest.TestCase):
 999         def runTest(self):
1000             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1001                 f.write("Hello world!")
1002             with self.assertRaises(OSError):
1003                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1004             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1005             with self.assertRaises(OSError):
1006                 os.remove(os.path.join(mounttmp, "testcollection"))
1007             os.rmdir(os.path.join(mounttmp, "testcollection"))
1008     Test().runTest()
1009
1010 class FuseProjectMkdirRmdirTest(MountTestBase):
1011     def runTest(self):
1012         self.make_mount(fuse.ProjectDirectory,
1013                         project_object=self.api.users().current().execute())
1014
1015         d1 = llfuse.listdir(self.mounttmp)
1016         self.assertNotIn('testcollection', d1)
1017
1018         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1019
1020         d1 = llfuse.listdir(self.mounttmp)
1021         self.assertIn('testcollection', d1)
1022
1023         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1024
1025         d1 = llfuse.listdir(self.mounttmp)
1026         self.assertNotIn('testcollection', d1)
1027
1028
1029 def fuseProjectMvTestHelper1(mounttmp):
1030     class Test(unittest.TestCase):
1031         def runTest(self):
1032             d1 = llfuse.listdir(mounttmp)
1033             self.assertNotIn('testcollection', d1)
1034
1035             os.mkdir(os.path.join(mounttmp, "testcollection"))
1036
1037             d1 = llfuse.listdir(mounttmp)
1038             self.assertIn('testcollection', d1)
1039
1040             with self.assertRaises(OSError):
1041                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1042
1043             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1044
1045             d1 = llfuse.listdir(mounttmp)
1046             self.assertNotIn('testcollection', d1)
1047
1048             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1049             self.assertIn('testcollection', d1)
1050
1051     Test().runTest()
1052
1053 class FuseProjectMvTest(MountTestBase):
1054     def runTest(self):
1055         self.make_mount(fuse.ProjectDirectory,
1056                         project_object=self.api.users().current().execute())
1057
1058         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1059
1060
1061 def fuseFsyncTestHelper(mounttmp, k):
1062     class Test(unittest.TestCase):
1063         def runTest(self):
1064             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1065             os.fsync(fd)
1066             os.close(fd)
1067
1068     Test().runTest()
1069
1070 class FuseFsyncTest(FuseMagicTest):
1071     def runTest(self):
1072         self.make_mount(fuse.MagicDirectory)
1073         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1074
1075
1076 class MagicDirApiError(FuseMagicTest):
1077     def setUp(self):
1078         api = mock.MagicMock()
1079         super(MagicDirApiError, self).setUp(api=api)
1080         api.collections().get().execute.side_effect = iter([
1081             Exception('API fail'),
1082             {
1083                 "manifest_text": self.test_manifest,
1084                 "portable_data_hash": self.test_manifest_pdh,
1085             },
1086         ])
1087         api.keep.get.side_effect = Exception('Keep fail')
1088
1089     def runTest(self):
1090         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1091             self.make_mount(fuse.MagicDirectory)
1092
1093             self.operations.inodes.inode_cache.cap = 1
1094             self.operations.inodes.inode_cache.min_entries = 2
1095
1096             with self.assertRaises(OSError):
1097                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1098
1099             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1100
1101
1102 class SanitizeFilenameTest(MountTestBase):
1103     def test_sanitize_filename(self):
1104         pdir = fuse.ProjectDirectory(1, {}, self.api, 0, project_object=self.api.users().current().execute())
1105         acceptable = [
1106             "foo.txt",
1107             ".foo",
1108             "..foo",
1109             "...",
1110             "foo...",
1111             "foo..",
1112             "foo.",
1113             "-",
1114             "\x01\x02\x03",
1115             ]
1116         unacceptable = [
1117             "f\00",
1118             "\00\00",
1119             "/foo",
1120             "foo/",
1121             "//",
1122             ]
1123         for f in acceptable:
1124             self.assertEqual(f, pdir.sanitize_filename(f))
1125         for f in unacceptable:
1126             self.assertNotEqual(f, pdir.sanitize_filename(f))
1127             # The sanitized filename should be the same length, though.
1128             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1129         # Special cases
1130         self.assertEqual("_", pdir.sanitize_filename(""))
1131         self.assertEqual("_", pdir.sanitize_filename("."))
1132         self.assertEqual("__", pdir.sanitize_filename(".."))
1133
1134
1135 class FuseMagicTestPDHOnly(MountTestBase):
1136     def setUp(self, api=None):
1137         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1138
1139         cw = arvados.CollectionWriter()
1140
1141         cw.start_new_file('thing1.txt')
1142         cw.write("data 1")
1143
1144         self.testcollection = cw.finish()
1145         self.test_manifest = cw.manifest_text()
1146         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1147         self.testcollectionuuid = str(created['uuid'])
1148
1149     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1150         if skip_pdh_only is True:
1151             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1152         else:
1153             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1154
1155         mount_ls = llfuse.listdir(self.mounttmp)
1156         self.assertIn('README', mount_ls)
1157         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1158                              arvados.util.uuid_pattern.match(fn)
1159                              for fn in mount_ls),
1160                          "new FUSE MagicDirectory lists Collection")
1161
1162         # look up using pdh should succeed in all cases
1163         self.assertDirContents(self.testcollection, ['thing1.txt'])
1164         self.assertDirContents(os.path.join('by_id', self.testcollection),
1165                                ['thing1.txt'])
1166         mount_ls = llfuse.listdir(self.mounttmp)
1167         self.assertIn('README', mount_ls)
1168         self.assertIn(self.testcollection, mount_ls)
1169         self.assertIn(self.testcollection,
1170                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1171
1172         files = {}
1173         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1174
1175         for k, v in viewitems(files):
1176             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1177                 self.assertEqual(v, f.read().decode())
1178
1179         # look up using uuid should fail when pdh_only is set
1180         if pdh_only is True:
1181             with self.assertRaises(OSError):
1182                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1183                                ['thing1.txt'])
1184         else:
1185             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1186                                ['thing1.txt'])
1187
1188     def test_with_pdh_only_true(self):
1189         self.verify_pdh_only(pdh_only=True)
1190
1191     def test_with_pdh_only_false(self):
1192         self.verify_pdh_only(pdh_only=False)
1193
1194     def test_with_default_by_id(self):
1195         self.verify_pdh_only(skip_pdh_only=True)
1196
1197
1198 class SlashSubstitutionTest(IntegrationTest):
1199     mnt_args = [
1200         '--read-write',
1201         '--mount-home', 'zzz',
1202     ]
1203
1204     def setUp(self):
1205         super(SlashSubstitutionTest, self).setUp()
1206         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1207         self.api.config = lambda: {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1208         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1209         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1210         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1211
1212     @IntegrationTest.mount(argv=mnt_args)
1213     @mock.patch('arvados.util.get_config_once')
1214     def test_slash_substitution_before_listing(self, get_config_once):
1215         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1216         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1217         self.checkContents()
1218     @staticmethod
1219     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1220         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1221             f.write('xxx')
1222         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1223             f.write('foo')
1224
1225     @IntegrationTest.mount(argv=mnt_args)
1226     @mock.patch('arvados.util.get_config_once')
1227     def test_slash_substitution_after_listing(self, get_config_once):
1228         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1229         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1230         self.checkContents()
1231     @staticmethod
1232     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1233         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1234             f.write('xxx')
1235         os.listdir(tmpdir)
1236         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1237             f.write('foo')
1238
1239     def checkContents(self):
1240         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1241         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1242
1243     @IntegrationTest.mount(argv=mnt_args)
1244     @mock.patch('arvados.util.get_config_once')
1245     def test_slash_substitution_conflict(self, get_config_once):
1246         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1247         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1248         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1249         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1250         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1251         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1252     @staticmethod
1253     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1254         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1255             f.write('foo')