services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 import json
   6 import llfuse
   7 import logging
   8 import mock
   9 import os
  10 import subprocess
  11 import time
  12 import unittest
  13
  14 import arvados
  15 import arvados_fuse as fuse
  16 import run_test_server
  17
  18 from mount_test_base import MountTestBase
  19
  20 logger = logging.getLogger('arvados.arv-mount')
  21
  22
  23 class AssertWithTimeout(object):
  24     """Allow some time for an assertion to pass."""
  25
  26     def __init__(self, timeout=0):
  27         self.timeout = timeout
  28
  29     def __iter__(self):
  30         self.deadline = time.time() + self.timeout
  31         self.done = False
  32         return self
  33
  34     def next(self):
  35         if self.done:
  36             raise StopIteration
  37         return self.attempt
  38
  39     def attempt(self, fn, *args, **kwargs):
  40         try:
  41             fn(*args, **kwargs)
  42         except AssertionError:
  43             if time.time() > self.deadline:
  44                 raise
  45             time.sleep(0.1)
  46         else:
  47             self.done = True
  48
  49
  50 class FuseMountTest(MountTestBase):
  51     def setUp(self):
  52         super(FuseMountTest, self).setUp()
  53
  54         cw = arvados.CollectionWriter()
  55
  56         cw.start_new_file('thing1.txt')
  57         cw.write("data 1")
  58         cw.start_new_file('thing2.txt')
  59         cw.write("data 2")
  60
  61         cw.start_new_stream('dir1')
  62         cw.start_new_file('thing3.txt')
  63         cw.write("data 3")
  64         cw.start_new_file('thing4.txt')
  65         cw.write("data 4")
  66
  67         cw.start_new_stream('dir2')
  68         cw.start_new_file('thing5.txt')
  69         cw.write("data 5")
  70         cw.start_new_file('thing6.txt')
  71         cw.write("data 6")
  72
  73         cw.start_new_stream('dir2/dir3')
  74         cw.start_new_file('thing7.txt')
  75         cw.write("data 7")
  76
  77         cw.start_new_file('thing8.txt')
  78         cw.write("data 8")
  79
  80         cw.start_new_stream('edgecases')
  81         for f in ":/.../-/*/\x01\\/ ".split("/"):
  82             cw.start_new_file(f)
  83             cw.write('x')
  84
  85         for f in ":/.../-/*/\x01\\/ ".split("/"):
  86             cw.start_new_stream('edgecases/dirs/' + f)
  87             cw.start_new_file('x/x')
  88             cw.write('x')
  89
  90         self.testcollection = cw.finish()
  91         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  92
  93     def runTest(self):
  94         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  95
  96         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
  97                                       'edgecases', 'dir1', 'dir2'])
  98         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
  99         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 100         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 101         self.assertDirContents('edgecases',
 102                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
 103         self.assertDirContents('edgecases/dirs',
 104                                ":/.../-/*/\x01\\/ ".split("/"))
 105
 106         files = {'thing1.txt': 'data 1',
 107                  'thing2.txt': 'data 2',
 108                  'dir1/thing3.txt': 'data 3',
 109                  'dir1/thing4.txt': 'data 4',
 110                  'dir2/thing5.txt': 'data 5',
 111                  'dir2/thing6.txt': 'data 6',
 112                  'dir2/dir3/thing7.txt': 'data 7',
 113                  'dir2/dir3/thing8.txt': 'data 8'}
 114
 115         for k, v in files.items():
 116             with open(os.path.join(self.mounttmp, k)) as f:
 117                 self.assertEqual(v, f.read())
 118
 119
 120 class FuseMagicTest(MountTestBase):
 121     def setUp(self, api=None):
 122         super(FuseMagicTest, self).setUp(api=api)
 123
 124         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 125         self.non_project_group = run_test_server.fixture('groups')['public']['uuid']
 126         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 127
 128         cw = arvados.CollectionWriter()
 129
 130         cw.start_new_file('thing1.txt')
 131         cw.write("data 1")
 132
 133         self.testcollection = cw.finish()
 134         self.test_manifest = cw.manifest_text()
 135         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 136         self.test_manifest_pdh = coll['portable_data_hash']
 137
 138     def runTest(self):
 139         self.make_mount(fuse.MagicDirectory)
 140
 141         mount_ls = llfuse.listdir(self.mounttmp)
 142         self.assertIn('README', mount_ls)
 143         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 144                              arvados.util.uuid_pattern.match(fn)
 145                              for fn in mount_ls),
 146                          "new FUSE MagicDirectory has no collections or projects")
 147         self.assertDirContents(self.testcollection, ['thing1.txt'])
 148         self.assertDirContents(os.path.join('by_id', self.testcollection),
 149                                ['thing1.txt'])
 150         self.assertIn(self.collection_in_test_project,
 151                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 152         self.assertIn(self.collection_in_test_project,
 153                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 154
 155         mount_ls = llfuse.listdir(self.mounttmp)
 156         self.assertIn('README', mount_ls)
 157         self.assertIn(self.testcollection, mount_ls)
 158         self.assertIn(self.testcollection,
 159                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 160         self.assertIn(self.test_project, mount_ls)
 161         self.assertIn(self.test_project,
 162                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 163
 164         with self.assertRaises(OSError):
 165             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 166
 167         files = {}
 168         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 169
 170         for k, v in files.items():
 171             with open(os.path.join(self.mounttmp, k)) as f:
 172                 self.assertEqual(v, f.read())
 173
 174
 175 class FuseTagsTest(MountTestBase):
 176     def runTest(self):
 177         self.make_mount(fuse.TagsDirectory)
 178
 179         d1 = llfuse.listdir(self.mounttmp)
 180         d1.sort()
 181         self.assertEqual(['foo_tag'], d1)
 182
 183         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 184         d2.sort()
 185         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 186
 187         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 188         d3.sort()
 189         self.assertEqual(['foo'], d3)
 190
 191
 192 class FuseTagsUpdateTest(MountTestBase):
 193     def tag_collection(self, coll_uuid, tag_name):
 194         return self.api.links().create(
 195             body={'link': {'head_uuid': coll_uuid,
 196                            'link_class': 'tag',
 197                            'name': tag_name,
 198         }}).execute()
 199
 200     def runTest(self):
 201         self.make_mount(fuse.TagsDirectory, poll_time=1)
 202
 203         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 204
 205         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 206         self.tag_collection(bar_uuid, 'fuse_test_tag')
 207         for attempt in AssertWithTimeout(10):
 208             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 209         self.assertDirContents('fuse_test_tag', [bar_uuid])
 210
 211         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 212         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 213         for attempt in AssertWithTimeout(10):
 214             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 215
 216         self.api.links().delete(uuid=l['uuid']).execute()
 217         for attempt in AssertWithTimeout(10):
 218             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 219
 220
 221 def fuseSharedTestHelper(mounttmp):
 222     class Test(unittest.TestCase):
 223         def runTest(self):
 224             # Double check that we can open and read objects in this folder as a file,
 225             # and that its contents are what we expect.
 226             baz_path = os.path.join(
 227                 mounttmp,
 228                 'FUSE User',
 229                 'FUSE Test Project',
 230                 'collection in FUSE project',
 231                 'baz')
 232             with open(baz_path) as f:
 233                 self.assertEqual("baz", f.read())
 234
 235             # check mtime on collection
 236             st = os.stat(baz_path)
 237             try:
 238                 mtime = st.st_mtime_ns / 1000000000
 239             except AttributeError:
 240                 mtime = st.st_mtime
 241             self.assertEqual(mtime, 1391448174)
 242
 243             # shared_dirs is a list of the directories exposed
 244             # by fuse.SharedDirectory (i.e. any object visible
 245             # to the current user)
 246             shared_dirs = llfuse.listdir(mounttmp)
 247             shared_dirs.sort()
 248             self.assertIn('FUSE User', shared_dirs)
 249
 250             # fuse_user_objs is a list of the objects owned by the FUSE
 251             # test user (which present as files in the 'FUSE User'
 252             # directory)
 253             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 254             fuse_user_objs.sort()
 255             self.assertEqual(['FUSE Test Project',                    # project owned by user
 256                               'collection #1 owned by FUSE',          # collection owned by user
 257                               'collection #2 owned by FUSE'          # collection owned by user
 258                           ], fuse_user_objs)
 259
 260             # test_proj_files is a list of the files in the FUSE Test Project.
 261             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 262             test_proj_files.sort()
 263             self.assertEqual(['collection in FUSE project'
 264                           ], test_proj_files)
 265
 266
 267     Test().runTest()
 268
 269 class FuseSharedTest(MountTestBase):
 270     def runTest(self):
 271         self.make_mount(fuse.SharedDirectory,
 272                         exclude=self.api.users().current().execute()['uuid'])
 273         keep = arvados.keep.KeepClient()
 274         keep.put("baz")
 275
 276         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 277
 278
 279 class FuseHomeTest(MountTestBase):
 280     def runTest(self):
 281         self.make_mount(fuse.ProjectDirectory,
 282                         project_object=self.api.users().current().execute())
 283
 284         d1 = llfuse.listdir(self.mounttmp)
 285         self.assertIn('Unrestricted public data', d1)
 286
 287         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 288         public_project = run_test_server.fixture('groups')[
 289             'anonymously_accessible_project']
 290         found_in = 0
 291         found_not_in = 0
 292         for name, item in run_test_server.fixture('collections').iteritems():
 293             if 'name' not in item:
 294                 pass
 295             elif item['owner_uuid'] == public_project['uuid']:
 296                 self.assertIn(item['name'], d2)
 297                 found_in += 1
 298             else:
 299                 # Artificial assumption here: there is no public
 300                 # collection fixture with the same name as a
 301                 # non-public collection.
 302                 self.assertNotIn(item['name'], d2)
 303                 found_not_in += 1
 304         self.assertNotEqual(0, found_in)
 305         self.assertNotEqual(0, found_not_in)
 306
 307         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 308         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 309
 310
 311 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 312     class Test(unittest.TestCase):
 313         def runTest(self):
 314             d1 = llfuse.listdir(mounttmp)
 315             self.assertEqual(["file1.txt"], d1)
 316             with open(os.path.join(mounttmp, "file1.txt")) as f:
 317                 self.assertEqual("blub", f.read())
 318     Test().runTest()
 319
 320 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 321     class Test(unittest.TestCase):
 322         def runTest(self):
 323             d1 = llfuse.listdir(mounttmp)
 324             self.assertEqual(["file1.txt"], d1)
 325             with open(os.path.join(mounttmp, "file1.txt")) as f:
 326                 self.assertEqual("plnp", f.read())
 327     Test().runTest()
 328
 329 class FuseModifyFileTest(MountTestBase):
 330     def runTest(self):
 331         collection = arvados.collection.Collection(api_client=self.api)
 332         with collection.open("file1.txt", "w") as f:
 333             f.write("blub")
 334
 335         collection.save_new()
 336
 337         m = self.make_mount(fuse.CollectionDirectory)
 338         with llfuse.lock:
 339             m.new_collection(collection.api_response(), collection)
 340
 341         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 342
 343         with collection.open("file1.txt", "w") as f:
 344             f.write("plnp")
 345
 346         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 347
 348
 349 class FuseAddFileToCollectionTest(MountTestBase):
 350     def runTest(self):
 351         collection = arvados.collection.Collection(api_client=self.api)
 352         with collection.open("file1.txt", "w") as f:
 353             f.write("blub")
 354
 355         collection.save_new()
 356
 357         m = self.make_mount(fuse.CollectionDirectory)
 358         with llfuse.lock:
 359             m.new_collection(collection.api_response(), collection)
 360
 361         d1 = llfuse.listdir(self.mounttmp)
 362         self.assertEqual(["file1.txt"], d1)
 363
 364         with collection.open("file2.txt", "w") as f:
 365             f.write("plnp")
 366
 367         d1 = llfuse.listdir(self.mounttmp)
 368         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 369
 370
 371 class FuseRemoveFileFromCollectionTest(MountTestBase):
 372     def runTest(self):
 373         collection = arvados.collection.Collection(api_client=self.api)
 374         with collection.open("file1.txt", "w") as f:
 375             f.write("blub")
 376
 377         with collection.open("file2.txt", "w") as f:
 378             f.write("plnp")
 379
 380         collection.save_new()
 381
 382         m = self.make_mount(fuse.CollectionDirectory)
 383         with llfuse.lock:
 384             m.new_collection(collection.api_response(), collection)
 385
 386         d1 = llfuse.listdir(self.mounttmp)
 387         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 388
 389         collection.remove("file2.txt")
 390
 391         d1 = llfuse.listdir(self.mounttmp)
 392         self.assertEqual(["file1.txt"], d1)
 393
 394
 395 def fuseCreateFileTestHelper(mounttmp):
 396     class Test(unittest.TestCase):
 397         def runTest(self):
 398             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 399                 pass
 400     Test().runTest()
 401
 402 class FuseCreateFileTest(MountTestBase):
 403     def runTest(self):
 404         collection = arvados.collection.Collection(api_client=self.api)
 405         collection.save_new()
 406
 407         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 408         self.assertEqual(collection2["manifest_text"], "")
 409
 410         collection.save_new()
 411
 412         m = self.make_mount(fuse.CollectionDirectory)
 413         with llfuse.lock:
 414             m.new_collection(collection.api_response(), collection)
 415         self.assertTrue(m.writable())
 416
 417         self.assertNotIn("file1.txt", collection)
 418
 419         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 420
 421         self.assertIn("file1.txt", collection)
 422
 423         d1 = llfuse.listdir(self.mounttmp)
 424         self.assertEqual(["file1.txt"], d1)
 425
 426         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 427         self.assertRegexpMatches(collection2["manifest_text"],
 428             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 429
 430
 431 def fuseWriteFileTestHelperWriteFile(mounttmp):
 432     class Test(unittest.TestCase):
 433         def runTest(self):
 434             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 435                 f.write("Hello world!")
 436     Test().runTest()
 437
 438 def fuseWriteFileTestHelperReadFile(mounttmp):
 439     class Test(unittest.TestCase):
 440         def runTest(self):
 441             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 442                 self.assertEqual(f.read(), "Hello world!")
 443     Test().runTest()
 444
 445 class FuseWriteFileTest(MountTestBase):
 446     def runTest(self):
 447         collection = arvados.collection.Collection(api_client=self.api)
 448         collection.save_new()
 449
 450         m = self.make_mount(fuse.CollectionDirectory)
 451         with llfuse.lock:
 452             m.new_collection(collection.api_response(), collection)
 453         self.assertTrue(m.writable())
 454
 455         self.assertNotIn("file1.txt", collection)
 456
 457         self.assertEqual(0, self.operations.write_counter.get())
 458         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 459         self.assertEqual(12, self.operations.write_counter.get())
 460
 461         with collection.open("file1.txt") as f:
 462             self.assertEqual(f.read(), "Hello world!")
 463
 464         self.assertEqual(0, self.operations.read_counter.get())
 465         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 466         self.assertEqual(12, self.operations.read_counter.get())
 467
 468         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 469         self.assertRegexpMatches(collection2["manifest_text"],
 470             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 471
 472
 473 def fuseUpdateFileTestHelper(mounttmp):
 474     class Test(unittest.TestCase):
 475         def runTest(self):
 476             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 477                 f.write("Hello world!")
 478
 479             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 480                 fr = f.read()
 481                 self.assertEqual(fr, "Hello world!")
 482                 f.seek(0)
 483                 f.write("Hola mundo!")
 484                 f.seek(0)
 485                 fr = f.read()
 486                 self.assertEqual(fr, "Hola mundo!!")
 487
 488             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 489                 self.assertEqual(f.read(), "Hola mundo!!")
 490
 491     Test().runTest()
 492
 493 class FuseUpdateFileTest(MountTestBase):
 494     def runTest(self):
 495         collection = arvados.collection.Collection(api_client=self.api)
 496         collection.save_new()
 497
 498         m = self.make_mount(fuse.CollectionDirectory)
 499         with llfuse.lock:
 500             m.new_collection(collection.api_response(), collection)
 501         self.assertTrue(m.writable())
 502
 503         # See note in MountTestBase.setUp
 504         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 505
 506         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 507         self.assertRegexpMatches(collection2["manifest_text"],
 508             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 509
 510
 511 def fuseMkdirTestHelper(mounttmp):
 512     class Test(unittest.TestCase):
 513         def runTest(self):
 514             with self.assertRaises(IOError):
 515                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 516                     f.write("Hello world!")
 517
 518             os.mkdir(os.path.join(mounttmp, "testdir"))
 519
 520             with self.assertRaises(OSError):
 521                 os.mkdir(os.path.join(mounttmp, "testdir"))
 522
 523             d1 = llfuse.listdir(mounttmp)
 524             self.assertEqual(["testdir"], d1)
 525
 526             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 527                 f.write("Hello world!")
 528
 529             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 530             self.assertEqual(["file1.txt"], d1)
 531
 532     Test().runTest()
 533
 534 class FuseMkdirTest(MountTestBase):
 535     def runTest(self):
 536         collection = arvados.collection.Collection(api_client=self.api)
 537         collection.save_new()
 538
 539         m = self.make_mount(fuse.CollectionDirectory)
 540         with llfuse.lock:
 541             m.new_collection(collection.api_response(), collection)
 542         self.assertTrue(m.writable())
 543
 544         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 545
 546         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 547         self.assertRegexpMatches(collection2["manifest_text"],
 548             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 549
 550
 551 def fuseRmTestHelperWriteFile(mounttmp):
 552     class Test(unittest.TestCase):
 553         def runTest(self):
 554             os.mkdir(os.path.join(mounttmp, "testdir"))
 555
 556             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 557                 f.write("Hello world!")
 558
 559     Test().runTest()
 560
 561 def fuseRmTestHelperDeleteFile(mounttmp):
 562     class Test(unittest.TestCase):
 563         def runTest(self):
 564             # Can't delete because it's not empty
 565             with self.assertRaises(OSError):
 566                 os.rmdir(os.path.join(mounttmp, "testdir"))
 567
 568             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 569             self.assertEqual(["file1.txt"], d1)
 570
 571             # Delete file
 572             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 573
 574             # Make sure it's empty
 575             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 576             self.assertEqual([], d1)
 577
 578             # Try to delete it again
 579             with self.assertRaises(OSError):
 580                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 581
 582     Test().runTest()
 583
 584 def fuseRmTestHelperRmdir(mounttmp):
 585     class Test(unittest.TestCase):
 586         def runTest(self):
 587             # Should be able to delete now that it is empty
 588             os.rmdir(os.path.join(mounttmp, "testdir"))
 589
 590             # Make sure it's empty
 591             d1 = llfuse.listdir(os.path.join(mounttmp))
 592             self.assertEqual([], d1)
 593
 594             # Try to delete it again
 595             with self.assertRaises(OSError):
 596                 os.rmdir(os.path.join(mounttmp, "testdir"))
 597
 598     Test().runTest()
 599
 600 class FuseRmTest(MountTestBase):
 601     def runTest(self):
 602         collection = arvados.collection.Collection(api_client=self.api)
 603         collection.save_new()
 604
 605         m = self.make_mount(fuse.CollectionDirectory)
 606         with llfuse.lock:
 607             m.new_collection(collection.api_response(), collection)
 608         self.assertTrue(m.writable())
 609
 610         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 611
 612         # Starting manifest
 613         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 614         self.assertRegexpMatches(collection2["manifest_text"],
 615             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 616         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 617
 618         # Empty directories are represented by an empty file named "."
 619         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 620         self.assertRegexpMatches(collection2["manifest_text"],
 621                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 622
 623         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 624
 625         # manifest should be empty now.
 626         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 627         self.assertEqual(collection2["manifest_text"], "")
 628
 629
 630 def fuseMvFileTestHelperWriteFile(mounttmp):
 631     class Test(unittest.TestCase):
 632         def runTest(self):
 633             os.mkdir(os.path.join(mounttmp, "testdir"))
 634
 635             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 636                 f.write("Hello world!")
 637
 638     Test().runTest()
 639
 640 def fuseMvFileTestHelperMoveFile(mounttmp):
 641     class Test(unittest.TestCase):
 642         def runTest(self):
 643             d1 = llfuse.listdir(os.path.join(mounttmp))
 644             self.assertEqual(["testdir"], d1)
 645             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 646             self.assertEqual(["file1.txt"], d1)
 647
 648             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 649
 650             d1 = llfuse.listdir(os.path.join(mounttmp))
 651             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 652             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 653             self.assertEqual([], d1)
 654
 655     Test().runTest()
 656
 657 class FuseMvFileTest(MountTestBase):
 658     def runTest(self):
 659         collection = arvados.collection.Collection(api_client=self.api)
 660         collection.save_new()
 661
 662         m = self.make_mount(fuse.CollectionDirectory)
 663         with llfuse.lock:
 664             m.new_collection(collection.api_response(), collection)
 665         self.assertTrue(m.writable())
 666
 667         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 668
 669         # Starting manifest
 670         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 671         self.assertRegexpMatches(collection2["manifest_text"],
 672             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 673
 674         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 675
 676         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 677         self.assertRegexpMatches(collection2["manifest_text"],
 678             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 679
 680
 681 def fuseRenameTestHelper(mounttmp):
 682     class Test(unittest.TestCase):
 683         def runTest(self):
 684             os.mkdir(os.path.join(mounttmp, "testdir"))
 685
 686             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 687                 f.write("Hello world!")
 688
 689     Test().runTest()
 690
 691 class FuseRenameTest(MountTestBase):
 692     def runTest(self):
 693         collection = arvados.collection.Collection(api_client=self.api)
 694         collection.save_new()
 695
 696         m = self.make_mount(fuse.CollectionDirectory)
 697         with llfuse.lock:
 698             m.new_collection(collection.api_response(), collection)
 699         self.assertTrue(m.writable())
 700
 701         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 702
 703         # Starting manifest
 704         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 705         self.assertRegexpMatches(collection2["manifest_text"],
 706             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 707
 708         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 709         self.assertEqual(["testdir"], d1)
 710         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 711         self.assertEqual(["file1.txt"], d1)
 712
 713         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 714
 715         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 716         self.assertEqual(["testdir2"], sorted(d1))
 717         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 718         self.assertEqual(["file1.txt"], d1)
 719
 720         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 721         self.assertRegexpMatches(collection2["manifest_text"],
 722             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 723
 724
 725 class FuseUpdateFromEventTest(MountTestBase):
 726     def runTest(self):
 727         collection = arvados.collection.Collection(api_client=self.api)
 728         collection.save_new()
 729
 730         m = self.make_mount(fuse.CollectionDirectory)
 731         with llfuse.lock:
 732             m.new_collection(collection.api_response(), collection)
 733
 734         self.operations.listen_for_events()
 735
 736         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 737         self.assertEqual([], sorted(d1))
 738
 739         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 740             with collection2.open("file1.txt", "w") as f:
 741                 f.write("foo")
 742
 743         for attempt in AssertWithTimeout(10):
 744             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 745
 746
 747 class FuseDeleteProjectEventTest(MountTestBase):
 748     def runTest(self):
 749
 750         aproject = self.api.groups().create(body={
 751             "name": "aproject",
 752             "group_class": "project"
 753         }).execute()
 754
 755         bproject = self.api.groups().create(body={
 756             "name": "bproject",
 757             "group_class": "project",
 758             "owner_uuid": aproject["uuid"]
 759         }).execute()
 760
 761         self.make_mount(fuse.ProjectDirectory,
 762                         project_object=self.api.users().current().execute())
 763
 764         self.operations.listen_for_events()
 765
 766         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 767         self.assertEqual(["bproject"], sorted(d1))
 768
 769         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 770
 771         for attempt in AssertWithTimeout(10):
 772             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 773
 774
 775 def fuseFileConflictTestHelper(mounttmp):
 776     class Test(unittest.TestCase):
 777         def runTest(self):
 778             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 779                 f.write("bar")
 780
 781             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 782             self.assertEqual(len(d1), 2)
 783
 784             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 785                 self.assertEqual(f.read(), "bar")
 786
 787             self.assertRegexpMatches(d1[1],
 788                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 789
 790             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 791                 self.assertEqual(f.read(), "foo")
 792
 793     Test().runTest()
 794
 795 class FuseFileConflictTest(MountTestBase):
 796     def runTest(self):
 797         collection = arvados.collection.Collection(api_client=self.api)
 798         collection.save_new()
 799
 800         m = self.make_mount(fuse.CollectionDirectory)
 801         with llfuse.lock:
 802             m.new_collection(collection.api_response(), collection)
 803
 804         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 805         self.assertEqual([], sorted(d1))
 806
 807         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 808             with collection2.open("file1.txt", "w") as f:
 809                 f.write("foo")
 810
 811         # See note in MountTestBase.setUp
 812         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 813
 814
 815 def fuseUnlinkOpenFileTest(mounttmp):
 816     class Test(unittest.TestCase):
 817         def runTest(self):
 818             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 819                 f.write("foo")
 820
 821                 d1 = llfuse.listdir(os.path.join(mounttmp))
 822                 self.assertEqual(["file1.txt"], sorted(d1))
 823
 824                 os.remove(os.path.join(mounttmp, "file1.txt"))
 825
 826                 d1 = llfuse.listdir(os.path.join(mounttmp))
 827                 self.assertEqual([], sorted(d1))
 828
 829                 f.seek(0)
 830                 self.assertEqual(f.read(), "foo")
 831                 f.write("bar")
 832
 833                 f.seek(0)
 834                 self.assertEqual(f.read(), "foobar")
 835
 836     Test().runTest()
 837
 838 class FuseUnlinkOpenFileTest(MountTestBase):
 839     def runTest(self):
 840         collection = arvados.collection.Collection(api_client=self.api)
 841         collection.save_new()
 842
 843         m = self.make_mount(fuse.CollectionDirectory)
 844         with llfuse.lock:
 845             m.new_collection(collection.api_response(), collection)
 846
 847         # See note in MountTestBase.setUp
 848         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 849
 850         self.assertEqual(collection.manifest_text(), "")
 851
 852
 853 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 854     class Test(unittest.TestCase):
 855         def runTest(self):
 856             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 857                 f.write("Hello world!")
 858
 859             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 860             self.assertEqual(["file1.txt"], sorted(d1))
 861             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 862             self.assertEqual([], sorted(d1))
 863
 864     Test().runTest()
 865
 866 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 867     class Test(unittest.TestCase):
 868         def runTest(self):
 869             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 870
 871             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 872             self.assertEqual([], sorted(d1))
 873             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 874             self.assertEqual(["file2.txt"], sorted(d1))
 875
 876     Test().runTest()
 877
 878 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 879     def runTest(self):
 880         collection1 = arvados.collection.Collection(api_client=self.api)
 881         collection1.save_new()
 882
 883         collection2 = arvados.collection.Collection(api_client=self.api)
 884         collection2.save_new()
 885
 886         m = self.make_mount(fuse.MagicDirectory)
 887
 888         # See note in MountTestBase.setUp
 889         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 890                                                   collection1.manifest_locator(),
 891                                                   collection2.manifest_locator()))
 892
 893         collection1.update()
 894         collection2.update()
 895
 896         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 897         self.assertEqual(collection2.manifest_text(), "")
 898
 899         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 900                                                   collection1.manifest_locator(),
 901                                                   collection2.manifest_locator()))
 902
 903         collection1.update()
 904         collection2.update()
 905
 906         self.assertEqual(collection1.manifest_text(), "")
 907         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 908
 909         collection1.stop_threads()
 910         collection2.stop_threads()
 911
 912
 913 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 914     class Test(unittest.TestCase):
 915         def runTest(self):
 916             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 917             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 918                 f.write("Hello world!")
 919
 920             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 921             self.assertEqual(["testdir"], sorted(d1))
 922             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 923             self.assertEqual(["file1.txt"], sorted(d1))
 924
 925             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 926             self.assertEqual([], sorted(d1))
 927
 928     Test().runTest()
 929
 930
 931 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 932     class Test(unittest.TestCase):
 933         def runTest(self):
 934             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 935
 936             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 937             self.assertEqual([], sorted(d1))
 938
 939             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 940             self.assertEqual(["testdir2"], sorted(d1))
 941             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 942             self.assertEqual(["file1.txt"], sorted(d1))
 943
 944             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 945                 self.assertEqual(f.read(), "Hello world!")
 946
 947     Test().runTest()
 948
 949 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 950     def runTest(self):
 951         collection1 = arvados.collection.Collection(api_client=self.api)
 952         collection1.save_new()
 953
 954         collection2 = arvados.collection.Collection(api_client=self.api)
 955         collection2.save_new()
 956
 957         m = self.make_mount(fuse.MagicDirectory)
 958
 959         # See note in MountTestBase.setUp
 960         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 961                                                   collection1.manifest_locator(),
 962                                                   collection2.manifest_locator()))
 963
 964         collection1.update()
 965         collection2.update()
 966
 967         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 968         self.assertEqual(collection2.manifest_text(), "")
 969
 970         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 971                                                   collection1.manifest_locator(),
 972                                                   collection2.manifest_locator()))
 973
 974         collection1.update()
 975         collection2.update()
 976
 977         self.assertEqual(collection1.manifest_text(), "")
 978         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 979
 980         collection1.stop_threads()
 981         collection2.stop_threads()
 982
 983 def fuseProjectMkdirTestHelper1(mounttmp):
 984     class Test(unittest.TestCase):
 985         def runTest(self):
 986             os.mkdir(os.path.join(mounttmp, "testcollection"))
 987             with self.assertRaises(OSError):
 988                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 989     Test().runTest()
 990
 991 def fuseProjectMkdirTestHelper2(mounttmp):
 992     class Test(unittest.TestCase):
 993         def runTest(self):
 994             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 995                 f.write("Hello world!")
 996             with self.assertRaises(OSError):
 997                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 998             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 999             with self.assertRaises(OSError):
1000                 os.remove(os.path.join(mounttmp, "testcollection"))
1001             os.rmdir(os.path.join(mounttmp, "testcollection"))
1002     Test().runTest()
1003
1004 class FuseProjectMkdirRmdirTest(MountTestBase):
1005     def runTest(self):
1006         self.make_mount(fuse.ProjectDirectory,
1007                         project_object=self.api.users().current().execute())
1008
1009         d1 = llfuse.listdir(self.mounttmp)
1010         self.assertNotIn('testcollection', d1)
1011
1012         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1013
1014         d1 = llfuse.listdir(self.mounttmp)
1015         self.assertIn('testcollection', d1)
1016
1017         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1018
1019         d1 = llfuse.listdir(self.mounttmp)
1020         self.assertNotIn('testcollection', d1)
1021
1022
1023 def fuseProjectMvTestHelper1(mounttmp):
1024     class Test(unittest.TestCase):
1025         def runTest(self):
1026             d1 = llfuse.listdir(mounttmp)
1027             self.assertNotIn('testcollection', d1)
1028
1029             os.mkdir(os.path.join(mounttmp, "testcollection"))
1030
1031             d1 = llfuse.listdir(mounttmp)
1032             self.assertIn('testcollection', d1)
1033
1034             with self.assertRaises(OSError):
1035                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1036
1037             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1038
1039             d1 = llfuse.listdir(mounttmp)
1040             self.assertNotIn('testcollection', d1)
1041
1042             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1043             self.assertIn('testcollection', d1)
1044
1045     Test().runTest()
1046
1047 class FuseProjectMvTest(MountTestBase):
1048     def runTest(self):
1049         self.make_mount(fuse.ProjectDirectory,
1050                         project_object=self.api.users().current().execute())
1051
1052         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1053
1054
1055 def fuseFsyncTestHelper(mounttmp, k):
1056     class Test(unittest.TestCase):
1057         def runTest(self):
1058             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1059             os.fsync(fd)
1060             os.close(fd)
1061
1062     Test().runTest()
1063
1064 class FuseFsyncTest(FuseMagicTest):
1065     def runTest(self):
1066         self.make_mount(fuse.MagicDirectory)
1067         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1068
1069
1070 class MagicDirApiError(FuseMagicTest):
1071     def setUp(self):
1072         api = mock.MagicMock()
1073         super(MagicDirApiError, self).setUp(api=api)
1074         api.collections().get().execute.side_effect = iter([
1075             Exception('API fail'),
1076             {
1077                 "manifest_text": self.test_manifest,
1078                 "portable_data_hash": self.test_manifest_pdh,
1079             },
1080         ])
1081         api.keep.get.side_effect = Exception('Keep fail')
1082
1083     def runTest(self):
1084         self.make_mount(fuse.MagicDirectory)
1085
1086         self.operations.inodes.inode_cache.cap = 1
1087         self.operations.inodes.inode_cache.min_entries = 2
1088
1089         with self.assertRaises(OSError):
1090             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1091
1092         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1093
1094
1095 class FuseUnitTest(unittest.TestCase):
1096     def test_sanitize_filename(self):
1097         acceptable = [
1098             "foo.txt",
1099             ".foo",
1100             "..foo",
1101             "...",
1102             "foo...",
1103             "foo..",
1104             "foo.",
1105             "-",
1106             "\x01\x02\x03",
1107             ]
1108         unacceptable = [
1109             "f\00",
1110             "\00\00",
1111             "/foo",
1112             "foo/",
1113             "//",
1114             ]
1115         for f in acceptable:
1116             self.assertEqual(f, fuse.sanitize_filename(f))
1117         for f in unacceptable:
1118             self.assertNotEqual(f, fuse.sanitize_filename(f))
1119             # The sanitized filename should be the same length, though.
1120             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1121         # Special cases
1122         self.assertEqual("_", fuse.sanitize_filename(""))
1123         self.assertEqual("_", fuse.sanitize_filename("."))
1124         self.assertEqual("__", fuse.sanitize_filename(".."))
1125
1126
1127 class FuseMagicTestPDHOnly(MountTestBase):
1128     def setUp(self, api=None):
1129         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1130
1131         cw = arvados.CollectionWriter()
1132
1133         cw.start_new_file('thing1.txt')
1134         cw.write("data 1")
1135
1136         self.testcollection = cw.finish()
1137         self.test_manifest = cw.manifest_text()
1138         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1139         self.testcollectionuuid = str(created['uuid'])
1140
1141     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1142         if skip_pdh_only is True:
1143             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1144         else:
1145             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1146
1147         mount_ls = llfuse.listdir(self.mounttmp)
1148         self.assertIn('README', mount_ls)
1149         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1150                              arvados.util.uuid_pattern.match(fn)
1151                              for fn in mount_ls),
1152                          "new FUSE MagicDirectory lists Collection")
1153
1154         # look up using pdh should succeed in all cases
1155         self.assertDirContents(self.testcollection, ['thing1.txt'])
1156         self.assertDirContents(os.path.join('by_id', self.testcollection),
1157                                ['thing1.txt'])
1158         mount_ls = llfuse.listdir(self.mounttmp)
1159         self.assertIn('README', mount_ls)
1160         self.assertIn(self.testcollection, mount_ls)
1161         self.assertIn(self.testcollection,
1162                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1163
1164         files = {}
1165         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1166
1167         for k, v in files.items():
1168             with open(os.path.join(self.mounttmp, k)) as f:
1169                 self.assertEqual(v, f.read())
1170
1171         # look up using uuid should fail when pdh_only is set
1172         if pdh_only is True:
1173             with self.assertRaises(OSError):
1174                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1175                                ['thing1.txt'])
1176         else:
1177             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1178                                ['thing1.txt'])
1179
1180     def test_with_pdh_only_true(self):
1181         self.verify_pdh_only(pdh_only=True)
1182
1183     def test_with_pdh_only_false(self):
1184         self.verify_pdh_only(pdh_only=False)
1185
1186     def test_with_default_by_id(self):
1187         self.verify_pdh_only(skip_pdh_only=True)