services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 import json
   6 import llfuse
   7 import logging
   8 import mock
   9 import os
  10 import subprocess
  11 import time
  12 import unittest
  13
  14 import arvados
  15 import arvados_fuse as fuse
  16 import run_test_server
  17
  18 from mount_test_base import MountTestBase
  19
  20 logger = logging.getLogger('arvados.arv-mount')
  21
  22
  23 class AssertWithTimeout(object):
  24     """Allow some time for an assertion to pass."""
  25
  26     def __init__(self, timeout=0):
  27         self.timeout = timeout
  28
  29     def __iter__(self):
  30         self.deadline = time.time() + self.timeout
  31         self.done = False
  32         return self
  33
  34     def next(self):
  35         if self.done:
  36             raise StopIteration
  37         return self.attempt
  38
  39     def attempt(self, fn, *args, **kwargs):
  40         try:
  41             fn(*args, **kwargs)
  42         except AssertionError:
  43             if time.time() > self.deadline:
  44                 raise
  45             time.sleep(0.1)
  46         else:
  47             self.done = True
  48
  49
  50 class FuseMountTest(MountTestBase):
  51     def setUp(self):
  52         super(FuseMountTest, self).setUp()
  53
  54         cw = arvados.CollectionWriter()
  55
  56         cw.start_new_file('thing1.txt')
  57         cw.write("data 1")
  58         cw.start_new_file('thing2.txt')
  59         cw.write("data 2")
  60
  61         cw.start_new_stream('dir1')
  62         cw.start_new_file('thing3.txt')
  63         cw.write("data 3")
  64         cw.start_new_file('thing4.txt')
  65         cw.write("data 4")
  66
  67         cw.start_new_stream('dir2')
  68         cw.start_new_file('thing5.txt')
  69         cw.write("data 5")
  70         cw.start_new_file('thing6.txt')
  71         cw.write("data 6")
  72
  73         cw.start_new_stream('dir2/dir3')
  74         cw.start_new_file('thing7.txt')
  75         cw.write("data 7")
  76
  77         cw.start_new_file('thing8.txt')
  78         cw.write("data 8")
  79
  80         cw.start_new_stream('edgecases')
  81         for f in ":/.../-/*/\x01\\/ ".split("/"):
  82             cw.start_new_file(f)
  83             cw.write('x')
  84
  85         for f in ":/.../-/*/\x01\\/ ".split("/"):
  86             cw.start_new_stream('edgecases/dirs/' + f)
  87             cw.start_new_file('x/x')
  88             cw.write('x')
  89
  90         self.testcollection = cw.finish()
  91         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  92
  93     def runTest(self):
  94         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  95
  96         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
  97                                       'edgecases', 'dir1', 'dir2'])
  98         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
  99         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 100         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 101         self.assertDirContents('edgecases',
 102                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
 103         self.assertDirContents('edgecases/dirs',
 104                                ":/.../-/*/\x01\\/ ".split("/"))
 105
 106         files = {'thing1.txt': 'data 1',
 107                  'thing2.txt': 'data 2',
 108                  'dir1/thing3.txt': 'data 3',
 109                  'dir1/thing4.txt': 'data 4',
 110                  'dir2/thing5.txt': 'data 5',
 111                  'dir2/thing6.txt': 'data 6',
 112                  'dir2/dir3/thing7.txt': 'data 7',
 113                  'dir2/dir3/thing8.txt': 'data 8'}
 114
 115         for k, v in files.items():
 116             with open(os.path.join(self.mounttmp, k)) as f:
 117                 self.assertEqual(v, f.read())
 118
 119
 120 class FuseMagicTest(MountTestBase):
 121     def setUp(self, api=None):
 122         super(FuseMagicTest, self).setUp(api=api)
 123
 124         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 125         self.non_project_group = run_test_server.fixture('groups')['public']['uuid']
 126         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 127
 128         cw = arvados.CollectionWriter()
 129
 130         cw.start_new_file('thing1.txt')
 131         cw.write("data 1")
 132
 133         self.testcollection = cw.finish()
 134         self.test_manifest = cw.manifest_text()
 135         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 136         self.test_manifest_pdh = coll['portable_data_hash']
 137
 138     def runTest(self):
 139         self.make_mount(fuse.MagicDirectory)
 140
 141         mount_ls = llfuse.listdir(self.mounttmp)
 142         self.assertIn('README', mount_ls)
 143         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 144                              arvados.util.uuid_pattern.match(fn)
 145                              for fn in mount_ls),
 146                          "new FUSE MagicDirectory has no collections or projects")
 147         self.assertDirContents(self.testcollection, ['thing1.txt'])
 148         self.assertDirContents(os.path.join('by_id', self.testcollection),
 149                                ['thing1.txt'])
 150         self.assertIn(self.collection_in_test_project,
 151                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 152         self.assertIn(self.collection_in_test_project,
 153                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 154
 155         mount_ls = llfuse.listdir(self.mounttmp)
 156         self.assertIn('README', mount_ls)
 157         self.assertIn(self.testcollection, mount_ls)
 158         self.assertIn(self.testcollection,
 159                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 160         self.assertIn(self.test_project, mount_ls)
 161         self.assertIn(self.test_project,
 162                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 163
 164         with self.assertRaises(OSError):
 165             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 166
 167         files = {}
 168         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 169
 170         for k, v in files.items():
 171             with open(os.path.join(self.mounttmp, k)) as f:
 172                 self.assertEqual(v, f.read())
 173
 174
 175 class FuseTagsTest(MountTestBase):
 176     def runTest(self):
 177         self.make_mount(fuse.TagsDirectory)
 178
 179         d1 = llfuse.listdir(self.mounttmp)
 180         d1.sort()
 181         self.assertEqual(['foo_tag'], d1)
 182
 183         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 184         d2.sort()
 185         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 186
 187         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 188         d3.sort()
 189         self.assertEqual(['foo'], d3)
 190
 191
 192 class FuseTagsUpdateTest(MountTestBase):
 193     def tag_collection(self, coll_uuid, tag_name):
 194         return self.api.links().create(
 195             body={'link': {'head_uuid': coll_uuid,
 196                            'link_class': 'tag',
 197                            'name': tag_name,
 198         }}).execute()
 199
 200     def runTest(self):
 201         self.make_mount(fuse.TagsDirectory, poll_time=1)
 202
 203         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 204
 205         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 206         self.tag_collection(bar_uuid, 'fuse_test_tag')
 207         for attempt in AssertWithTimeout(10):
 208             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 209         self.assertDirContents('fuse_test_tag', [bar_uuid])
 210
 211         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 212         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 213         for attempt in AssertWithTimeout(10):
 214             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 215
 216         self.api.links().delete(uuid=l['uuid']).execute()
 217         for attempt in AssertWithTimeout(10):
 218             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 219
 220
 221 def fuseSharedTestHelper(mounttmp):
 222     class Test(unittest.TestCase):
 223         def runTest(self):
 224             # Double check that we can open and read objects in this folder as a file,
 225             # and that its contents are what we expect.
 226             baz_path = os.path.join(
 227                 mounttmp,
 228                 'FUSE User',
 229                 'FUSE Test Project',
 230                 'collection in FUSE project',
 231                 'baz')
 232             with open(baz_path) as f:
 233                 self.assertEqual("baz", f.read())
 234
 235             # check mtime on collection
 236             st = os.stat(baz_path)
 237             try:
 238                 mtime = st.st_mtime_ns / 1000000000
 239             except AttributeError:
 240                 mtime = st.st_mtime
 241             self.assertEqual(mtime, 1391448174)
 242
 243             # shared_dirs is a list of the directories exposed
 244             # by fuse.SharedDirectory (i.e. any object visible
 245             # to the current user)
 246             shared_dirs = llfuse.listdir(mounttmp)
 247             shared_dirs.sort()
 248             self.assertIn('FUSE User', shared_dirs)
 249
 250             # fuse_user_objs is a list of the objects owned by the FUSE
 251             # test user (which present as files in the 'FUSE User'
 252             # directory)
 253             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 254             fuse_user_objs.sort()
 255             self.assertEqual(['FUSE Test Project',                    # project owned by user
 256                               'collection #1 owned by FUSE',          # collection owned by user
 257                               'collection #2 owned by FUSE'          # collection owned by user
 258                           ], fuse_user_objs)
 259
 260             # test_proj_files is a list of the files in the FUSE Test Project.
 261             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 262             test_proj_files.sort()
 263             self.assertEqual(['collection in FUSE project'
 264                           ], test_proj_files)
 265
 266
 267     Test().runTest()
 268
 269 class FuseSharedTest(MountTestBase):
 270     def runTest(self):
 271         self.make_mount(fuse.SharedDirectory,
 272                         exclude=self.api.users().current().execute()['uuid'])
 273         keep = arvados.keep.KeepClient()
 274         keep.put("baz")
 275
 276         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 277
 278
 279 class FuseHomeTest(MountTestBase):
 280     def runTest(self):
 281         self.make_mount(fuse.ProjectDirectory,
 282                         project_object=self.api.users().current().execute())
 283
 284         d1 = llfuse.listdir(self.mounttmp)
 285         self.assertIn('Unrestricted public data', d1)
 286
 287         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 288         public_project = run_test_server.fixture('groups')[
 289             'anonymously_accessible_project']
 290         found_in = 0
 291         found_not_in = 0
 292         for name, item in run_test_server.fixture('collections').iteritems():
 293             if 'name' not in item:
 294                 pass
 295             elif item['owner_uuid'] == public_project['uuid']:
 296                 self.assertIn(item['name'], d2)
 297                 found_in += 1
 298             else:
 299                 # Artificial assumption here: there is no public
 300                 # collection fixture with the same name as a
 301                 # non-public collection.
 302                 self.assertNotIn(item['name'], d2)
 303                 found_not_in += 1
 304         self.assertNotEqual(0, found_in)
 305         self.assertNotEqual(0, found_not_in)
 306
 307         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 308         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 309
 310
 311 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 312     class Test(unittest.TestCase):
 313         def runTest(self):
 314             d1 = llfuse.listdir(mounttmp)
 315             self.assertEqual(["file1.txt"], d1)
 316             with open(os.path.join(mounttmp, "file1.txt")) as f:
 317                 self.assertEqual("blub", f.read())
 318     Test().runTest()
 319
 320 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 321     class Test(unittest.TestCase):
 322         def runTest(self):
 323             d1 = llfuse.listdir(mounttmp)
 324             self.assertEqual(["file1.txt"], d1)
 325             with open(os.path.join(mounttmp, "file1.txt")) as f:
 326                 self.assertEqual("plnp", f.read())
 327     Test().runTest()
 328
 329 class FuseModifyFileTest(MountTestBase):
 330     def runTest(self):
 331         collection = arvados.collection.Collection(api_client=self.api)
 332         with collection.open("file1.txt", "w") as f:
 333             f.write("blub")
 334
 335         collection.save_new()
 336
 337         m = self.make_mount(fuse.CollectionDirectory)
 338         with llfuse.lock:
 339             m.new_collection(collection.api_response(), collection)
 340
 341         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 342
 343         with collection.open("file1.txt", "w") as f:
 344             f.write("plnp")
 345
 346         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 347
 348
 349 class FuseAddFileToCollectionTest(MountTestBase):
 350     def runTest(self):
 351         collection = arvados.collection.Collection(api_client=self.api)
 352         with collection.open("file1.txt", "w") as f:
 353             f.write("blub")
 354
 355         collection.save_new()
 356
 357         m = self.make_mount(fuse.CollectionDirectory)
 358         with llfuse.lock:
 359             m.new_collection(collection.api_response(), collection)
 360
 361         d1 = llfuse.listdir(self.mounttmp)
 362         self.assertEqual(["file1.txt"], d1)
 363
 364         with collection.open("file2.txt", "w") as f:
 365             f.write("plnp")
 366
 367         d1 = llfuse.listdir(self.mounttmp)
 368         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 369
 370
 371 class FuseRemoveFileFromCollectionTest(MountTestBase):
 372     def runTest(self):
 373         collection = arvados.collection.Collection(api_client=self.api)
 374         with collection.open("file1.txt", "w") as f:
 375             f.write("blub")
 376
 377         with collection.open("file2.txt", "w") as f:
 378             f.write("plnp")
 379
 380         collection.save_new()
 381
 382         m = self.make_mount(fuse.CollectionDirectory)
 383         with llfuse.lock:
 384             m.new_collection(collection.api_response(), collection)
 385
 386         d1 = llfuse.listdir(self.mounttmp)
 387         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 388
 389         collection.remove("file2.txt")
 390
 391         d1 = llfuse.listdir(self.mounttmp)
 392         self.assertEqual(["file1.txt"], d1)
 393
 394
 395 def fuseCreateFileTestHelper(mounttmp):
 396     class Test(unittest.TestCase):
 397         def runTest(self):
 398             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 399                 pass
 400     Test().runTest()
 401
 402 class FuseCreateFileTest(MountTestBase):
 403     def runTest(self):
 404         collection = arvados.collection.Collection(api_client=self.api)
 405         collection.save_new()
 406
 407         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 408         self.assertEqual(collection2["manifest_text"], "")
 409
 410         collection.save_new()
 411
 412         m = self.make_mount(fuse.CollectionDirectory)
 413         with llfuse.lock:
 414             m.new_collection(collection.api_response(), collection)
 415         self.assertTrue(m.writable())
 416
 417         self.assertNotIn("file1.txt", collection)
 418
 419         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 420
 421         self.assertIn("file1.txt", collection)
 422
 423         d1 = llfuse.listdir(self.mounttmp)
 424         self.assertEqual(["file1.txt"], d1)
 425
 426         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 427         self.assertRegexpMatches(collection2["manifest_text"],
 428             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 429
 430
 431 def fuseWriteFileTestHelperWriteFile(mounttmp):
 432     class Test(unittest.TestCase):
 433         def runTest(self):
 434             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 435                 f.write("Hello world!")
 436     Test().runTest()
 437
 438 def fuseWriteFileTestHelperReadFile(mounttmp):
 439     class Test(unittest.TestCase):
 440         def runTest(self):
 441             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 442                 self.assertEqual(f.read(), "Hello world!")
 443     Test().runTest()
 444
 445 class FuseWriteFileTest(MountTestBase):
 446     def runTest(self):
 447         collection = arvados.collection.Collection(api_client=self.api)
 448         collection.save_new()
 449
 450         m = self.make_mount(fuse.CollectionDirectory)
 451         with llfuse.lock:
 452             m.new_collection(collection.api_response(), collection)
 453         self.assertTrue(m.writable())
 454
 455         self.assertNotIn("file1.txt", collection)
 456
 457         self.assertEqual(0, self.operations.write_counter.get())
 458         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 459         self.assertEqual(12, self.operations.write_counter.get())
 460
 461         with collection.open("file1.txt") as f:
 462             self.assertEqual(f.read(), "Hello world!")
 463
 464         self.assertEqual(0, self.operations.read_counter.get())
 465         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 466         self.assertEqual(12, self.operations.read_counter.get())
 467
 468         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 469         self.assertRegexpMatches(collection2["manifest_text"],
 470             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 471
 472
 473 def fuseUpdateFileTestHelper(mounttmp):
 474     class Test(unittest.TestCase):
 475         def runTest(self):
 476             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 477                 f.write("Hello world!")
 478
 479             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 480                 fr = f.read()
 481                 self.assertEqual(fr, "Hello world!")
 482                 f.seek(0)
 483                 f.write("Hola mundo!")
 484                 f.seek(0)
 485                 fr = f.read()
 486                 self.assertEqual(fr, "Hola mundo!!")
 487
 488             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 489                 self.assertEqual(f.read(), "Hola mundo!!")
 490
 491     Test().runTest()
 492
 493 class FuseUpdateFileTest(MountTestBase):
 494     def runTest(self):
 495         collection = arvados.collection.Collection(api_client=self.api)
 496         collection.save_new()
 497
 498         m = self.make_mount(fuse.CollectionDirectory)
 499         with llfuse.lock:
 500             m.new_collection(collection.api_response(), collection)
 501         self.assertTrue(m.writable())
 502
 503         # See note in MountTestBase.setUp
 504         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 505
 506         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 507         self.assertRegexpMatches(collection2["manifest_text"],
 508             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 509
 510
 511 def fuseMkdirTestHelper(mounttmp):
 512     class Test(unittest.TestCase):
 513         def runTest(self):
 514             with self.assertRaises(IOError):
 515                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 516                     f.write("Hello world!")
 517
 518             os.mkdir(os.path.join(mounttmp, "testdir"))
 519
 520             with self.assertRaises(OSError):
 521                 os.mkdir(os.path.join(mounttmp, "testdir"))
 522
 523             d1 = llfuse.listdir(mounttmp)
 524             self.assertEqual(["testdir"], d1)
 525
 526             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 527                 f.write("Hello world!")
 528
 529             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 530             self.assertEqual(["file1.txt"], d1)
 531
 532     Test().runTest()
 533
 534 class FuseMkdirTest(MountTestBase):
 535     def runTest(self):
 536         collection = arvados.collection.Collection(api_client=self.api)
 537         collection.save_new()
 538
 539         m = self.make_mount(fuse.CollectionDirectory)
 540         with llfuse.lock:
 541             m.new_collection(collection.api_response(), collection)
 542         self.assertTrue(m.writable())
 543
 544         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 545
 546         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 547         self.assertRegexpMatches(collection2["manifest_text"],
 548             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 549
 550
 551 def fuseRmTestHelperWriteFile(mounttmp):
 552     class Test(unittest.TestCase):
 553         def runTest(self):
 554             os.mkdir(os.path.join(mounttmp, "testdir"))
 555
 556             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 557                 f.write("Hello world!")
 558
 559     Test().runTest()
 560
 561 def fuseRmTestHelperDeleteFile(mounttmp):
 562     class Test(unittest.TestCase):
 563         def runTest(self):
 564             # Can't delete because it's not empty
 565             with self.assertRaises(OSError):
 566                 os.rmdir(os.path.join(mounttmp, "testdir"))
 567
 568             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 569             self.assertEqual(["file1.txt"], d1)
 570
 571             # Delete file
 572             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 573
 574             # Make sure it's empty
 575             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 576             self.assertEqual([], d1)
 577
 578             # Try to delete it again
 579             with self.assertRaises(OSError):
 580                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 581
 582     Test().runTest()
 583
 584 def fuseRmTestHelperRmdir(mounttmp):
 585     class Test(unittest.TestCase):
 586         def runTest(self):
 587             # Should be able to delete now that it is empty
 588             os.rmdir(os.path.join(mounttmp, "testdir"))
 589
 590             # Make sure it's empty
 591             d1 = llfuse.listdir(os.path.join(mounttmp))
 592             self.assertEqual([], d1)
 593
 594             # Try to delete it again
 595             with self.assertRaises(OSError):
 596                 os.rmdir(os.path.join(mounttmp, "testdir"))
 597
 598     Test().runTest()
 599
 600 class FuseRmTest(MountTestBase):
 601     def runTest(self):
 602         collection = arvados.collection.Collection(api_client=self.api)
 603         collection.save_new()
 604
 605         m = self.make_mount(fuse.CollectionDirectory)
 606         with llfuse.lock:
 607             m.new_collection(collection.api_response(), collection)
 608         self.assertTrue(m.writable())
 609
 610         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 611
 612         # Starting manifest
 613         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 614         self.assertRegexpMatches(collection2["manifest_text"],
 615             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 616         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 617
 618         # Can't have empty directories :-( so manifest will be empty.
 619         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 620         self.assertEqual(collection2["manifest_text"], "")
 621
 622         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 623
 624         # manifest should be empty now.
 625         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 626         self.assertEqual(collection2["manifest_text"], "")
 627
 628
 629 def fuseMvFileTestHelperWriteFile(mounttmp):
 630     class Test(unittest.TestCase):
 631         def runTest(self):
 632             os.mkdir(os.path.join(mounttmp, "testdir"))
 633
 634             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 635                 f.write("Hello world!")
 636
 637     Test().runTest()
 638
 639 def fuseMvFileTestHelperMoveFile(mounttmp):
 640     class Test(unittest.TestCase):
 641         def runTest(self):
 642             d1 = llfuse.listdir(os.path.join(mounttmp))
 643             self.assertEqual(["testdir"], d1)
 644             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 645             self.assertEqual(["file1.txt"], d1)
 646
 647             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 648
 649             d1 = llfuse.listdir(os.path.join(mounttmp))
 650             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 651             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 652             self.assertEqual([], d1)
 653
 654     Test().runTest()
 655
 656 class FuseMvFileTest(MountTestBase):
 657     def runTest(self):
 658         collection = arvados.collection.Collection(api_client=self.api)
 659         collection.save_new()
 660
 661         m = self.make_mount(fuse.CollectionDirectory)
 662         with llfuse.lock:
 663             m.new_collection(collection.api_response(), collection)
 664         self.assertTrue(m.writable())
 665
 666         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 667
 668         # Starting manifest
 669         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 670         self.assertRegexpMatches(collection2["manifest_text"],
 671             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 672
 673         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 674
 675         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 676         self.assertRegexpMatches(collection2["manifest_text"],
 677             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 678
 679
 680 def fuseRenameTestHelper(mounttmp):
 681     class Test(unittest.TestCase):
 682         def runTest(self):
 683             os.mkdir(os.path.join(mounttmp, "testdir"))
 684
 685             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 686                 f.write("Hello world!")
 687
 688     Test().runTest()
 689
 690 class FuseRenameTest(MountTestBase):
 691     def runTest(self):
 692         collection = arvados.collection.Collection(api_client=self.api)
 693         collection.save_new()
 694
 695         m = self.make_mount(fuse.CollectionDirectory)
 696         with llfuse.lock:
 697             m.new_collection(collection.api_response(), collection)
 698         self.assertTrue(m.writable())
 699
 700         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 701
 702         # Starting manifest
 703         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 704         self.assertRegexpMatches(collection2["manifest_text"],
 705             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 706
 707         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 708         self.assertEqual(["testdir"], d1)
 709         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 710         self.assertEqual(["file1.txt"], d1)
 711
 712         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 713
 714         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 715         self.assertEqual(["testdir2"], sorted(d1))
 716         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 717         self.assertEqual(["file1.txt"], d1)
 718
 719         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 720         self.assertRegexpMatches(collection2["manifest_text"],
 721             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 722
 723
 724 class FuseUpdateFromEventTest(MountTestBase):
 725     def runTest(self):
 726         collection = arvados.collection.Collection(api_client=self.api)
 727         collection.save_new()
 728
 729         m = self.make_mount(fuse.CollectionDirectory)
 730         with llfuse.lock:
 731             m.new_collection(collection.api_response(), collection)
 732
 733         self.operations.listen_for_events()
 734
 735         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 736         self.assertEqual([], sorted(d1))
 737
 738         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 739             with collection2.open("file1.txt", "w") as f:
 740                 f.write("foo")
 741
 742         for attempt in AssertWithTimeout(10):
 743             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 744
 745
 746 class FuseDeleteProjectEventTest(MountTestBase):
 747     def runTest(self):
 748
 749         aproject = self.api.groups().create(body={
 750             "name": "aproject",
 751             "group_class": "project"
 752         }).execute()
 753
 754         bproject = self.api.groups().create(body={
 755             "name": "bproject",
 756             "group_class": "project",
 757             "owner_uuid": aproject["uuid"]
 758         }).execute()
 759
 760         self.make_mount(fuse.ProjectDirectory,
 761                         project_object=self.api.users().current().execute())
 762
 763         self.operations.listen_for_events()
 764
 765         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 766         self.assertEqual(["bproject"], sorted(d1))
 767
 768         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 769
 770         for attempt in AssertWithTimeout(10):
 771             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 772
 773
 774 def fuseFileConflictTestHelper(mounttmp):
 775     class Test(unittest.TestCase):
 776         def runTest(self):
 777             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 778                 f.write("bar")
 779
 780             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 781             self.assertEqual(len(d1), 2)
 782
 783             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 784                 self.assertEqual(f.read(), "bar")
 785
 786             self.assertRegexpMatches(d1[1],
 787                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 788
 789             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 790                 self.assertEqual(f.read(), "foo")
 791
 792     Test().runTest()
 793
 794 class FuseFileConflictTest(MountTestBase):
 795     def runTest(self):
 796         collection = arvados.collection.Collection(api_client=self.api)
 797         collection.save_new()
 798
 799         m = self.make_mount(fuse.CollectionDirectory)
 800         with llfuse.lock:
 801             m.new_collection(collection.api_response(), collection)
 802
 803         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 804         self.assertEqual([], sorted(d1))
 805
 806         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 807             with collection2.open("file1.txt", "w") as f:
 808                 f.write("foo")
 809
 810         # See note in MountTestBase.setUp
 811         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 812
 813
 814 def fuseUnlinkOpenFileTest(mounttmp):
 815     class Test(unittest.TestCase):
 816         def runTest(self):
 817             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 818                 f.write("foo")
 819
 820                 d1 = llfuse.listdir(os.path.join(mounttmp))
 821                 self.assertEqual(["file1.txt"], sorted(d1))
 822
 823                 os.remove(os.path.join(mounttmp, "file1.txt"))
 824
 825                 d1 = llfuse.listdir(os.path.join(mounttmp))
 826                 self.assertEqual([], sorted(d1))
 827
 828                 f.seek(0)
 829                 self.assertEqual(f.read(), "foo")
 830                 f.write("bar")
 831
 832                 f.seek(0)
 833                 self.assertEqual(f.read(), "foobar")
 834
 835     Test().runTest()
 836
 837 class FuseUnlinkOpenFileTest(MountTestBase):
 838     def runTest(self):
 839         collection = arvados.collection.Collection(api_client=self.api)
 840         collection.save_new()
 841
 842         m = self.make_mount(fuse.CollectionDirectory)
 843         with llfuse.lock:
 844             m.new_collection(collection.api_response(), collection)
 845
 846         # See note in MountTestBase.setUp
 847         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 848
 849         self.assertEqual(collection.manifest_text(), "")
 850
 851
 852 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 853     class Test(unittest.TestCase):
 854         def runTest(self):
 855             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 856                 f.write("Hello world!")
 857
 858             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 859             self.assertEqual(["file1.txt"], sorted(d1))
 860             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 861             self.assertEqual([], sorted(d1))
 862
 863     Test().runTest()
 864
 865 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 866     class Test(unittest.TestCase):
 867         def runTest(self):
 868             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 869
 870             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 871             self.assertEqual([], sorted(d1))
 872             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 873             self.assertEqual(["file2.txt"], sorted(d1))
 874
 875     Test().runTest()
 876
 877 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 878     def runTest(self):
 879         collection1 = arvados.collection.Collection(api_client=self.api)
 880         collection1.save_new()
 881
 882         collection2 = arvados.collection.Collection(api_client=self.api)
 883         collection2.save_new()
 884
 885         m = self.make_mount(fuse.MagicDirectory)
 886
 887         # See note in MountTestBase.setUp
 888         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 889                                                   collection1.manifest_locator(),
 890                                                   collection2.manifest_locator()))
 891
 892         collection1.update()
 893         collection2.update()
 894
 895         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 896         self.assertEqual(collection2.manifest_text(), "")
 897
 898         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 899                                                   collection1.manifest_locator(),
 900                                                   collection2.manifest_locator()))
 901
 902         collection1.update()
 903         collection2.update()
 904
 905         self.assertEqual(collection1.manifest_text(), "")
 906         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 907
 908         collection1.stop_threads()
 909         collection2.stop_threads()
 910
 911
 912 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 913     class Test(unittest.TestCase):
 914         def runTest(self):
 915             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 916             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 917                 f.write("Hello world!")
 918
 919             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 920             self.assertEqual(["testdir"], sorted(d1))
 921             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 922             self.assertEqual(["file1.txt"], sorted(d1))
 923
 924             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 925             self.assertEqual([], sorted(d1))
 926
 927     Test().runTest()
 928
 929
 930 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 931     class Test(unittest.TestCase):
 932         def runTest(self):
 933             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 934
 935             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 936             self.assertEqual([], sorted(d1))
 937
 938             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 939             self.assertEqual(["testdir2"], sorted(d1))
 940             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 941             self.assertEqual(["file1.txt"], sorted(d1))
 942
 943             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 944                 self.assertEqual(f.read(), "Hello world!")
 945
 946     Test().runTest()
 947
 948 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 949     def runTest(self):
 950         collection1 = arvados.collection.Collection(api_client=self.api)
 951         collection1.save_new()
 952
 953         collection2 = arvados.collection.Collection(api_client=self.api)
 954         collection2.save_new()
 955
 956         m = self.make_mount(fuse.MagicDirectory)
 957
 958         # See note in MountTestBase.setUp
 959         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 960                                                   collection1.manifest_locator(),
 961                                                   collection2.manifest_locator()))
 962
 963         collection1.update()
 964         collection2.update()
 965
 966         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 967         self.assertEqual(collection2.manifest_text(), "")
 968
 969         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 970                                                   collection1.manifest_locator(),
 971                                                   collection2.manifest_locator()))
 972
 973         collection1.update()
 974         collection2.update()
 975
 976         self.assertEqual(collection1.manifest_text(), "")
 977         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 978
 979         collection1.stop_threads()
 980         collection2.stop_threads()
 981
 982 def fuseProjectMkdirTestHelper1(mounttmp):
 983     class Test(unittest.TestCase):
 984         def runTest(self):
 985             os.mkdir(os.path.join(mounttmp, "testcollection"))
 986             with self.assertRaises(OSError):
 987                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 988     Test().runTest()
 989
 990 def fuseProjectMkdirTestHelper2(mounttmp):
 991     class Test(unittest.TestCase):
 992         def runTest(self):
 993             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 994                 f.write("Hello world!")
 995             with self.assertRaises(OSError):
 996                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 997             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 998             with self.assertRaises(OSError):
 999                 os.remove(os.path.join(mounttmp, "testcollection"))
1000             os.rmdir(os.path.join(mounttmp, "testcollection"))
1001     Test().runTest()
1002
1003 class FuseProjectMkdirRmdirTest(MountTestBase):
1004     def runTest(self):
1005         self.make_mount(fuse.ProjectDirectory,
1006                         project_object=self.api.users().current().execute())
1007
1008         d1 = llfuse.listdir(self.mounttmp)
1009         self.assertNotIn('testcollection', d1)
1010
1011         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1012
1013         d1 = llfuse.listdir(self.mounttmp)
1014         self.assertIn('testcollection', d1)
1015
1016         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1017
1018         d1 = llfuse.listdir(self.mounttmp)
1019         self.assertNotIn('testcollection', d1)
1020
1021
1022 def fuseProjectMvTestHelper1(mounttmp):
1023     class Test(unittest.TestCase):
1024         def runTest(self):
1025             d1 = llfuse.listdir(mounttmp)
1026             self.assertNotIn('testcollection', d1)
1027
1028             os.mkdir(os.path.join(mounttmp, "testcollection"))
1029
1030             d1 = llfuse.listdir(mounttmp)
1031             self.assertIn('testcollection', d1)
1032
1033             with self.assertRaises(OSError):
1034                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1035
1036             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1037
1038             d1 = llfuse.listdir(mounttmp)
1039             self.assertNotIn('testcollection', d1)
1040
1041             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1042             self.assertIn('testcollection', d1)
1043
1044     Test().runTest()
1045
1046 class FuseProjectMvTest(MountTestBase):
1047     def runTest(self):
1048         self.make_mount(fuse.ProjectDirectory,
1049                         project_object=self.api.users().current().execute())
1050
1051         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1052
1053
1054 def fuseFsyncTestHelper(mounttmp, k):
1055     class Test(unittest.TestCase):
1056         def runTest(self):
1057             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1058             os.fsync(fd)
1059             os.close(fd)
1060
1061     Test().runTest()
1062
1063 class FuseFsyncTest(FuseMagicTest):
1064     def runTest(self):
1065         self.make_mount(fuse.MagicDirectory)
1066         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1067
1068
1069 class MagicDirApiError(FuseMagicTest):
1070     def setUp(self):
1071         api = mock.MagicMock()
1072         super(MagicDirApiError, self).setUp(api=api)
1073         api.collections().get().execute.side_effect = iter([
1074             Exception('API fail'),
1075             {
1076                 "manifest_text": self.test_manifest,
1077                 "portable_data_hash": self.test_manifest_pdh,
1078             },
1079         ])
1080         api.keep.get.side_effect = Exception('Keep fail')
1081
1082     def runTest(self):
1083         self.make_mount(fuse.MagicDirectory)
1084
1085         self.operations.inodes.inode_cache.cap = 1
1086         self.operations.inodes.inode_cache.min_entries = 2
1087
1088         with self.assertRaises(OSError):
1089             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1090
1091         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1092
1093
1094 class FuseUnitTest(unittest.TestCase):
1095     def test_sanitize_filename(self):
1096         acceptable = [
1097             "foo.txt",
1098             ".foo",
1099             "..foo",
1100             "...",
1101             "foo...",
1102             "foo..",
1103             "foo.",
1104             "-",
1105             "\x01\x02\x03",
1106             ]
1107         unacceptable = [
1108             "f\00",
1109             "\00\00",
1110             "/foo",
1111             "foo/",
1112             "//",
1113             ]
1114         for f in acceptable:
1115             self.assertEqual(f, fuse.sanitize_filename(f))
1116         for f in unacceptable:
1117             self.assertNotEqual(f, fuse.sanitize_filename(f))
1118             # The sanitized filename should be the same length, though.
1119             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1120         # Special cases
1121         self.assertEqual("_", fuse.sanitize_filename(""))
1122         self.assertEqual("_", fuse.sanitize_filename("."))
1123         self.assertEqual("__", fuse.sanitize_filename(".."))
1124
1125
1126 class FuseMagicTestPDHOnly(MountTestBase):
1127     def setUp(self, api=None):
1128         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1129
1130         cw = arvados.CollectionWriter()
1131
1132         cw.start_new_file('thing1.txt')
1133         cw.write("data 1")
1134
1135         self.testcollection = cw.finish()
1136         self.test_manifest = cw.manifest_text()
1137         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1138         self.testcollectionuuid = str(created['uuid'])
1139
1140     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1141         if skip_pdh_only is True:
1142             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1143         else:
1144             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1145
1146         mount_ls = llfuse.listdir(self.mounttmp)
1147         self.assertIn('README', mount_ls)
1148         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1149                              arvados.util.uuid_pattern.match(fn)
1150                              for fn in mount_ls),
1151                          "new FUSE MagicDirectory lists Collection")
1152
1153         # look up using pdh should succeed in all cases
1154         self.assertDirContents(self.testcollection, ['thing1.txt'])
1155         self.assertDirContents(os.path.join('by_id', self.testcollection),
1156                                ['thing1.txt'])
1157         mount_ls = llfuse.listdir(self.mounttmp)
1158         self.assertIn('README', mount_ls)
1159         self.assertIn(self.testcollection, mount_ls)
1160         self.assertIn(self.testcollection,
1161                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1162
1163         files = {}
1164         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1165
1166         for k, v in files.items():
1167             with open(os.path.join(self.mounttmp, k)) as f:
1168                 self.assertEqual(v, f.read())
1169
1170         # look up using uuid should fail when pdh_only is set
1171         if pdh_only is True:
1172             with self.assertRaises(OSError):
1173                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1174                                ['thing1.txt'])
1175         else:
1176             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1177                                ['thing1.txt'])
1178
1179     def test_with_pdh_only_true(self):
1180         self.verify_pdh_only(pdh_only=True)
1181
1182     def test_with_pdh_only_false(self):
1183         self.verify_pdh_only(pdh_only=False)
1184
1185     def test_with_default_by_id(self):
1186         self.verify_pdh_only(skip_pdh_only=True)