services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 import json
   6 import llfuse
   7 import logging
   8 import mock
   9 import os
  10 import subprocess
  11 import time
  12 import unittest
  13
  14 import arvados
  15 import arvados_fuse as fuse
  16 import run_test_server
  17
  18 from mount_test_base import MountTestBase
  19
  20 logger = logging.getLogger('arvados.arv-mount')
  21
  22
  23 class AssertWithTimeout(object):
  24     """Allow some time for an assertion to pass."""
  25
  26     def __init__(self, timeout=0):
  27         self.timeout = timeout
  28
  29     def __iter__(self):
  30         self.deadline = time.time() + self.timeout
  31         self.done = False
  32         return self
  33
  34     def next(self):
  35         if self.done:
  36             raise StopIteration
  37         return self.attempt
  38
  39     def attempt(self, fn, *args, **kwargs):
  40         try:
  41             fn(*args, **kwargs)
  42         except AssertionError:
  43             if time.time() > self.deadline:
  44                 raise
  45             time.sleep(0.1)
  46         else:
  47             self.done = True
  48
  49
  50 class FuseMountTest(MountTestBase):
  51     def setUp(self):
  52         super(FuseMountTest, self).setUp()
  53
  54         cw = arvados.CollectionWriter()
  55
  56         cw.start_new_file('thing1.txt')
  57         cw.write("data 1")
  58         cw.start_new_file('thing2.txt')
  59         cw.write("data 2")
  60
  61         cw.start_new_stream('dir1')
  62         cw.start_new_file('thing3.txt')
  63         cw.write("data 3")
  64         cw.start_new_file('thing4.txt')
  65         cw.write("data 4")
  66
  67         cw.start_new_stream('dir2')
  68         cw.start_new_file('thing5.txt')
  69         cw.write("data 5")
  70         cw.start_new_file('thing6.txt')
  71         cw.write("data 6")
  72
  73         cw.start_new_stream('dir2/dir3')
  74         cw.start_new_file('thing7.txt')
  75         cw.write("data 7")
  76
  77         cw.start_new_file('thing8.txt')
  78         cw.write("data 8")
  79
  80         cw.start_new_stream('edgecases')
  81         for f in ":/.../-/*/\x01\\/ ".split("/"):
  82             cw.start_new_file(f)
  83             cw.write('x')
  84
  85         for f in ":/.../-/*/\x01\\/ ".split("/"):
  86             cw.start_new_stream('edgecases/dirs/' + f)
  87             cw.start_new_file('x/x')
  88             cw.write('x')
  89
  90         self.testcollection = cw.finish()
  91         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  92
  93     def runTest(self):
  94         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  95
  96         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
  97                                       'edgecases', 'dir1', 'dir2'])
  98         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
  99         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 100         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 101         self.assertDirContents('edgecases',
 102                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
 103         self.assertDirContents('edgecases/dirs',
 104                                ":/.../-/*/\x01\\/ ".split("/"))
 105
 106         files = {'thing1.txt': 'data 1',
 107                  'thing2.txt': 'data 2',
 108                  'dir1/thing3.txt': 'data 3',
 109                  'dir1/thing4.txt': 'data 4',
 110                  'dir2/thing5.txt': 'data 5',
 111                  'dir2/thing6.txt': 'data 6',
 112                  'dir2/dir3/thing7.txt': 'data 7',
 113                  'dir2/dir3/thing8.txt': 'data 8'}
 114
 115         for k, v in files.items():
 116             with open(os.path.join(self.mounttmp, k)) as f:
 117                 self.assertEqual(v, f.read())
 118
 119
 120 class FuseMagicTest(MountTestBase):
 121     def setUp(self, api=None):
 122         super(FuseMagicTest, self).setUp(api=api)
 123
 124         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 125         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 126
 127         cw = arvados.CollectionWriter()
 128
 129         cw.start_new_file('thing1.txt')
 130         cw.write("data 1")
 131
 132         self.testcollection = cw.finish()
 133         self.test_manifest = cw.manifest_text()
 134         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 135         self.test_manifest_pdh = coll['portable_data_hash']
 136
 137     def runTest(self):
 138         self.make_mount(fuse.MagicDirectory)
 139
 140         mount_ls = llfuse.listdir(self.mounttmp)
 141         self.assertIn('README', mount_ls)
 142         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 143                              arvados.util.uuid_pattern.match(fn)
 144                              for fn in mount_ls),
 145                          "new FUSE MagicDirectory has no collections or projects")
 146         self.assertDirContents(self.testcollection, ['thing1.txt'])
 147         self.assertDirContents(os.path.join('by_id', self.testcollection),
 148                                ['thing1.txt'])
 149         self.assertIn(self.collection_in_test_project,
 150                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 151         self.assertIn(self.collection_in_test_project,
 152                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 153
 154         mount_ls = llfuse.listdir(self.mounttmp)
 155         self.assertIn('README', mount_ls)
 156         self.assertIn(self.testcollection, mount_ls)
 157         self.assertIn(self.testcollection,
 158                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 159         self.assertIn(self.test_project, mount_ls)
 160         self.assertIn(self.test_project,
 161                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 162
 163         files = {}
 164         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 165
 166         for k, v in files.items():
 167             with open(os.path.join(self.mounttmp, k)) as f:
 168                 self.assertEqual(v, f.read())
 169
 170
 171 class FuseTagsTest(MountTestBase):
 172     def runTest(self):
 173         self.make_mount(fuse.TagsDirectory)
 174
 175         d1 = llfuse.listdir(self.mounttmp)
 176         d1.sort()
 177         self.assertEqual(['foo_tag'], d1)
 178
 179         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 180         d2.sort()
 181         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 182
 183         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 184         d3.sort()
 185         self.assertEqual(['foo'], d3)
 186
 187
 188 class FuseTagsUpdateTest(MountTestBase):
 189     def tag_collection(self, coll_uuid, tag_name):
 190         return self.api.links().create(
 191             body={'link': {'head_uuid': coll_uuid,
 192                            'link_class': 'tag',
 193                            'name': tag_name,
 194         }}).execute()
 195
 196     def runTest(self):
 197         self.make_mount(fuse.TagsDirectory, poll_time=1)
 198
 199         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 200
 201         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 202         self.tag_collection(bar_uuid, 'fuse_test_tag')
 203         for attempt in AssertWithTimeout(10):
 204             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 205         self.assertDirContents('fuse_test_tag', [bar_uuid])
 206
 207         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 208         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 209         for attempt in AssertWithTimeout(10):
 210             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 211
 212         self.api.links().delete(uuid=l['uuid']).execute()
 213         for attempt in AssertWithTimeout(10):
 214             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 215
 216
 217 def fuseSharedTestHelper(mounttmp):
 218     class Test(unittest.TestCase):
 219         def runTest(self):
 220             # Double check that we can open and read objects in this folder as a file,
 221             # and that its contents are what we expect.
 222             baz_path = os.path.join(
 223                 mounttmp,
 224                 'FUSE User',
 225                 'FUSE Test Project',
 226                 'collection in FUSE project',
 227                 'baz')
 228             with open(baz_path) as f:
 229                 self.assertEqual("baz", f.read())
 230
 231             # check mtime on collection
 232             st = os.stat(baz_path)
 233             try:
 234                 mtime = st.st_mtime_ns / 1000000000
 235             except AttributeError:
 236                 mtime = st.st_mtime
 237             self.assertEqual(mtime, 1391448174)
 238
 239             # shared_dirs is a list of the directories exposed
 240             # by fuse.SharedDirectory (i.e. any object visible
 241             # to the current user)
 242             shared_dirs = llfuse.listdir(mounttmp)
 243             shared_dirs.sort()
 244             self.assertIn('FUSE User', shared_dirs)
 245
 246             # fuse_user_objs is a list of the objects owned by the FUSE
 247             # test user (which present as files in the 'FUSE User'
 248             # directory)
 249             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 250             fuse_user_objs.sort()
 251             self.assertEqual(['FUSE Test Project',                    # project owned by user
 252                               'collection #1 owned by FUSE',          # collection owned by user
 253                               'collection #2 owned by FUSE'          # collection owned by user
 254                           ], fuse_user_objs)
 255
 256             # test_proj_files is a list of the files in the FUSE Test Project.
 257             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 258             test_proj_files.sort()
 259             self.assertEqual(['collection in FUSE project'
 260                           ], test_proj_files)
 261
 262
 263     Test().runTest()
 264
 265 class FuseSharedTest(MountTestBase):
 266     def runTest(self):
 267         self.make_mount(fuse.SharedDirectory,
 268                         exclude=self.api.users().current().execute()['uuid'])
 269         keep = arvados.keep.KeepClient()
 270         keep.put("baz")
 271
 272         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 273
 274
 275 class FuseHomeTest(MountTestBase):
 276     def runTest(self):
 277         self.make_mount(fuse.ProjectDirectory,
 278                         project_object=self.api.users().current().execute())
 279
 280         d1 = llfuse.listdir(self.mounttmp)
 281         self.assertIn('Unrestricted public data', d1)
 282
 283         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 284         public_project = run_test_server.fixture('groups')[
 285             'anonymously_accessible_project']
 286         found_in = 0
 287         found_not_in = 0
 288         for name, item in run_test_server.fixture('collections').iteritems():
 289             if 'name' not in item:
 290                 pass
 291             elif item['owner_uuid'] == public_project['uuid']:
 292                 self.assertIn(item['name'], d2)
 293                 found_in += 1
 294             else:
 295                 # Artificial assumption here: there is no public
 296                 # collection fixture with the same name as a
 297                 # non-public collection.
 298                 self.assertNotIn(item['name'], d2)
 299                 found_not_in += 1
 300         self.assertNotEqual(0, found_in)
 301         self.assertNotEqual(0, found_not_in)
 302
 303         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 304         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 305
 306
 307 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 308     class Test(unittest.TestCase):
 309         def runTest(self):
 310             d1 = llfuse.listdir(mounttmp)
 311             self.assertEqual(["file1.txt"], d1)
 312             with open(os.path.join(mounttmp, "file1.txt")) as f:
 313                 self.assertEqual("blub", f.read())
 314     Test().runTest()
 315
 316 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 317     class Test(unittest.TestCase):
 318         def runTest(self):
 319             d1 = llfuse.listdir(mounttmp)
 320             self.assertEqual(["file1.txt"], d1)
 321             with open(os.path.join(mounttmp, "file1.txt")) as f:
 322                 self.assertEqual("plnp", f.read())
 323     Test().runTest()
 324
 325 class FuseModifyFileTest(MountTestBase):
 326     def runTest(self):
 327         collection = arvados.collection.Collection(api_client=self.api)
 328         with collection.open("file1.txt", "w") as f:
 329             f.write("blub")
 330
 331         collection.save_new()
 332
 333         m = self.make_mount(fuse.CollectionDirectory)
 334         with llfuse.lock:
 335             m.new_collection(collection.api_response(), collection)
 336
 337         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 338
 339         with collection.open("file1.txt", "w") as f:
 340             f.write("plnp")
 341
 342         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 343
 344
 345 class FuseAddFileToCollectionTest(MountTestBase):
 346     def runTest(self):
 347         collection = arvados.collection.Collection(api_client=self.api)
 348         with collection.open("file1.txt", "w") as f:
 349             f.write("blub")
 350
 351         collection.save_new()
 352
 353         m = self.make_mount(fuse.CollectionDirectory)
 354         with llfuse.lock:
 355             m.new_collection(collection.api_response(), collection)
 356
 357         d1 = llfuse.listdir(self.mounttmp)
 358         self.assertEqual(["file1.txt"], d1)
 359
 360         with collection.open("file2.txt", "w") as f:
 361             f.write("plnp")
 362
 363         d1 = llfuse.listdir(self.mounttmp)
 364         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 365
 366
 367 class FuseRemoveFileFromCollectionTest(MountTestBase):
 368     def runTest(self):
 369         collection = arvados.collection.Collection(api_client=self.api)
 370         with collection.open("file1.txt", "w") as f:
 371             f.write("blub")
 372
 373         with collection.open("file2.txt", "w") as f:
 374             f.write("plnp")
 375
 376         collection.save_new()
 377
 378         m = self.make_mount(fuse.CollectionDirectory)
 379         with llfuse.lock:
 380             m.new_collection(collection.api_response(), collection)
 381
 382         d1 = llfuse.listdir(self.mounttmp)
 383         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 384
 385         collection.remove("file2.txt")
 386
 387         d1 = llfuse.listdir(self.mounttmp)
 388         self.assertEqual(["file1.txt"], d1)
 389
 390
 391 def fuseCreateFileTestHelper(mounttmp):
 392     class Test(unittest.TestCase):
 393         def runTest(self):
 394             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 395                 pass
 396     Test().runTest()
 397
 398 class FuseCreateFileTest(MountTestBase):
 399     def runTest(self):
 400         collection = arvados.collection.Collection(api_client=self.api)
 401         collection.save_new()
 402
 403         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 404         self.assertEqual(collection2["manifest_text"], "")
 405
 406         collection.save_new()
 407
 408         m = self.make_mount(fuse.CollectionDirectory)
 409         with llfuse.lock:
 410             m.new_collection(collection.api_response(), collection)
 411         self.assertTrue(m.writable())
 412
 413         self.assertNotIn("file1.txt", collection)
 414
 415         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 416
 417         self.assertIn("file1.txt", collection)
 418
 419         d1 = llfuse.listdir(self.mounttmp)
 420         self.assertEqual(["file1.txt"], d1)
 421
 422         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 423         self.assertRegexpMatches(collection2["manifest_text"],
 424             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 425
 426
 427 def fuseWriteFileTestHelperWriteFile(mounttmp):
 428     class Test(unittest.TestCase):
 429         def runTest(self):
 430             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 431                 f.write("Hello world!")
 432     Test().runTest()
 433
 434 def fuseWriteFileTestHelperReadFile(mounttmp):
 435     class Test(unittest.TestCase):
 436         def runTest(self):
 437             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 438                 self.assertEqual(f.read(), "Hello world!")
 439     Test().runTest()
 440
 441 class FuseWriteFileTest(MountTestBase):
 442     def runTest(self):
 443         collection = arvados.collection.Collection(api_client=self.api)
 444         collection.save_new()
 445
 446         m = self.make_mount(fuse.CollectionDirectory)
 447         with llfuse.lock:
 448             m.new_collection(collection.api_response(), collection)
 449         self.assertTrue(m.writable())
 450
 451         self.assertNotIn("file1.txt", collection)
 452
 453         self.assertEqual(0, self.operations.write_counter.get())
 454         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 455         self.assertEqual(12, self.operations.write_counter.get())
 456
 457         with collection.open("file1.txt") as f:
 458             self.assertEqual(f.read(), "Hello world!")
 459
 460         self.assertEqual(0, self.operations.read_counter.get())
 461         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 462         self.assertEqual(12, self.operations.read_counter.get())
 463
 464         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 465         self.assertRegexpMatches(collection2["manifest_text"],
 466             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 467
 468
 469 def fuseUpdateFileTestHelper(mounttmp):
 470     class Test(unittest.TestCase):
 471         def runTest(self):
 472             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 473                 f.write("Hello world!")
 474
 475             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 476                 fr = f.read()
 477                 self.assertEqual(fr, "Hello world!")
 478                 f.seek(0)
 479                 f.write("Hola mundo!")
 480                 f.seek(0)
 481                 fr = f.read()
 482                 self.assertEqual(fr, "Hola mundo!!")
 483
 484             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 485                 self.assertEqual(f.read(), "Hola mundo!!")
 486
 487     Test().runTest()
 488
 489 class FuseUpdateFileTest(MountTestBase):
 490     def runTest(self):
 491         collection = arvados.collection.Collection(api_client=self.api)
 492         collection.save_new()
 493
 494         m = self.make_mount(fuse.CollectionDirectory)
 495         with llfuse.lock:
 496             m.new_collection(collection.api_response(), collection)
 497         self.assertTrue(m.writable())
 498
 499         # See note in MountTestBase.setUp
 500         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 501
 502         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 503         self.assertRegexpMatches(collection2["manifest_text"],
 504             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 505
 506
 507 def fuseMkdirTestHelper(mounttmp):
 508     class Test(unittest.TestCase):
 509         def runTest(self):
 510             with self.assertRaises(IOError):
 511                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 512                     f.write("Hello world!")
 513
 514             os.mkdir(os.path.join(mounttmp, "testdir"))
 515
 516             with self.assertRaises(OSError):
 517                 os.mkdir(os.path.join(mounttmp, "testdir"))
 518
 519             d1 = llfuse.listdir(mounttmp)
 520             self.assertEqual(["testdir"], d1)
 521
 522             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 523                 f.write("Hello world!")
 524
 525             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 526             self.assertEqual(["file1.txt"], d1)
 527
 528     Test().runTest()
 529
 530 class FuseMkdirTest(MountTestBase):
 531     def runTest(self):
 532         collection = arvados.collection.Collection(api_client=self.api)
 533         collection.save_new()
 534
 535         m = self.make_mount(fuse.CollectionDirectory)
 536         with llfuse.lock:
 537             m.new_collection(collection.api_response(), collection)
 538         self.assertTrue(m.writable())
 539
 540         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 541
 542         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 543         self.assertRegexpMatches(collection2["manifest_text"],
 544             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 545
 546
 547 def fuseRmTestHelperWriteFile(mounttmp):
 548     class Test(unittest.TestCase):
 549         def runTest(self):
 550             os.mkdir(os.path.join(mounttmp, "testdir"))
 551
 552             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 553                 f.write("Hello world!")
 554
 555     Test().runTest()
 556
 557 def fuseRmTestHelperDeleteFile(mounttmp):
 558     class Test(unittest.TestCase):
 559         def runTest(self):
 560             # Can't delete because it's not empty
 561             with self.assertRaises(OSError):
 562                 os.rmdir(os.path.join(mounttmp, "testdir"))
 563
 564             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 565             self.assertEqual(["file1.txt"], d1)
 566
 567             # Delete file
 568             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 569
 570             # Make sure it's empty
 571             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 572             self.assertEqual([], d1)
 573
 574             # Try to delete it again
 575             with self.assertRaises(OSError):
 576                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 577
 578     Test().runTest()
 579
 580 def fuseRmTestHelperRmdir(mounttmp):
 581     class Test(unittest.TestCase):
 582         def runTest(self):
 583             # Should be able to delete now that it is empty
 584             os.rmdir(os.path.join(mounttmp, "testdir"))
 585
 586             # Make sure it's empty
 587             d1 = llfuse.listdir(os.path.join(mounttmp))
 588             self.assertEqual([], d1)
 589
 590             # Try to delete it again
 591             with self.assertRaises(OSError):
 592                 os.rmdir(os.path.join(mounttmp, "testdir"))
 593
 594     Test().runTest()
 595
 596 class FuseRmTest(MountTestBase):
 597     def runTest(self):
 598         collection = arvados.collection.Collection(api_client=self.api)
 599         collection.save_new()
 600
 601         m = self.make_mount(fuse.CollectionDirectory)
 602         with llfuse.lock:
 603             m.new_collection(collection.api_response(), collection)
 604         self.assertTrue(m.writable())
 605
 606         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 607
 608         # Starting manifest
 609         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 610         self.assertRegexpMatches(collection2["manifest_text"],
 611             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 612         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 613
 614         # Can't have empty directories :-( so manifest will be empty.
 615         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 616         self.assertEqual(collection2["manifest_text"], "")
 617
 618         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 619
 620         # manifest should be empty now.
 621         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 622         self.assertEqual(collection2["manifest_text"], "")
 623
 624
 625 def fuseMvFileTestHelperWriteFile(mounttmp):
 626     class Test(unittest.TestCase):
 627         def runTest(self):
 628             os.mkdir(os.path.join(mounttmp, "testdir"))
 629
 630             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 631                 f.write("Hello world!")
 632
 633     Test().runTest()
 634
 635 def fuseMvFileTestHelperMoveFile(mounttmp):
 636     class Test(unittest.TestCase):
 637         def runTest(self):
 638             d1 = llfuse.listdir(os.path.join(mounttmp))
 639             self.assertEqual(["testdir"], d1)
 640             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 641             self.assertEqual(["file1.txt"], d1)
 642
 643             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 644
 645             d1 = llfuse.listdir(os.path.join(mounttmp))
 646             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 647             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 648             self.assertEqual([], d1)
 649
 650     Test().runTest()
 651
 652 class FuseMvFileTest(MountTestBase):
 653     def runTest(self):
 654         collection = arvados.collection.Collection(api_client=self.api)
 655         collection.save_new()
 656
 657         m = self.make_mount(fuse.CollectionDirectory)
 658         with llfuse.lock:
 659             m.new_collection(collection.api_response(), collection)
 660         self.assertTrue(m.writable())
 661
 662         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 663
 664         # Starting manifest
 665         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 666         self.assertRegexpMatches(collection2["manifest_text"],
 667             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 668
 669         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 670
 671         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 672         self.assertRegexpMatches(collection2["manifest_text"],
 673             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 674
 675
 676 def fuseRenameTestHelper(mounttmp):
 677     class Test(unittest.TestCase):
 678         def runTest(self):
 679             os.mkdir(os.path.join(mounttmp, "testdir"))
 680
 681             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 682                 f.write("Hello world!")
 683
 684     Test().runTest()
 685
 686 class FuseRenameTest(MountTestBase):
 687     def runTest(self):
 688         collection = arvados.collection.Collection(api_client=self.api)
 689         collection.save_new()
 690
 691         m = self.make_mount(fuse.CollectionDirectory)
 692         with llfuse.lock:
 693             m.new_collection(collection.api_response(), collection)
 694         self.assertTrue(m.writable())
 695
 696         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 697
 698         # Starting manifest
 699         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 700         self.assertRegexpMatches(collection2["manifest_text"],
 701             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 702
 703         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 704         self.assertEqual(["testdir"], d1)
 705         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 706         self.assertEqual(["file1.txt"], d1)
 707
 708         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 709
 710         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 711         self.assertEqual(["testdir2"], sorted(d1))
 712         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 713         self.assertEqual(["file1.txt"], d1)
 714
 715         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 716         self.assertRegexpMatches(collection2["manifest_text"],
 717             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 718
 719
 720 class FuseUpdateFromEventTest(MountTestBase):
 721     def runTest(self):
 722         collection = arvados.collection.Collection(api_client=self.api)
 723         collection.save_new()
 724
 725         m = self.make_mount(fuse.CollectionDirectory)
 726         with llfuse.lock:
 727             m.new_collection(collection.api_response(), collection)
 728
 729         self.operations.listen_for_events()
 730
 731         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 732         self.assertEqual([], sorted(d1))
 733
 734         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 735             with collection2.open("file1.txt", "w") as f:
 736                 f.write("foo")
 737
 738         for attempt in AssertWithTimeout(10):
 739             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 740
 741
 742 class FuseDeleteProjectEventTest(MountTestBase):
 743     def runTest(self):
 744
 745         aproject = self.api.groups().create(body={
 746             "name": "aproject",
 747             "group_class": "project"
 748         }).execute()
 749
 750         bproject = self.api.groups().create(body={
 751             "name": "bproject",
 752             "group_class": "project",
 753             "owner_uuid": aproject["uuid"]
 754         }).execute()
 755
 756         self.make_mount(fuse.ProjectDirectory,
 757                         project_object=self.api.users().current().execute())
 758
 759         self.operations.listen_for_events()
 760
 761         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 762         self.assertEqual(["bproject"], sorted(d1))
 763
 764         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 765
 766         for attempt in AssertWithTimeout(10):
 767             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 768
 769
 770 def fuseFileConflictTestHelper(mounttmp):
 771     class Test(unittest.TestCase):
 772         def runTest(self):
 773             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 774                 f.write("bar")
 775
 776             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 777             self.assertEqual(len(d1), 2)
 778
 779             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 780                 self.assertEqual(f.read(), "bar")
 781
 782             self.assertRegexpMatches(d1[1],
 783                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 784
 785             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 786                 self.assertEqual(f.read(), "foo")
 787
 788     Test().runTest()
 789
 790 class FuseFileConflictTest(MountTestBase):
 791     def runTest(self):
 792         collection = arvados.collection.Collection(api_client=self.api)
 793         collection.save_new()
 794
 795         m = self.make_mount(fuse.CollectionDirectory)
 796         with llfuse.lock:
 797             m.new_collection(collection.api_response(), collection)
 798
 799         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 800         self.assertEqual([], sorted(d1))
 801
 802         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 803             with collection2.open("file1.txt", "w") as f:
 804                 f.write("foo")
 805
 806         # See note in MountTestBase.setUp
 807         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 808
 809
 810 def fuseUnlinkOpenFileTest(mounttmp):
 811     class Test(unittest.TestCase):
 812         def runTest(self):
 813             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 814                 f.write("foo")
 815
 816                 d1 = llfuse.listdir(os.path.join(mounttmp))
 817                 self.assertEqual(["file1.txt"], sorted(d1))
 818
 819                 os.remove(os.path.join(mounttmp, "file1.txt"))
 820
 821                 d1 = llfuse.listdir(os.path.join(mounttmp))
 822                 self.assertEqual([], sorted(d1))
 823
 824                 f.seek(0)
 825                 self.assertEqual(f.read(), "foo")
 826                 f.write("bar")
 827
 828                 f.seek(0)
 829                 self.assertEqual(f.read(), "foobar")
 830
 831     Test().runTest()
 832
 833 class FuseUnlinkOpenFileTest(MountTestBase):
 834     def runTest(self):
 835         collection = arvados.collection.Collection(api_client=self.api)
 836         collection.save_new()
 837
 838         m = self.make_mount(fuse.CollectionDirectory)
 839         with llfuse.lock:
 840             m.new_collection(collection.api_response(), collection)
 841
 842         # See note in MountTestBase.setUp
 843         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 844
 845         self.assertEqual(collection.manifest_text(), "")
 846
 847
 848 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 849     class Test(unittest.TestCase):
 850         def runTest(self):
 851             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 852                 f.write("Hello world!")
 853
 854             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 855             self.assertEqual(["file1.txt"], sorted(d1))
 856             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 857             self.assertEqual([], sorted(d1))
 858
 859     Test().runTest()
 860
 861 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 862     class Test(unittest.TestCase):
 863         def runTest(self):
 864             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 865
 866             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 867             self.assertEqual([], sorted(d1))
 868             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 869             self.assertEqual(["file2.txt"], sorted(d1))
 870
 871     Test().runTest()
 872
 873 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 874     def runTest(self):
 875         collection1 = arvados.collection.Collection(api_client=self.api)
 876         collection1.save_new()
 877
 878         collection2 = arvados.collection.Collection(api_client=self.api)
 879         collection2.save_new()
 880
 881         m = self.make_mount(fuse.MagicDirectory)
 882
 883         # See note in MountTestBase.setUp
 884         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 885                                                   collection1.manifest_locator(),
 886                                                   collection2.manifest_locator()))
 887
 888         collection1.update()
 889         collection2.update()
 890
 891         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 892         self.assertEqual(collection2.manifest_text(), "")
 893
 894         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 895                                                   collection1.manifest_locator(),
 896                                                   collection2.manifest_locator()))
 897
 898         collection1.update()
 899         collection2.update()
 900
 901         self.assertEqual(collection1.manifest_text(), "")
 902         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 903
 904         collection1.stop_threads()
 905         collection2.stop_threads()
 906
 907
 908 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 909     class Test(unittest.TestCase):
 910         def runTest(self):
 911             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 912             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 913                 f.write("Hello world!")
 914
 915             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 916             self.assertEqual(["testdir"], sorted(d1))
 917             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 918             self.assertEqual(["file1.txt"], sorted(d1))
 919
 920             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 921             self.assertEqual([], sorted(d1))
 922
 923     Test().runTest()
 924
 925
 926 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 927     class Test(unittest.TestCase):
 928         def runTest(self):
 929             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 930
 931             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 932             self.assertEqual([], sorted(d1))
 933
 934             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 935             self.assertEqual(["testdir2"], sorted(d1))
 936             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 937             self.assertEqual(["file1.txt"], sorted(d1))
 938
 939             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 940                 self.assertEqual(f.read(), "Hello world!")
 941
 942     Test().runTest()
 943
 944 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 945     def runTest(self):
 946         collection1 = arvados.collection.Collection(api_client=self.api)
 947         collection1.save_new()
 948
 949         collection2 = arvados.collection.Collection(api_client=self.api)
 950         collection2.save_new()
 951
 952         m = self.make_mount(fuse.MagicDirectory)
 953
 954         # See note in MountTestBase.setUp
 955         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 956                                                   collection1.manifest_locator(),
 957                                                   collection2.manifest_locator()))
 958
 959         collection1.update()
 960         collection2.update()
 961
 962         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 963         self.assertEqual(collection2.manifest_text(), "")
 964
 965         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 966                                                   collection1.manifest_locator(),
 967                                                   collection2.manifest_locator()))
 968
 969         collection1.update()
 970         collection2.update()
 971
 972         self.assertEqual(collection1.manifest_text(), "")
 973         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 974
 975         collection1.stop_threads()
 976         collection2.stop_threads()
 977
 978 def fuseProjectMkdirTestHelper1(mounttmp):
 979     class Test(unittest.TestCase):
 980         def runTest(self):
 981             os.mkdir(os.path.join(mounttmp, "testcollection"))
 982             with self.assertRaises(OSError):
 983                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 984     Test().runTest()
 985
 986 def fuseProjectMkdirTestHelper2(mounttmp):
 987     class Test(unittest.TestCase):
 988         def runTest(self):
 989             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 990                 f.write("Hello world!")
 991             with self.assertRaises(OSError):
 992                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 993             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 994             with self.assertRaises(OSError):
 995                 os.remove(os.path.join(mounttmp, "testcollection"))
 996             os.rmdir(os.path.join(mounttmp, "testcollection"))
 997     Test().runTest()
 998
 999 class FuseProjectMkdirRmdirTest(MountTestBase):
1000     def runTest(self):
1001         self.make_mount(fuse.ProjectDirectory,
1002                         project_object=self.api.users().current().execute())
1003
1004         d1 = llfuse.listdir(self.mounttmp)
1005         self.assertNotIn('testcollection', d1)
1006
1007         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1008
1009         d1 = llfuse.listdir(self.mounttmp)
1010         self.assertIn('testcollection', d1)
1011
1012         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1013
1014         d1 = llfuse.listdir(self.mounttmp)
1015         self.assertNotIn('testcollection', d1)
1016
1017
1018 def fuseProjectMvTestHelper1(mounttmp):
1019     class Test(unittest.TestCase):
1020         def runTest(self):
1021             d1 = llfuse.listdir(mounttmp)
1022             self.assertNotIn('testcollection', d1)
1023
1024             os.mkdir(os.path.join(mounttmp, "testcollection"))
1025
1026             d1 = llfuse.listdir(mounttmp)
1027             self.assertIn('testcollection', d1)
1028
1029             with self.assertRaises(OSError):
1030                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1031
1032             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1033
1034             d1 = llfuse.listdir(mounttmp)
1035             self.assertNotIn('testcollection', d1)
1036
1037             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1038             self.assertIn('testcollection', d1)
1039
1040     Test().runTest()
1041
1042 class FuseProjectMvTest(MountTestBase):
1043     def runTest(self):
1044         self.make_mount(fuse.ProjectDirectory,
1045                         project_object=self.api.users().current().execute())
1046
1047         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1048
1049
1050 def fuseFsyncTestHelper(mounttmp, k):
1051     class Test(unittest.TestCase):
1052         def runTest(self):
1053             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1054             os.fsync(fd)
1055             os.close(fd)
1056
1057     Test().runTest()
1058
1059 class FuseFsyncTest(FuseMagicTest):
1060     def runTest(self):
1061         self.make_mount(fuse.MagicDirectory)
1062         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1063
1064
1065 class MagicDirApiError(FuseMagicTest):
1066     def setUp(self):
1067         api = mock.MagicMock()
1068         super(MagicDirApiError, self).setUp(api=api)
1069         api.collections().get().execute.side_effect = iter([
1070             Exception('API fail'),
1071             {
1072                 "manifest_text": self.test_manifest,
1073                 "portable_data_hash": self.test_manifest_pdh,
1074             },
1075         ])
1076         api.keep.get.side_effect = Exception('Keep fail')
1077
1078     def runTest(self):
1079         self.make_mount(fuse.MagicDirectory)
1080
1081         self.operations.inodes.inode_cache.cap = 1
1082         self.operations.inodes.inode_cache.min_entries = 2
1083
1084         with self.assertRaises(OSError):
1085             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1086
1087         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1088
1089
1090 class FuseUnitTest(unittest.TestCase):
1091     def test_sanitize_filename(self):
1092         acceptable = [
1093             "foo.txt",
1094             ".foo",
1095             "..foo",
1096             "...",
1097             "foo...",
1098             "foo..",
1099             "foo.",
1100             "-",
1101             "\x01\x02\x03",
1102             ]
1103         unacceptable = [
1104             "f\00",
1105             "\00\00",
1106             "/foo",
1107             "foo/",
1108             "//",
1109             ]
1110         for f in acceptable:
1111             self.assertEqual(f, fuse.sanitize_filename(f))
1112         for f in unacceptable:
1113             self.assertNotEqual(f, fuse.sanitize_filename(f))
1114             # The sanitized filename should be the same length, though.
1115             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1116         # Special cases
1117         self.assertEqual("_", fuse.sanitize_filename(""))
1118         self.assertEqual("_", fuse.sanitize_filename("."))
1119         self.assertEqual("__", fuse.sanitize_filename(".."))
1120
1121
1122 class FuseMagicTestPDHOnly(MountTestBase):
1123     def setUp(self, api=None):
1124         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1125
1126         cw = arvados.CollectionWriter()
1127
1128         cw.start_new_file('thing1.txt')
1129         cw.write("data 1")
1130
1131         self.testcollection = cw.finish()
1132         self.test_manifest = cw.manifest_text()
1133         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1134         self.testcollectionuuid = str(created['uuid'])
1135
1136     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1137         if skip_pdh_only is True:
1138             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1139         else:
1140             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1141
1142         mount_ls = llfuse.listdir(self.mounttmp)
1143         self.assertIn('README', mount_ls)
1144         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1145                              arvados.util.uuid_pattern.match(fn)
1146                              for fn in mount_ls),
1147                          "new FUSE MagicDirectory lists Collection")
1148
1149         # look up using pdh should succeed in all cases
1150         self.assertDirContents(self.testcollection, ['thing1.txt'])
1151         self.assertDirContents(os.path.join('by_id', self.testcollection),
1152                                ['thing1.txt'])
1153         mount_ls = llfuse.listdir(self.mounttmp)
1154         self.assertIn('README', mount_ls)
1155         self.assertIn(self.testcollection, mount_ls)
1156         self.assertIn(self.testcollection,
1157                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1158
1159         files = {}
1160         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1161
1162         for k, v in files.items():
1163             with open(os.path.join(self.mounttmp, k)) as f:
1164                 self.assertEqual(v, f.read())
1165
1166         # look up using uuid should fail when pdh_only is set
1167         if pdh_only is True:
1168             with self.assertRaises(OSError):
1169                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1170                                ['thing1.txt'])
1171         else:
1172             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1173                                ['thing1.txt'])
1174
1175     def test_with_pdh_only_true(self):
1176         self.verify_pdh_only(pdh_only=True)
1177
1178     def test_with_pdh_only_false(self):
1179         self.verify_pdh_only(pdh_only=False)
1180
1181     def test_with_default_by_id(self):
1182         self.verify_pdh_only(skip_pdh_only=True)