services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 from __future__ import absolute_import
   6 from future.utils import viewitems
   7 from builtins import str
   8 from builtins import object
   9 from six import assertRegex
  10 import json
  11 import llfuse
  12 import logging
  13 import mock
  14 import os
  15 import subprocess
  16 import time
  17 import unittest
  18
  19 import arvados
  20 import arvados_fuse as fuse
  21 from . import run_test_server
  22
  23 from .mount_test_base import MountTestBase
  24
  25 logger = logging.getLogger('arvados.arv-mount')
  26
  27
  28 class AssertWithTimeout(object):
  29     """Allow some time for an assertion to pass."""
  30
  31     def __init__(self, timeout=0):
  32         self.timeout = timeout
  33
  34     def __iter__(self):
  35         self.deadline = time.time() + self.timeout
  36         self.done = False
  37         return self
  38
  39     def __next__(self):
  40         if self.done:
  41             raise StopIteration
  42         return self.attempt
  43
  44     def attempt(self, fn, *args, **kwargs):
  45         try:
  46             fn(*args, **kwargs)
  47         except AssertionError:
  48             if time.time() > self.deadline:
  49                 raise
  50             time.sleep(0.1)
  51         else:
  52             self.done = True
  53
  54
  55 class FuseMountTest(MountTestBase):
  56     def setUp(self):
  57         super(FuseMountTest, self).setUp()
  58
  59         cw = arvados.CollectionWriter()
  60
  61         cw.start_new_file('thing1.txt')
  62         cw.write("data 1")
  63         cw.start_new_file('thing2.txt')
  64         cw.write("data 2")
  65
  66         cw.start_new_stream('dir1')
  67         cw.start_new_file('thing3.txt')
  68         cw.write("data 3")
  69         cw.start_new_file('thing4.txt')
  70         cw.write("data 4")
  71
  72         cw.start_new_stream('dir2')
  73         cw.start_new_file('thing5.txt')
  74         cw.write("data 5")
  75         cw.start_new_file('thing6.txt')
  76         cw.write("data 6")
  77
  78         cw.start_new_stream('dir2/dir3')
  79         cw.start_new_file('thing7.txt')
  80         cw.write("data 7")
  81
  82         cw.start_new_file('thing8.txt')
  83         cw.write("data 8")
  84
  85         cw.start_new_stream('edgecases')
  86         for f in ":/.../-/*/ ".split("/"):
  87             cw.start_new_file(f)
  88             cw.write('x')
  89
  90         for f in ":/.../-/*/ ".split("/"):
  91             cw.start_new_stream('edgecases/dirs/' + f)
  92             cw.start_new_file('x/x')
  93             cw.write('x')
  94
  95         self.testcollection = cw.finish()
  96         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  97
  98     def runTest(self):
  99         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
 100
 101         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 102                                       'edgecases', 'dir1', 'dir2'])
 103         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 104         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 105         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 106         self.assertDirContents('edgecases',
 107                                "dirs/:/.../-/*/ ".split("/"))
 108         self.assertDirContents('edgecases/dirs',
 109                                ":/.../-/*/ ".split("/"))
 110
 111         files = {'thing1.txt': 'data 1',
 112                  'thing2.txt': 'data 2',
 113                  'dir1/thing3.txt': 'data 3',
 114                  'dir1/thing4.txt': 'data 4',
 115                  'dir2/thing5.txt': 'data 5',
 116                  'dir2/thing6.txt': 'data 6',
 117                  'dir2/dir3/thing7.txt': 'data 7',
 118                  'dir2/dir3/thing8.txt': 'data 8'}
 119
 120         for k, v in viewitems(files):
 121             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 122                 self.assertEqual(v, f.read().decode())
 123
 124
 125 class FuseMagicTest(MountTestBase):
 126     def setUp(self, api=None):
 127         super(FuseMagicTest, self).setUp(api=api)
 128
 129         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 130         self.non_project_group = run_test_server.fixture('groups')['public']['uuid']
 131         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 132
 133         cw = arvados.CollectionWriter()
 134
 135         cw.start_new_file('thing1.txt')
 136         cw.write("data 1")
 137
 138         self.testcollection = cw.finish()
 139         self.test_manifest = cw.manifest_text()
 140         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 141         self.test_manifest_pdh = coll['portable_data_hash']
 142
 143     def runTest(self):
 144         self.make_mount(fuse.MagicDirectory)
 145
 146         mount_ls = llfuse.listdir(self.mounttmp)
 147         self.assertIn('README', mount_ls)
 148         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 149                              arvados.util.uuid_pattern.match(fn)
 150                              for fn in mount_ls),
 151                          "new FUSE MagicDirectory has no collections or projects")
 152         self.assertDirContents(self.testcollection, ['thing1.txt'])
 153         self.assertDirContents(os.path.join('by_id', self.testcollection),
 154                                ['thing1.txt'])
 155         self.assertIn(self.collection_in_test_project,
 156                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 157         self.assertIn(self.collection_in_test_project,
 158                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 159
 160         mount_ls = llfuse.listdir(self.mounttmp)
 161         self.assertIn('README', mount_ls)
 162         self.assertIn(self.testcollection, mount_ls)
 163         self.assertIn(self.testcollection,
 164                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 165         self.assertIn(self.test_project, mount_ls)
 166         self.assertIn(self.test_project,
 167                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 168
 169         with self.assertRaises(OSError):
 170             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 171
 172         files = {}
 173         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 174
 175         for k, v in viewitems(files):
 176             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 177                 self.assertEqual(v, f.read().decode())
 178
 179
 180 class FuseTagsTest(MountTestBase):
 181     def runTest(self):
 182         self.make_mount(fuse.TagsDirectory)
 183
 184         d1 = llfuse.listdir(self.mounttmp)
 185         d1.sort()
 186         self.assertEqual(['foo_tag'], d1)
 187
 188         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 189         d2.sort()
 190         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 191
 192         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 193         d3.sort()
 194         self.assertEqual(['foo'], d3)
 195
 196
 197 class FuseTagsUpdateTest(MountTestBase):
 198     def tag_collection(self, coll_uuid, tag_name):
 199         return self.api.links().create(
 200             body={'link': {'head_uuid': coll_uuid,
 201                            'link_class': 'tag',
 202                            'name': tag_name,
 203         }}).execute()
 204
 205     def runTest(self):
 206         self.make_mount(fuse.TagsDirectory, poll_time=1)
 207
 208         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 209
 210         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 211         self.tag_collection(bar_uuid, 'fuse_test_tag')
 212         for attempt in AssertWithTimeout(10):
 213             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 214         self.assertDirContents('fuse_test_tag', [bar_uuid])
 215
 216         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 217         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 218         for attempt in AssertWithTimeout(10):
 219             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 220
 221         self.api.links().delete(uuid=l['uuid']).execute()
 222         for attempt in AssertWithTimeout(10):
 223             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 224
 225
 226 def fuseSharedTestHelper(mounttmp):
 227     class Test(unittest.TestCase):
 228         def runTest(self):
 229             # Double check that we can open and read objects in this folder as a file,
 230             # and that its contents are what we expect.
 231             baz_path = os.path.join(
 232                 mounttmp,
 233                 'FUSE User',
 234                 'FUSE Test Project',
 235                 'collection in FUSE project',
 236                 'baz')
 237             with open(baz_path) as f:
 238                 self.assertEqual("baz", f.read())
 239
 240             # check mtime on collection
 241             st = os.stat(baz_path)
 242             try:
 243                 mtime = st.st_mtime_ns // 1000000000
 244             except AttributeError:
 245                 mtime = st.st_mtime
 246             self.assertEqual(mtime, 1391448174)
 247
 248             # shared_dirs is a list of the directories exposed
 249             # by fuse.SharedDirectory (i.e. any object visible
 250             # to the current user)
 251             shared_dirs = llfuse.listdir(mounttmp)
 252             shared_dirs.sort()
 253             self.assertIn('FUSE User', shared_dirs)
 254
 255             # fuse_user_objs is a list of the objects owned by the FUSE
 256             # test user (which present as files in the 'FUSE User'
 257             # directory)
 258             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 259             fuse_user_objs.sort()
 260             self.assertEqual(['FUSE Test Project',                    # project owned by user
 261                               'collection #1 owned by FUSE',          # collection owned by user
 262                               'collection #2 owned by FUSE'          # collection owned by user
 263                           ], fuse_user_objs)
 264
 265             # test_proj_files is a list of the files in the FUSE Test Project.
 266             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 267             test_proj_files.sort()
 268             self.assertEqual(['collection in FUSE project'
 269                           ], test_proj_files)
 270
 271
 272     Test().runTest()
 273
 274 class FuseSharedTest(MountTestBase):
 275     def runTest(self):
 276         self.make_mount(fuse.SharedDirectory,
 277                         exclude=self.api.users().current().execute()['uuid'])
 278         keep = arvados.keep.KeepClient()
 279         keep.put("baz".encode())
 280
 281         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 282
 283
 284 class FuseHomeTest(MountTestBase):
 285     def runTest(self):
 286         self.make_mount(fuse.ProjectDirectory,
 287                         project_object=self.api.users().current().execute())
 288
 289         d1 = llfuse.listdir(self.mounttmp)
 290         self.assertIn('Unrestricted public data', d1)
 291
 292         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 293         public_project = run_test_server.fixture('groups')[
 294             'anonymously_accessible_project']
 295         found_in = 0
 296         found_not_in = 0
 297         for name, item in viewitems(run_test_server.fixture('collections')):
 298             if 'name' not in item:
 299                 pass
 300             elif item['owner_uuid'] == public_project['uuid']:
 301                 self.assertIn(item['name'], d2)
 302                 found_in += 1
 303             else:
 304                 # Artificial assumption here: there is no public
 305                 # collection fixture with the same name as a
 306                 # non-public collection.
 307                 self.assertNotIn(item['name'], d2)
 308                 found_not_in += 1
 309         self.assertNotEqual(0, found_in)
 310         self.assertNotEqual(0, found_not_in)
 311
 312         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 313         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 314
 315
 316 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 317     class Test(unittest.TestCase):
 318         def runTest(self):
 319             d1 = llfuse.listdir(mounttmp)
 320             self.assertEqual(["file1.txt"], d1)
 321             with open(os.path.join(mounttmp, "file1.txt")) as f:
 322                 self.assertEqual("blub", f.read())
 323     Test().runTest()
 324
 325 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 326     class Test(unittest.TestCase):
 327         def runTest(self):
 328             d1 = llfuse.listdir(mounttmp)
 329             self.assertEqual(["file1.txt"], d1)
 330             with open(os.path.join(mounttmp, "file1.txt")) as f:
 331                 self.assertEqual("plnp", f.read())
 332     Test().runTest()
 333
 334 class FuseModifyFileTest(MountTestBase):
 335     def runTest(self):
 336         collection = arvados.collection.Collection(api_client=self.api)
 337         with collection.open("file1.txt", "w") as f:
 338             f.write("blub")
 339
 340         collection.save_new()
 341
 342         m = self.make_mount(fuse.CollectionDirectory)
 343         with llfuse.lock:
 344             m.new_collection(collection.api_response(), collection)
 345
 346         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 347
 348         with collection.open("file1.txt", "w") as f:
 349             f.write("plnp")
 350
 351         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 352
 353
 354 class FuseAddFileToCollectionTest(MountTestBase):
 355     def runTest(self):
 356         collection = arvados.collection.Collection(api_client=self.api)
 357         with collection.open("file1.txt", "w") as f:
 358             f.write("blub")
 359
 360         collection.save_new()
 361
 362         m = self.make_mount(fuse.CollectionDirectory)
 363         with llfuse.lock:
 364             m.new_collection(collection.api_response(), collection)
 365
 366         d1 = llfuse.listdir(self.mounttmp)
 367         self.assertEqual(["file1.txt"], d1)
 368
 369         with collection.open("file2.txt", "w") as f:
 370             f.write("plnp")
 371
 372         d1 = llfuse.listdir(self.mounttmp)
 373         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 374
 375
 376 class FuseRemoveFileFromCollectionTest(MountTestBase):
 377     def runTest(self):
 378         collection = arvados.collection.Collection(api_client=self.api)
 379         with collection.open("file1.txt", "w") as f:
 380             f.write("blub")
 381
 382         with collection.open("file2.txt", "w") as f:
 383             f.write("plnp")
 384
 385         collection.save_new()
 386
 387         m = self.make_mount(fuse.CollectionDirectory)
 388         with llfuse.lock:
 389             m.new_collection(collection.api_response(), collection)
 390
 391         d1 = llfuse.listdir(self.mounttmp)
 392         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 393
 394         collection.remove("file2.txt")
 395
 396         d1 = llfuse.listdir(self.mounttmp)
 397         self.assertEqual(["file1.txt"], d1)
 398
 399
 400 def fuseCreateFileTestHelper(mounttmp):
 401     class Test(unittest.TestCase):
 402         def runTest(self):
 403             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 404                 pass
 405     Test().runTest()
 406
 407 class FuseCreateFileTest(MountTestBase):
 408     def runTest(self):
 409         collection = arvados.collection.Collection(api_client=self.api)
 410         collection.save_new()
 411
 412         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 413         self.assertEqual(collection2["manifest_text"], "")
 414
 415         collection.save_new()
 416
 417         m = self.make_mount(fuse.CollectionDirectory)
 418         with llfuse.lock:
 419             m.new_collection(collection.api_response(), collection)
 420         self.assertTrue(m.writable())
 421
 422         self.assertNotIn("file1.txt", collection)
 423
 424         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 425
 426         self.assertIn("file1.txt", collection)
 427
 428         d1 = llfuse.listdir(self.mounttmp)
 429         self.assertEqual(["file1.txt"], d1)
 430
 431         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 432         assertRegex(self, collection2["manifest_text"],
 433             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 434
 435
 436 def fuseWriteFileTestHelperWriteFile(mounttmp):
 437     class Test(unittest.TestCase):
 438         def runTest(self):
 439             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 440                 f.write("Hello world!")
 441     Test().runTest()
 442
 443 def fuseWriteFileTestHelperReadFile(mounttmp):
 444     class Test(unittest.TestCase):
 445         def runTest(self):
 446             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 447                 self.assertEqual(f.read(), "Hello world!")
 448     Test().runTest()
 449
 450 class FuseWriteFileTest(MountTestBase):
 451     def runTest(self):
 452         collection = arvados.collection.Collection(api_client=self.api)
 453         collection.save_new()
 454
 455         m = self.make_mount(fuse.CollectionDirectory)
 456         with llfuse.lock:
 457             m.new_collection(collection.api_response(), collection)
 458         self.assertTrue(m.writable())
 459
 460         self.assertNotIn("file1.txt", collection)
 461
 462         self.assertEqual(0, self.operations.write_counter.get())
 463         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 464         self.assertEqual(12, self.operations.write_counter.get())
 465
 466         with collection.open("file1.txt") as f:
 467             self.assertEqual(f.read(), "Hello world!")
 468
 469         self.assertEqual(0, self.operations.read_counter.get())
 470         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 471         self.assertEqual(12, self.operations.read_counter.get())
 472
 473         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 474         assertRegex(self, collection2["manifest_text"],
 475             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 476
 477
 478 def fuseUpdateFileTestHelper(mounttmp):
 479     class Test(unittest.TestCase):
 480         def runTest(self):
 481             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 482                 f.write("Hello world!")
 483
 484             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 485                 fr = f.read()
 486                 self.assertEqual(fr, "Hello world!")
 487                 f.seek(0)
 488                 f.write("Hola mundo!")
 489                 f.seek(0)
 490                 fr = f.read()
 491                 self.assertEqual(fr, "Hola mundo!!")
 492
 493             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 494                 self.assertEqual(f.read(), "Hola mundo!!")
 495
 496     Test().runTest()
 497
 498 class FuseUpdateFileTest(MountTestBase):
 499     def runTest(self):
 500         collection = arvados.collection.Collection(api_client=self.api)
 501         collection.save_new()
 502
 503         m = self.make_mount(fuse.CollectionDirectory)
 504         with llfuse.lock:
 505             m.new_collection(collection.api_response(), collection)
 506         self.assertTrue(m.writable())
 507
 508         # See note in MountTestBase.setUp
 509         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 510
 511         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 512         assertRegex(self, collection2["manifest_text"],
 513             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 514
 515
 516 def fuseMkdirTestHelper(mounttmp):
 517     class Test(unittest.TestCase):
 518         def runTest(self):
 519             with self.assertRaises(IOError):
 520                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 521                     f.write("Hello world!")
 522
 523             os.mkdir(os.path.join(mounttmp, "testdir"))
 524
 525             with self.assertRaises(OSError):
 526                 os.mkdir(os.path.join(mounttmp, "testdir"))
 527
 528             d1 = llfuse.listdir(mounttmp)
 529             self.assertEqual(["testdir"], d1)
 530
 531             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 532                 f.write("Hello world!")
 533
 534             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 535             self.assertEqual(["file1.txt"], d1)
 536
 537     Test().runTest()
 538
 539 class FuseMkdirTest(MountTestBase):
 540     def runTest(self):
 541         collection = arvados.collection.Collection(api_client=self.api)
 542         collection.save_new()
 543
 544         m = self.make_mount(fuse.CollectionDirectory)
 545         with llfuse.lock:
 546             m.new_collection(collection.api_response(), collection)
 547         self.assertTrue(m.writable())
 548
 549         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 550
 551         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 552         assertRegex(self, collection2["manifest_text"],
 553             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 554
 555
 556 def fuseRmTestHelperWriteFile(mounttmp):
 557     class Test(unittest.TestCase):
 558         def runTest(self):
 559             os.mkdir(os.path.join(mounttmp, "testdir"))
 560
 561             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 562                 f.write("Hello world!")
 563
 564     Test().runTest()
 565
 566 def fuseRmTestHelperDeleteFile(mounttmp):
 567     class Test(unittest.TestCase):
 568         def runTest(self):
 569             # Can't delete because it's not empty
 570             with self.assertRaises(OSError):
 571                 os.rmdir(os.path.join(mounttmp, "testdir"))
 572
 573             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 574             self.assertEqual(["file1.txt"], d1)
 575
 576             # Delete file
 577             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 578
 579             # Make sure it's empty
 580             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 581             self.assertEqual([], d1)
 582
 583             # Try to delete it again
 584             with self.assertRaises(OSError):
 585                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 586
 587     Test().runTest()
 588
 589 def fuseRmTestHelperRmdir(mounttmp):
 590     class Test(unittest.TestCase):
 591         def runTest(self):
 592             # Should be able to delete now that it is empty
 593             os.rmdir(os.path.join(mounttmp, "testdir"))
 594
 595             # Make sure it's empty
 596             d1 = llfuse.listdir(os.path.join(mounttmp))
 597             self.assertEqual([], d1)
 598
 599             # Try to delete it again
 600             with self.assertRaises(OSError):
 601                 os.rmdir(os.path.join(mounttmp, "testdir"))
 602
 603     Test().runTest()
 604
 605 class FuseRmTest(MountTestBase):
 606     def runTest(self):
 607         collection = arvados.collection.Collection(api_client=self.api)
 608         collection.save_new()
 609
 610         m = self.make_mount(fuse.CollectionDirectory)
 611         with llfuse.lock:
 612             m.new_collection(collection.api_response(), collection)
 613         self.assertTrue(m.writable())
 614
 615         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 616
 617         # Starting manifest
 618         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 619         assertRegex(self, collection2["manifest_text"],
 620             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 621         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 622
 623         # Empty directories are represented by an empty file named "."
 624         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 625         assertRegex(self, collection2["manifest_text"],
 626                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 627
 628         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 629
 630         # manifest should be empty now.
 631         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 632         self.assertEqual(collection2["manifest_text"], "")
 633
 634
 635 def fuseMvFileTestHelperWriteFile(mounttmp):
 636     class Test(unittest.TestCase):
 637         def runTest(self):
 638             os.mkdir(os.path.join(mounttmp, "testdir"))
 639
 640             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 641                 f.write("Hello world!")
 642
 643     Test().runTest()
 644
 645 def fuseMvFileTestHelperMoveFile(mounttmp):
 646     class Test(unittest.TestCase):
 647         def runTest(self):
 648             d1 = llfuse.listdir(os.path.join(mounttmp))
 649             self.assertEqual(["testdir"], d1)
 650             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 651             self.assertEqual(["file1.txt"], d1)
 652
 653             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 654
 655             d1 = llfuse.listdir(os.path.join(mounttmp))
 656             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 657             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 658             self.assertEqual([], d1)
 659
 660     Test().runTest()
 661
 662 class FuseMvFileTest(MountTestBase):
 663     def runTest(self):
 664         collection = arvados.collection.Collection(api_client=self.api)
 665         collection.save_new()
 666
 667         m = self.make_mount(fuse.CollectionDirectory)
 668         with llfuse.lock:
 669             m.new_collection(collection.api_response(), collection)
 670         self.assertTrue(m.writable())
 671
 672         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 673
 674         # Starting manifest
 675         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 676         assertRegex(self, collection2["manifest_text"],
 677             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 678
 679         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 680
 681         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 682         assertRegex(self, collection2["manifest_text"],
 683             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 684
 685
 686 def fuseRenameTestHelper(mounttmp):
 687     class Test(unittest.TestCase):
 688         def runTest(self):
 689             os.mkdir(os.path.join(mounttmp, "testdir"))
 690
 691             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 692                 f.write("Hello world!")
 693
 694     Test().runTest()
 695
 696 class FuseRenameTest(MountTestBase):
 697     def runTest(self):
 698         collection = arvados.collection.Collection(api_client=self.api)
 699         collection.save_new()
 700
 701         m = self.make_mount(fuse.CollectionDirectory)
 702         with llfuse.lock:
 703             m.new_collection(collection.api_response(), collection)
 704         self.assertTrue(m.writable())
 705
 706         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 707
 708         # Starting manifest
 709         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 710         assertRegex(self, collection2["manifest_text"],
 711             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 712
 713         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 714         self.assertEqual(["testdir"], d1)
 715         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 716         self.assertEqual(["file1.txt"], d1)
 717
 718         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 719
 720         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 721         self.assertEqual(["testdir2"], sorted(d1))
 722         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 723         self.assertEqual(["file1.txt"], d1)
 724
 725         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 726         assertRegex(self, collection2["manifest_text"],
 727             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 728
 729
 730 class FuseUpdateFromEventTest(MountTestBase):
 731     def runTest(self):
 732         collection = arvados.collection.Collection(api_client=self.api)
 733         collection.save_new()
 734
 735         m = self.make_mount(fuse.CollectionDirectory)
 736         with llfuse.lock:
 737             m.new_collection(collection.api_response(), collection)
 738
 739         self.operations.listen_for_events()
 740
 741         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 742         self.assertEqual([], sorted(d1))
 743
 744         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 745             with collection2.open("file1.txt", "w") as f:
 746                 f.write("foo")
 747
 748         for attempt in AssertWithTimeout(10):
 749             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 750
 751
 752 class FuseDeleteProjectEventTest(MountTestBase):
 753     def runTest(self):
 754
 755         aproject = self.api.groups().create(body={
 756             "name": "aproject",
 757             "group_class": "project"
 758         }).execute()
 759
 760         bproject = self.api.groups().create(body={
 761             "name": "bproject",
 762             "group_class": "project",
 763             "owner_uuid": aproject["uuid"]
 764         }).execute()
 765
 766         self.make_mount(fuse.ProjectDirectory,
 767                         project_object=self.api.users().current().execute())
 768
 769         self.operations.listen_for_events()
 770
 771         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 772         self.assertEqual(["bproject"], sorted(d1))
 773
 774         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 775
 776         for attempt in AssertWithTimeout(10):
 777             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 778
 779
 780 def fuseFileConflictTestHelper(mounttmp):
 781     class Test(unittest.TestCase):
 782         def runTest(self):
 783             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 784                 f.write("bar")
 785
 786             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 787             self.assertEqual(len(d1), 2)
 788
 789             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 790                 self.assertEqual(f.read(), "bar")
 791
 792             assertRegex(self, d1[1],
 793                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 794
 795             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 796                 self.assertEqual(f.read(), "foo")
 797
 798     Test().runTest()
 799
 800 class FuseFileConflictTest(MountTestBase):
 801     def runTest(self):
 802         collection = arvados.collection.Collection(api_client=self.api)
 803         collection.save_new()
 804
 805         m = self.make_mount(fuse.CollectionDirectory)
 806         with llfuse.lock:
 807             m.new_collection(collection.api_response(), collection)
 808
 809         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 810         self.assertEqual([], sorted(d1))
 811
 812         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 813             with collection2.open("file1.txt", "w") as f:
 814                 f.write("foo")
 815
 816         # See note in MountTestBase.setUp
 817         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 818
 819
 820 def fuseUnlinkOpenFileTest(mounttmp):
 821     class Test(unittest.TestCase):
 822         def runTest(self):
 823             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 824                 f.write("foo")
 825
 826                 d1 = llfuse.listdir(os.path.join(mounttmp))
 827                 self.assertEqual(["file1.txt"], sorted(d1))
 828
 829                 os.remove(os.path.join(mounttmp, "file1.txt"))
 830
 831                 d1 = llfuse.listdir(os.path.join(mounttmp))
 832                 self.assertEqual([], sorted(d1))
 833
 834                 f.seek(0)
 835                 self.assertEqual(f.read(), "foo")
 836                 f.write("bar")
 837
 838                 f.seek(0)
 839                 self.assertEqual(f.read(), "foobar")
 840
 841     Test().runTest()
 842
 843 class FuseUnlinkOpenFileTest(MountTestBase):
 844     def runTest(self):
 845         collection = arvados.collection.Collection(api_client=self.api)
 846         collection.save_new()
 847
 848         m = self.make_mount(fuse.CollectionDirectory)
 849         with llfuse.lock:
 850             m.new_collection(collection.api_response(), collection)
 851
 852         # See note in MountTestBase.setUp
 853         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 854
 855         self.assertEqual(collection.manifest_text(), "")
 856
 857
 858 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 859     class Test(unittest.TestCase):
 860         def runTest(self):
 861             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 862                 f.write("Hello world!")
 863
 864             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 865             self.assertEqual(["file1.txt"], sorted(d1))
 866             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 867             self.assertEqual([], sorted(d1))
 868
 869     Test().runTest()
 870
 871 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 872     class Test(unittest.TestCase):
 873         def runTest(self):
 874             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 875
 876             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 877             self.assertEqual([], sorted(d1))
 878             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 879             self.assertEqual(["file2.txt"], sorted(d1))
 880
 881     Test().runTest()
 882
 883 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 884     def runTest(self):
 885         collection1 = arvados.collection.Collection(api_client=self.api)
 886         collection1.save_new()
 887
 888         collection2 = arvados.collection.Collection(api_client=self.api)
 889         collection2.save_new()
 890
 891         m = self.make_mount(fuse.MagicDirectory)
 892
 893         # See note in MountTestBase.setUp
 894         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 895                                                   collection1.manifest_locator(),
 896                                                   collection2.manifest_locator()))
 897
 898         collection1.update()
 899         collection2.update()
 900
 901         assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 902         self.assertEqual(collection2.manifest_text(), "")
 903
 904         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 905                                                   collection1.manifest_locator(),
 906                                                   collection2.manifest_locator()))
 907
 908         collection1.update()
 909         collection2.update()
 910
 911         self.assertEqual(collection1.manifest_text(), "")
 912         assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 913
 914         collection1.stop_threads()
 915         collection2.stop_threads()
 916
 917
 918 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 919     class Test(unittest.TestCase):
 920         def runTest(self):
 921             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 922             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 923                 f.write("Hello world!")
 924
 925             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 926             self.assertEqual(["testdir"], sorted(d1))
 927             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 928             self.assertEqual(["file1.txt"], sorted(d1))
 929
 930             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 931             self.assertEqual([], sorted(d1))
 932
 933     Test().runTest()
 934
 935
 936 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 937     class Test(unittest.TestCase):
 938         def runTest(self):
 939             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 940
 941             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 942             self.assertEqual([], sorted(d1))
 943
 944             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 945             self.assertEqual(["testdir2"], sorted(d1))
 946             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 947             self.assertEqual(["file1.txt"], sorted(d1))
 948
 949             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 950                 self.assertEqual(f.read(), "Hello world!")
 951
 952     Test().runTest()
 953
 954 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 955     def runTest(self):
 956         collection1 = arvados.collection.Collection(api_client=self.api)
 957         collection1.save_new()
 958
 959         collection2 = arvados.collection.Collection(api_client=self.api)
 960         collection2.save_new()
 961
 962         m = self.make_mount(fuse.MagicDirectory)
 963
 964         # See note in MountTestBase.setUp
 965         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 966                                                   collection1.manifest_locator(),
 967                                                   collection2.manifest_locator()))
 968
 969         collection1.update()
 970         collection2.update()
 971
 972         assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 973         self.assertEqual(collection2.manifest_text(), "")
 974
 975         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 976                                                   collection1.manifest_locator(),
 977                                                   collection2.manifest_locator()))
 978
 979         collection1.update()
 980         collection2.update()
 981
 982         self.assertEqual(collection1.manifest_text(), "")
 983         assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 984
 985         collection1.stop_threads()
 986         collection2.stop_threads()
 987
 988 def fuseProjectMkdirTestHelper1(mounttmp):
 989     class Test(unittest.TestCase):
 990         def runTest(self):
 991             os.mkdir(os.path.join(mounttmp, "testcollection"))
 992             with self.assertRaises(OSError):
 993                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 994     Test().runTest()
 995
 996 def fuseProjectMkdirTestHelper2(mounttmp):
 997     class Test(unittest.TestCase):
 998         def runTest(self):
 999             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1000                 f.write("Hello world!")
1001             with self.assertRaises(OSError):
1002                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1003             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1004             with self.assertRaises(OSError):
1005                 os.remove(os.path.join(mounttmp, "testcollection"))
1006             os.rmdir(os.path.join(mounttmp, "testcollection"))
1007     Test().runTest()
1008
1009 class FuseProjectMkdirRmdirTest(MountTestBase):
1010     def runTest(self):
1011         self.make_mount(fuse.ProjectDirectory,
1012                         project_object=self.api.users().current().execute())
1013
1014         d1 = llfuse.listdir(self.mounttmp)
1015         self.assertNotIn('testcollection', d1)
1016
1017         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1018
1019         d1 = llfuse.listdir(self.mounttmp)
1020         self.assertIn('testcollection', d1)
1021
1022         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1023
1024         d1 = llfuse.listdir(self.mounttmp)
1025         self.assertNotIn('testcollection', d1)
1026
1027
1028 def fuseProjectMvTestHelper1(mounttmp):
1029     class Test(unittest.TestCase):
1030         def runTest(self):
1031             d1 = llfuse.listdir(mounttmp)
1032             self.assertNotIn('testcollection', d1)
1033
1034             os.mkdir(os.path.join(mounttmp, "testcollection"))
1035
1036             d1 = llfuse.listdir(mounttmp)
1037             self.assertIn('testcollection', d1)
1038
1039             with self.assertRaises(OSError):
1040                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1041
1042             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1043
1044             d1 = llfuse.listdir(mounttmp)
1045             self.assertNotIn('testcollection', d1)
1046
1047             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1048             self.assertIn('testcollection', d1)
1049
1050     Test().runTest()
1051
1052 class FuseProjectMvTest(MountTestBase):
1053     def runTest(self):
1054         self.make_mount(fuse.ProjectDirectory,
1055                         project_object=self.api.users().current().execute())
1056
1057         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1058
1059
1060 def fuseFsyncTestHelper(mounttmp, k):
1061     class Test(unittest.TestCase):
1062         def runTest(self):
1063             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1064             os.fsync(fd)
1065             os.close(fd)
1066
1067     Test().runTest()
1068
1069 class FuseFsyncTest(FuseMagicTest):
1070     def runTest(self):
1071         self.make_mount(fuse.MagicDirectory)
1072         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1073
1074
1075 class MagicDirApiError(FuseMagicTest):
1076     def setUp(self):
1077         api = mock.MagicMock()
1078         super(MagicDirApiError, self).setUp(api=api)
1079         api.collections().get().execute.side_effect = iter([
1080             Exception('API fail'),
1081             {
1082                 "manifest_text": self.test_manifest,
1083                 "portable_data_hash": self.test_manifest_pdh,
1084             },
1085         ])
1086         api.keep.get.side_effect = Exception('Keep fail')
1087
1088     def runTest(self):
1089         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1090             self.make_mount(fuse.MagicDirectory)
1091
1092             self.operations.inodes.inode_cache.cap = 1
1093             self.operations.inodes.inode_cache.min_entries = 2
1094
1095             with self.assertRaises(OSError):
1096                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1097
1098             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1099
1100
1101 class FuseUnitTest(unittest.TestCase):
1102     def test_sanitize_filename(self):
1103         acceptable = [
1104             "foo.txt",
1105             ".foo",
1106             "..foo",
1107             "...",
1108             "foo...",
1109             "foo..",
1110             "foo.",
1111             "-",
1112             "\x01\x02\x03",
1113             ]
1114         unacceptable = [
1115             "f\00",
1116             "\00\00",
1117             "/foo",
1118             "foo/",
1119             "//",
1120             ]
1121         for f in acceptable:
1122             self.assertEqual(f, fuse.sanitize_filename(f))
1123         for f in unacceptable:
1124             self.assertNotEqual(f, fuse.sanitize_filename(f))
1125             # The sanitized filename should be the same length, though.
1126             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1127         # Special cases
1128         self.assertEqual("_", fuse.sanitize_filename(""))
1129         self.assertEqual("_", fuse.sanitize_filename("."))
1130         self.assertEqual("__", fuse.sanitize_filename(".."))
1131
1132
1133 class FuseMagicTestPDHOnly(MountTestBase):
1134     def setUp(self, api=None):
1135         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1136
1137         cw = arvados.CollectionWriter()
1138
1139         cw.start_new_file('thing1.txt')
1140         cw.write("data 1")
1141
1142         self.testcollection = cw.finish()
1143         self.test_manifest = cw.manifest_text()
1144         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1145         self.testcollectionuuid = str(created['uuid'])
1146
1147     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1148         if skip_pdh_only is True:
1149             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1150         else:
1151             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1152
1153         mount_ls = llfuse.listdir(self.mounttmp)
1154         self.assertIn('README', mount_ls)
1155         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1156                              arvados.util.uuid_pattern.match(fn)
1157                              for fn in mount_ls),
1158                          "new FUSE MagicDirectory lists Collection")
1159
1160         # look up using pdh should succeed in all cases
1161         self.assertDirContents(self.testcollection, ['thing1.txt'])
1162         self.assertDirContents(os.path.join('by_id', self.testcollection),
1163                                ['thing1.txt'])
1164         mount_ls = llfuse.listdir(self.mounttmp)
1165         self.assertIn('README', mount_ls)
1166         self.assertIn(self.testcollection, mount_ls)
1167         self.assertIn(self.testcollection,
1168                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1169
1170         files = {}
1171         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1172
1173         for k, v in viewitems(files):
1174             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1175                 self.assertEqual(v, f.read().decode())
1176
1177         # look up using uuid should fail when pdh_only is set
1178         if pdh_only is True:
1179             with self.assertRaises(OSError):
1180                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1181                                ['thing1.txt'])
1182         else:
1183             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1184                                ['thing1.txt'])
1185
1186     def test_with_pdh_only_true(self):
1187         self.verify_pdh_only(pdh_only=True)
1188
1189     def test_with_pdh_only_false(self):
1190         self.verify_pdh_only(pdh_only=False)
1191
1192     def test_with_default_by_id(self):
1193         self.verify_pdh_only(skip_pdh_only=True)