services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 from __future__ import absolute_import
   6 from future.utils import viewitems
   7 from builtins import str
   8 from builtins import object
   9 from six import assertRegex
  10 import json
  11 import llfuse
  12 import logging
  13 import mock
  14 import os
  15 import subprocess
  16 import time
  17 import unittest
  18
  19 import arvados
  20 import arvados_fuse as fuse
  21 from . import run_test_server
  22
  23 from .integration_test import IntegrationTest
  24 from .mount_test_base import MountTestBase
  25 from .test_tmp_collection import storage_classes_desired
  26
  27 logger = logging.getLogger('arvados.arv-mount')
  28
  29
  30 class AssertWithTimeout(object):
  31     """Allow some time for an assertion to pass."""
  32
  33     def __init__(self, timeout=0):
  34         self.timeout = timeout
  35
  36     def __iter__(self):
  37         self.deadline = time.time() + self.timeout
  38         self.done = False
  39         return self
  40
  41     def __next__(self):
  42         if self.done:
  43             raise StopIteration
  44         return self.attempt
  45
  46     def attempt(self, fn, *args, **kwargs):
  47         try:
  48             fn(*args, **kwargs)
  49         except AssertionError:
  50             if time.time() > self.deadline:
  51                 raise
  52             time.sleep(0.1)
  53         else:
  54             self.done = True
  55
  56
  57 class FuseMountTest(MountTestBase):
  58     def setUp(self):
  59         super(FuseMountTest, self).setUp()
  60
  61         cw = arvados.CollectionWriter()
  62
  63         cw.start_new_file('thing1.txt')
  64         cw.write("data 1")
  65         cw.start_new_file('thing2.txt')
  66         cw.write("data 2")
  67
  68         cw.start_new_stream('dir1')
  69         cw.start_new_file('thing3.txt')
  70         cw.write("data 3")
  71         cw.start_new_file('thing4.txt')
  72         cw.write("data 4")
  73
  74         cw.start_new_stream('dir2')
  75         cw.start_new_file('thing5.txt')
  76         cw.write("data 5")
  77         cw.start_new_file('thing6.txt')
  78         cw.write("data 6")
  79
  80         cw.start_new_stream('dir2/dir3')
  81         cw.start_new_file('thing7.txt')
  82         cw.write("data 7")
  83
  84         cw.start_new_file('thing8.txt')
  85         cw.write("data 8")
  86
  87         cw.start_new_stream('edgecases')
  88         for f in ":/.../-/*/ ".split("/"):
  89             cw.start_new_file(f)
  90             cw.write('x')
  91
  92         for f in ":/.../-/*/ ".split("/"):
  93             cw.start_new_stream('edgecases/dirs/' + f)
  94             cw.start_new_file('x/x')
  95             cw.write('x')
  96
  97         self.testcollection = cw.finish()
  98         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  99
 100     def runTest(self):
 101         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
 102
 103         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 104                                       'edgecases', 'dir1', 'dir2'])
 105         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 106         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 107         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 108         self.assertDirContents('edgecases',
 109                                "dirs/:/.../-/*/ ".split("/"))
 110         self.assertDirContents('edgecases/dirs',
 111                                ":/.../-/*/ ".split("/"))
 112
 113         files = {'thing1.txt': 'data 1',
 114                  'thing2.txt': 'data 2',
 115                  'dir1/thing3.txt': 'data 3',
 116                  'dir1/thing4.txt': 'data 4',
 117                  'dir2/thing5.txt': 'data 5',
 118                  'dir2/thing6.txt': 'data 6',
 119                  'dir2/dir3/thing7.txt': 'data 7',
 120                  'dir2/dir3/thing8.txt': 'data 8'}
 121
 122         for k, v in viewitems(files):
 123             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 124                 self.assertEqual(v, f.read().decode())
 125
 126
 127 class FuseMagicTest(MountTestBase):
 128     def setUp(self, api=None):
 129         super(FuseMagicTest, self).setUp(api=api)
 130
 131         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 132         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
 133         self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
 134         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 135         self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
 136
 137         cw = arvados.CollectionWriter()
 138
 139         cw.start_new_file('thing1.txt')
 140         cw.write("data 1")
 141
 142         self.testcollection = cw.finish()
 143         self.test_manifest = cw.manifest_text()
 144         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 145         self.test_manifest_pdh = coll['portable_data_hash']
 146
 147     def runTest(self):
 148         self.make_mount(fuse.MagicDirectory)
 149
 150         mount_ls = llfuse.listdir(self.mounttmp)
 151         self.assertIn('README', mount_ls)
 152         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 153                              arvados.util.uuid_pattern.match(fn)
 154                              for fn in mount_ls),
 155                          "new FUSE MagicDirectory has no collections or projects")
 156         self.assertDirContents(self.testcollection, ['thing1.txt'])
 157         self.assertDirContents(os.path.join('by_id', self.testcollection),
 158                                ['thing1.txt'])
 159         self.assertIn(self.collection_in_test_project,
 160                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 161         self.assertIn(self.collection_in_test_project,
 162                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 163         self.assertIn(self.collection_in_filter_group,
 164                       llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
 165         self.assertIn(self.collection_in_filter_group,
 166                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
 167
 168
 169         mount_ls = llfuse.listdir(self.mounttmp)
 170         self.assertIn('README', mount_ls)
 171         self.assertIn(self.testcollection, mount_ls)
 172         self.assertIn(self.testcollection,
 173                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 174         self.assertIn(self.test_project, mount_ls)
 175         self.assertIn(self.test_project,
 176                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 177         self.assertIn(self.filter_group,
 178                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 179
 180         with self.assertRaises(OSError):
 181             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 182
 183         files = {}
 184         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 185
 186         for k, v in viewitems(files):
 187             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 188                 self.assertEqual(v, f.read().decode())
 189
 190
 191 class FuseTagsTest(MountTestBase):
 192     def runTest(self):
 193         self.make_mount(fuse.TagsDirectory)
 194
 195         d1 = llfuse.listdir(self.mounttmp)
 196         d1.sort()
 197         self.assertEqual(['foo_tag'], d1)
 198
 199         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 200         d2.sort()
 201         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 202
 203         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 204         d3.sort()
 205         self.assertEqual(['foo'], d3)
 206
 207
 208 class FuseTagsUpdateTest(MountTestBase):
 209     def tag_collection(self, coll_uuid, tag_name):
 210         return self.api.links().create(
 211             body={'link': {'head_uuid': coll_uuid,
 212                            'link_class': 'tag',
 213                            'name': tag_name,
 214         }}).execute()
 215
 216     def runTest(self):
 217         self.make_mount(fuse.TagsDirectory, poll_time=1)
 218
 219         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 220
 221         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 222         self.tag_collection(bar_uuid, 'fuse_test_tag')
 223         for attempt in AssertWithTimeout(10):
 224             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 225         self.assertDirContents('fuse_test_tag', [bar_uuid])
 226
 227         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 228         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 229         for attempt in AssertWithTimeout(10):
 230             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 231
 232         self.api.links().delete(uuid=l['uuid']).execute()
 233         for attempt in AssertWithTimeout(10):
 234             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 235
 236
 237 def fuseSharedTestHelper(mounttmp):
 238     class Test(unittest.TestCase):
 239         def runTest(self):
 240             # Double check that we can open and read objects in this folder as a file,
 241             # and that its contents are what we expect.
 242             baz_path = os.path.join(
 243                 mounttmp,
 244                 'FUSE User',
 245                 'FUSE Test Project',
 246                 'collection in FUSE project',
 247                 'baz')
 248             with open(baz_path) as f:
 249                 self.assertEqual("baz", f.read())
 250
 251             # check mtime on collection
 252             st = os.stat(baz_path)
 253             try:
 254                 mtime = st.st_mtime_ns // 1000000000
 255             except AttributeError:
 256                 mtime = st.st_mtime
 257             self.assertEqual(mtime, 1391448174)
 258
 259             # shared_dirs is a list of the directories exposed
 260             # by fuse.SharedDirectory (i.e. any object visible
 261             # to the current user)
 262             shared_dirs = llfuse.listdir(mounttmp)
 263             shared_dirs.sort()
 264             self.assertIn('FUSE User', shared_dirs)
 265
 266             # fuse_user_objs is a list of the objects owned by the FUSE
 267             # test user (which present as files in the 'FUSE User'
 268             # directory)
 269             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 270             fuse_user_objs.sort()
 271             self.assertEqual(['FUSE Test Project',                    # project owned by user
 272                               'collection #1 owned by FUSE',          # collection owned by user
 273                               'collection #2 owned by FUSE'          # collection owned by user
 274                           ], fuse_user_objs)
 275
 276             # test_proj_files is a list of the files in the FUSE Test Project.
 277             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 278             test_proj_files.sort()
 279             self.assertEqual(['collection in FUSE project'
 280                           ], test_proj_files)
 281
 282
 283     Test().runTest()
 284
 285 class FuseSharedTest(MountTestBase):
 286     def runTest(self):
 287         self.make_mount(fuse.SharedDirectory,
 288                         exclude=self.api.users().current().execute()['uuid'])
 289         keep = arvados.keep.KeepClient()
 290         keep.put("baz".encode())
 291
 292         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 293
 294
 295 class FuseHomeTest(MountTestBase):
 296     def runTest(self):
 297         self.make_mount(fuse.ProjectDirectory,
 298                         project_object=self.api.users().current().execute())
 299
 300         d1 = llfuse.listdir(self.mounttmp)
 301         self.assertIn('Unrestricted public data', d1)
 302
 303         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 304         public_project = run_test_server.fixture('groups')[
 305             'anonymously_accessible_project']
 306         found_in = 0
 307         found_not_in = 0
 308         for name, item in viewitems(run_test_server.fixture('collections')):
 309             if 'name' not in item:
 310                 pass
 311             elif item['owner_uuid'] == public_project['uuid']:
 312                 self.assertIn(item['name'], d2)
 313                 found_in += 1
 314             else:
 315                 # Artificial assumption here: there is no public
 316                 # collection fixture with the same name as a
 317                 # non-public collection.
 318                 self.assertNotIn(item['name'], d2)
 319                 found_not_in += 1
 320         self.assertNotEqual(0, found_in)
 321         self.assertNotEqual(0, found_not_in)
 322
 323         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 324         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 325
 326
 327 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 328     class Test(unittest.TestCase):
 329         def runTest(self):
 330             d1 = llfuse.listdir(mounttmp)
 331             self.assertEqual(["file1.txt"], d1)
 332             with open(os.path.join(mounttmp, "file1.txt")) as f:
 333                 self.assertEqual("blub", f.read())
 334     Test().runTest()
 335
 336 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 337     class Test(unittest.TestCase):
 338         def runTest(self):
 339             d1 = llfuse.listdir(mounttmp)
 340             self.assertEqual(["file1.txt"], d1)
 341             with open(os.path.join(mounttmp, "file1.txt")) as f:
 342                 self.assertEqual("plnp", f.read())
 343     Test().runTest()
 344
 345 class FuseModifyFileTest(MountTestBase):
 346     def runTest(self):
 347         collection = arvados.collection.Collection(api_client=self.api)
 348         with collection.open("file1.txt", "w") as f:
 349             f.write("blub")
 350
 351         collection.save_new()
 352
 353         m = self.make_mount(fuse.CollectionDirectory)
 354         with llfuse.lock:
 355             m.new_collection(collection.api_response(), collection)
 356
 357         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 358
 359         with collection.open("file1.txt", "w") as f:
 360             f.write("plnp")
 361
 362         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 363
 364
 365 class FuseAddFileToCollectionTest(MountTestBase):
 366     def runTest(self):
 367         collection = arvados.collection.Collection(api_client=self.api)
 368         with collection.open("file1.txt", "w") as f:
 369             f.write("blub")
 370
 371         collection.save_new()
 372
 373         m = self.make_mount(fuse.CollectionDirectory)
 374         with llfuse.lock:
 375             m.new_collection(collection.api_response(), collection)
 376
 377         d1 = llfuse.listdir(self.mounttmp)
 378         self.assertEqual(["file1.txt"], d1)
 379
 380         with collection.open("file2.txt", "w") as f:
 381             f.write("plnp")
 382
 383         d1 = llfuse.listdir(self.mounttmp)
 384         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 385
 386
 387 class FuseRemoveFileFromCollectionTest(MountTestBase):
 388     def runTest(self):
 389         collection = arvados.collection.Collection(api_client=self.api)
 390         with collection.open("file1.txt", "w") as f:
 391             f.write("blub")
 392
 393         with collection.open("file2.txt", "w") as f:
 394             f.write("plnp")
 395
 396         collection.save_new()
 397
 398         m = self.make_mount(fuse.CollectionDirectory)
 399         with llfuse.lock:
 400             m.new_collection(collection.api_response(), collection)
 401
 402         d1 = llfuse.listdir(self.mounttmp)
 403         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 404
 405         collection.remove("file2.txt")
 406
 407         d1 = llfuse.listdir(self.mounttmp)
 408         self.assertEqual(["file1.txt"], d1)
 409
 410
 411 def fuseCreateFileTestHelper(mounttmp):
 412     class Test(unittest.TestCase):
 413         def runTest(self):
 414             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 415                 pass
 416     Test().runTest()
 417
 418 class FuseCreateFileTest(MountTestBase):
 419     def runTest(self):
 420         collection = arvados.collection.Collection(api_client=self.api)
 421         collection.save_new()
 422
 423         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 424         self.assertEqual(collection2["manifest_text"], "")
 425
 426         collection.save_new()
 427
 428         m = self.make_mount(fuse.CollectionDirectory)
 429         with llfuse.lock:
 430             m.new_collection(collection.api_response(), collection)
 431         self.assertTrue(m.writable())
 432
 433         self.assertNotIn("file1.txt", collection)
 434
 435         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 436
 437         self.assertIn("file1.txt", collection)
 438
 439         d1 = llfuse.listdir(self.mounttmp)
 440         self.assertEqual(["file1.txt"], d1)
 441
 442         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 443         assertRegex(self, collection2["manifest_text"],
 444             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 445
 446
 447 def fuseWriteFileTestHelperWriteFile(mounttmp):
 448     class Test(unittest.TestCase):
 449         def runTest(self):
 450             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 451                 f.write("Hello world!")
 452     Test().runTest()
 453
 454 def fuseWriteFileTestHelperReadFile(mounttmp):
 455     class Test(unittest.TestCase):
 456         def runTest(self):
 457             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 458                 self.assertEqual(f.read(), "Hello world!")
 459     Test().runTest()
 460
 461 class FuseWriteFileTest(MountTestBase):
 462     def runTest(self):
 463         collection = arvados.collection.Collection(api_client=self.api)
 464         collection.save_new()
 465
 466         m = self.make_mount(fuse.CollectionDirectory)
 467         with llfuse.lock:
 468             m.new_collection(collection.api_response(), collection)
 469         self.assertTrue(m.writable())
 470
 471         self.assertNotIn("file1.txt", collection)
 472
 473         self.assertEqual(0, self.operations.write_counter.get())
 474         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 475         self.assertEqual(12, self.operations.write_counter.get())
 476
 477         with collection.open("file1.txt") as f:
 478             self.assertEqual(f.read(), "Hello world!")
 479
 480         self.assertEqual(0, self.operations.read_counter.get())
 481         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 482         self.assertEqual(12, self.operations.read_counter.get())
 483
 484         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 485         assertRegex(self, collection2["manifest_text"],
 486             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 487
 488
 489 def fuseUpdateFileTestHelper(mounttmp):
 490     class Test(unittest.TestCase):
 491         def runTest(self):
 492             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 493                 f.write("Hello world!")
 494
 495             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 496                 fr = f.read()
 497                 self.assertEqual(fr, "Hello world!")
 498                 f.seek(0)
 499                 f.write("Hola mundo!")
 500                 f.seek(0)
 501                 fr = f.read()
 502                 self.assertEqual(fr, "Hola mundo!!")
 503
 504             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 505                 self.assertEqual(f.read(), "Hola mundo!!")
 506
 507     Test().runTest()
 508
 509 class FuseUpdateFileTest(MountTestBase):
 510     def runTest(self):
 511         collection = arvados.collection.Collection(api_client=self.api)
 512         collection.save_new()
 513
 514         m = self.make_mount(fuse.CollectionDirectory)
 515         with llfuse.lock:
 516             m.new_collection(collection.api_response(), collection)
 517         self.assertTrue(m.writable())
 518
 519         # See note in MountTestBase.setUp
 520         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 521
 522         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 523         assertRegex(self, collection2["manifest_text"],
 524             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 525
 526
 527 def fuseMkdirTestHelper(mounttmp):
 528     class Test(unittest.TestCase):
 529         def runTest(self):
 530             with self.assertRaises(IOError):
 531                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 532                     f.write("Hello world!")
 533
 534             os.mkdir(os.path.join(mounttmp, "testdir"))
 535
 536             with self.assertRaises(OSError):
 537                 os.mkdir(os.path.join(mounttmp, "testdir"))
 538
 539             d1 = llfuse.listdir(mounttmp)
 540             self.assertEqual(["testdir"], d1)
 541
 542             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 543                 f.write("Hello world!")
 544
 545             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 546             self.assertEqual(["file1.txt"], d1)
 547
 548     Test().runTest()
 549
 550 class FuseMkdirTest(MountTestBase):
 551     def runTest(self):
 552         collection = arvados.collection.Collection(api_client=self.api)
 553         collection.save_new()
 554
 555         m = self.make_mount(fuse.CollectionDirectory)
 556         with llfuse.lock:
 557             m.new_collection(collection.api_response(), collection)
 558         self.assertTrue(m.writable())
 559
 560         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 561
 562         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 563         assertRegex(self, collection2["manifest_text"],
 564             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 565
 566
 567 def fuseRmTestHelperWriteFile(mounttmp):
 568     class Test(unittest.TestCase):
 569         def runTest(self):
 570             os.mkdir(os.path.join(mounttmp, "testdir"))
 571
 572             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 573                 f.write("Hello world!")
 574
 575     Test().runTest()
 576
 577 def fuseRmTestHelperDeleteFile(mounttmp):
 578     class Test(unittest.TestCase):
 579         def runTest(self):
 580             # Can't delete because it's not empty
 581             with self.assertRaises(OSError):
 582                 os.rmdir(os.path.join(mounttmp, "testdir"))
 583
 584             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 585             self.assertEqual(["file1.txt"], d1)
 586
 587             # Delete file
 588             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 589
 590             # Make sure it's empty
 591             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 592             self.assertEqual([], d1)
 593
 594             # Try to delete it again
 595             with self.assertRaises(OSError):
 596                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 597
 598     Test().runTest()
 599
 600 def fuseRmTestHelperRmdir(mounttmp):
 601     class Test(unittest.TestCase):
 602         def runTest(self):
 603             # Should be able to delete now that it is empty
 604             os.rmdir(os.path.join(mounttmp, "testdir"))
 605
 606             # Make sure it's empty
 607             d1 = llfuse.listdir(os.path.join(mounttmp))
 608             self.assertEqual([], d1)
 609
 610             # Try to delete it again
 611             with self.assertRaises(OSError):
 612                 os.rmdir(os.path.join(mounttmp, "testdir"))
 613
 614     Test().runTest()
 615
 616 class FuseRmTest(MountTestBase):
 617     def runTest(self):
 618         collection = arvados.collection.Collection(api_client=self.api)
 619         collection.save_new()
 620
 621         m = self.make_mount(fuse.CollectionDirectory)
 622         with llfuse.lock:
 623             m.new_collection(collection.api_response(), collection)
 624         self.assertTrue(m.writable())
 625
 626         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 627
 628         # Starting manifest
 629         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 630         assertRegex(self, collection2["manifest_text"],
 631             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 632         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 633
 634         # Empty directories are represented by an empty file named "."
 635         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 636         assertRegex(self, collection2["manifest_text"],
 637                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 638
 639         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 640
 641         # manifest should be empty now.
 642         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 643         self.assertEqual(collection2["manifest_text"], "")
 644
 645
 646 def fuseMvFileTestHelperWriteFile(mounttmp):
 647     class Test(unittest.TestCase):
 648         def runTest(self):
 649             os.mkdir(os.path.join(mounttmp, "testdir"))
 650
 651             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 652                 f.write("Hello world!")
 653
 654     Test().runTest()
 655
 656 def fuseMvFileTestHelperMoveFile(mounttmp):
 657     class Test(unittest.TestCase):
 658         def runTest(self):
 659             d1 = llfuse.listdir(os.path.join(mounttmp))
 660             self.assertEqual(["testdir"], d1)
 661             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 662             self.assertEqual(["file1.txt"], d1)
 663
 664             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 665
 666             d1 = llfuse.listdir(os.path.join(mounttmp))
 667             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 668             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 669             self.assertEqual([], d1)
 670
 671     Test().runTest()
 672
 673 class FuseMvFileTest(MountTestBase):
 674     def runTest(self):
 675         collection = arvados.collection.Collection(api_client=self.api)
 676         collection.save_new()
 677
 678         m = self.make_mount(fuse.CollectionDirectory)
 679         with llfuse.lock:
 680             m.new_collection(collection.api_response(), collection)
 681         self.assertTrue(m.writable())
 682
 683         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 684
 685         # Starting manifest
 686         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 687         assertRegex(self, collection2["manifest_text"],
 688             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 689
 690         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 691
 692         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 693         assertRegex(self, collection2["manifest_text"],
 694             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 695
 696
 697 def fuseRenameTestHelper(mounttmp):
 698     class Test(unittest.TestCase):
 699         def runTest(self):
 700             os.mkdir(os.path.join(mounttmp, "testdir"))
 701
 702             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 703                 f.write("Hello world!")
 704
 705     Test().runTest()
 706
 707 class FuseRenameTest(MountTestBase):
 708     def runTest(self):
 709         collection = arvados.collection.Collection(api_client=self.api)
 710         collection.save_new()
 711
 712         m = self.make_mount(fuse.CollectionDirectory)
 713         with llfuse.lock:
 714             m.new_collection(collection.api_response(), collection)
 715         self.assertTrue(m.writable())
 716
 717         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 718
 719         # Starting manifest
 720         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 721         assertRegex(self, collection2["manifest_text"],
 722             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 723
 724         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 725         self.assertEqual(["testdir"], d1)
 726         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 727         self.assertEqual(["file1.txt"], d1)
 728
 729         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 730
 731         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 732         self.assertEqual(["testdir2"], sorted(d1))
 733         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 734         self.assertEqual(["file1.txt"], d1)
 735
 736         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 737         assertRegex(self, collection2["manifest_text"],
 738             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 739
 740
 741 class FuseUpdateFromEventTest(MountTestBase):
 742     def runTest(self):
 743         collection = arvados.collection.Collection(api_client=self.api)
 744         collection.save_new()
 745
 746         m = self.make_mount(fuse.CollectionDirectory)
 747         with llfuse.lock:
 748             m.new_collection(collection.api_response(), collection)
 749
 750         self.operations.listen_for_events()
 751
 752         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 753         self.assertEqual([], sorted(d1))
 754
 755         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 756             with collection2.open("file1.txt", "w") as f:
 757                 f.write("foo")
 758
 759         for attempt in AssertWithTimeout(10):
 760             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 761
 762
 763 class FuseDeleteProjectEventTest(MountTestBase):
 764     def runTest(self):
 765
 766         aproject = self.api.groups().create(body={
 767             "name": "aproject",
 768             "group_class": "project"
 769         }).execute()
 770
 771         bproject = self.api.groups().create(body={
 772             "name": "bproject",
 773             "group_class": "project",
 774             "owner_uuid": aproject["uuid"]
 775         }).execute()
 776
 777         self.make_mount(fuse.ProjectDirectory,
 778                         project_object=self.api.users().current().execute())
 779
 780         self.operations.listen_for_events()
 781
 782         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 783         self.assertEqual(["bproject"], sorted(d1))
 784
 785         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 786
 787         for attempt in AssertWithTimeout(10):
 788             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 789
 790
 791 def fuseFileConflictTestHelper(mounttmp):
 792     class Test(unittest.TestCase):
 793         def runTest(self):
 794             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 795                 f.write("bar")
 796
 797             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 798             self.assertEqual(len(d1), 2)
 799
 800             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 801                 self.assertEqual(f.read(), "bar")
 802
 803             assertRegex(self, d1[1],
 804                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 805
 806             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 807                 self.assertEqual(f.read(), "foo")
 808
 809     Test().runTest()
 810
 811 class FuseFileConflictTest(MountTestBase):
 812     def runTest(self):
 813         collection = arvados.collection.Collection(api_client=self.api)
 814         collection.save_new()
 815
 816         m = self.make_mount(fuse.CollectionDirectory)
 817         with llfuse.lock:
 818             m.new_collection(collection.api_response(), collection)
 819
 820         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 821         self.assertEqual([], sorted(d1))
 822
 823         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 824             with collection2.open("file1.txt", "w") as f:
 825                 f.write("foo")
 826
 827         # See note in MountTestBase.setUp
 828         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 829
 830
 831 def fuseUnlinkOpenFileTest(mounttmp):
 832     class Test(unittest.TestCase):
 833         def runTest(self):
 834             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 835                 f.write("foo")
 836
 837                 d1 = llfuse.listdir(os.path.join(mounttmp))
 838                 self.assertEqual(["file1.txt"], sorted(d1))
 839
 840                 os.remove(os.path.join(mounttmp, "file1.txt"))
 841
 842                 d1 = llfuse.listdir(os.path.join(mounttmp))
 843                 self.assertEqual([], sorted(d1))
 844
 845                 f.seek(0)
 846                 self.assertEqual(f.read(), "foo")
 847                 f.write("bar")
 848
 849                 f.seek(0)
 850                 self.assertEqual(f.read(), "foobar")
 851
 852     Test().runTest()
 853
 854 class FuseUnlinkOpenFileTest(MountTestBase):
 855     def runTest(self):
 856         collection = arvados.collection.Collection(api_client=self.api)
 857         collection.save_new()
 858
 859         m = self.make_mount(fuse.CollectionDirectory)
 860         with llfuse.lock:
 861             m.new_collection(collection.api_response(), collection)
 862
 863         # See note in MountTestBase.setUp
 864         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 865
 866         self.assertEqual(collection.manifest_text(), "")
 867
 868
 869 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 870     class Test(unittest.TestCase):
 871         def runTest(self):
 872             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 873                 f.write("Hello world!")
 874
 875             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 876             self.assertEqual(["file1.txt"], sorted(d1))
 877             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 878             self.assertEqual([], sorted(d1))
 879
 880     Test().runTest()
 881
 882 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 883     class Test(unittest.TestCase):
 884         def runTest(self):
 885             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 886
 887             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 888             self.assertEqual([], sorted(d1))
 889             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 890             self.assertEqual(["file2.txt"], sorted(d1))
 891
 892     Test().runTest()
 893
 894 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 895     def runTest(self):
 896         collection1 = arvados.collection.Collection(api_client=self.api)
 897         collection1.save_new()
 898
 899         collection2 = arvados.collection.Collection(api_client=self.api)
 900         collection2.save_new()
 901
 902         m = self.make_mount(fuse.MagicDirectory)
 903
 904         # See note in MountTestBase.setUp
 905         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 906                                                   collection1.manifest_locator(),
 907                                                   collection2.manifest_locator()))
 908
 909         collection1.update()
 910         collection2.update()
 911
 912         assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 913         self.assertEqual(collection2.manifest_text(), "")
 914
 915         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 916                                                   collection1.manifest_locator(),
 917                                                   collection2.manifest_locator()))
 918
 919         collection1.update()
 920         collection2.update()
 921
 922         self.assertEqual(collection1.manifest_text(), "")
 923         assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 924
 925         collection1.stop_threads()
 926         collection2.stop_threads()
 927
 928
 929 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 930     class Test(unittest.TestCase):
 931         def runTest(self):
 932             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 933             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 934                 f.write("Hello world!")
 935
 936             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 937             self.assertEqual(["testdir"], sorted(d1))
 938             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 939             self.assertEqual(["file1.txt"], sorted(d1))
 940
 941             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 942             self.assertEqual([], sorted(d1))
 943
 944     Test().runTest()
 945
 946
 947 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 948     class Test(unittest.TestCase):
 949         def runTest(self):
 950             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 951
 952             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 953             self.assertEqual([], sorted(d1))
 954
 955             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 956             self.assertEqual(["testdir2"], sorted(d1))
 957             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 958             self.assertEqual(["file1.txt"], sorted(d1))
 959
 960             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 961                 self.assertEqual(f.read(), "Hello world!")
 962
 963     Test().runTest()
 964
 965 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 966     def runTest(self):
 967         collection1 = arvados.collection.Collection(api_client=self.api)
 968         collection1.save_new()
 969
 970         collection2 = arvados.collection.Collection(api_client=self.api)
 971         collection2.save_new()
 972
 973         m = self.make_mount(fuse.MagicDirectory)
 974
 975         # See note in MountTestBase.setUp
 976         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 977                                                   collection1.manifest_locator(),
 978                                                   collection2.manifest_locator()))
 979
 980         collection1.update()
 981         collection2.update()
 982
 983         assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 984         self.assertEqual(collection2.manifest_text(), "")
 985
 986         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 987                                                   collection1.manifest_locator(),
 988                                                   collection2.manifest_locator()))
 989
 990         collection1.update()
 991         collection2.update()
 992
 993         self.assertEqual(collection1.manifest_text(), "")
 994         assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 995
 996         collection1.stop_threads()
 997         collection2.stop_threads()
 998
 999 def fuseProjectMkdirTestHelper1(mounttmp):
1000     class Test(unittest.TestCase):
1001         def runTest(self):
1002             os.mkdir(os.path.join(mounttmp, "testcollection"))
1003             with self.assertRaises(OSError):
1004                 os.mkdir(os.path.join(mounttmp, "testcollection"))
1005     Test().runTest()
1006
1007 def fuseProjectMkdirTestHelper2(mounttmp):
1008     class Test(unittest.TestCase):
1009         def runTest(self):
1010             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1011                 f.write("Hello world!")
1012             with self.assertRaises(OSError):
1013                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1014             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1015             with self.assertRaises(OSError):
1016                 os.remove(os.path.join(mounttmp, "testcollection"))
1017             os.rmdir(os.path.join(mounttmp, "testcollection"))
1018     Test().runTest()
1019
1020 class FuseProjectMkdirRmdirTest(MountTestBase):
1021     def runTest(self):
1022         self.make_mount(fuse.ProjectDirectory,
1023                         project_object=self.api.users().current().execute())
1024
1025         d1 = llfuse.listdir(self.mounttmp)
1026         self.assertNotIn('testcollection', d1)
1027
1028         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1029
1030         d1 = llfuse.listdir(self.mounttmp)
1031         self.assertIn('testcollection', d1)
1032
1033         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1034
1035         d1 = llfuse.listdir(self.mounttmp)
1036         self.assertNotIn('testcollection', d1)
1037
1038
1039 def fuseProjectMvTestHelper1(mounttmp):
1040     class Test(unittest.TestCase):
1041         def runTest(self):
1042             d1 = llfuse.listdir(mounttmp)
1043             self.assertNotIn('testcollection', d1)
1044
1045             os.mkdir(os.path.join(mounttmp, "testcollection"))
1046
1047             d1 = llfuse.listdir(mounttmp)
1048             self.assertIn('testcollection', d1)
1049
1050             with self.assertRaises(OSError):
1051                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1052
1053             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1054
1055             d1 = llfuse.listdir(mounttmp)
1056             self.assertNotIn('testcollection', d1)
1057
1058             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1059             self.assertIn('testcollection', d1)
1060
1061     Test().runTest()
1062
1063 class FuseProjectMvTest(MountTestBase):
1064     def runTest(self):
1065         self.make_mount(fuse.ProjectDirectory,
1066                         project_object=self.api.users().current().execute())
1067
1068         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1069
1070
1071 def fuseFsyncTestHelper(mounttmp, k):
1072     class Test(unittest.TestCase):
1073         def runTest(self):
1074             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1075             os.fsync(fd)
1076             os.close(fd)
1077
1078     Test().runTest()
1079
1080 class FuseFsyncTest(FuseMagicTest):
1081     def runTest(self):
1082         self.make_mount(fuse.MagicDirectory)
1083         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1084
1085
1086 class MagicDirApiError(FuseMagicTest):
1087     def setUp(self):
1088         api = mock.MagicMock()
1089         super(MagicDirApiError, self).setUp(api=api)
1090         api.collections().get().execute.side_effect = iter([
1091             Exception('API fail'),
1092             {
1093                 "manifest_text": self.test_manifest,
1094                 "portable_data_hash": self.test_manifest_pdh,
1095             },
1096         ])
1097         api.keep.get.side_effect = Exception('Keep fail')
1098
1099     def runTest(self):
1100         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1101             self.make_mount(fuse.MagicDirectory)
1102
1103             self.operations.inodes.inode_cache.cap = 1
1104             self.operations.inodes.inode_cache.min_entries = 2
1105
1106             with self.assertRaises(OSError):
1107                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1108
1109             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1110
1111
1112 class SanitizeFilenameTest(MountTestBase):
1113     def test_sanitize_filename(self):
1114         pdir = fuse.ProjectDirectory(1, {}, self.api, 0, project_object=self.api.users().current().execute())
1115         acceptable = [
1116             "foo.txt",
1117             ".foo",
1118             "..foo",
1119             "...",
1120             "foo...",
1121             "foo..",
1122             "foo.",
1123             "-",
1124             "\x01\x02\x03",
1125             ]
1126         unacceptable = [
1127             "f\00",
1128             "\00\00",
1129             "/foo",
1130             "foo/",
1131             "//",
1132             ]
1133         for f in acceptable:
1134             self.assertEqual(f, pdir.sanitize_filename(f))
1135         for f in unacceptable:
1136             self.assertNotEqual(f, pdir.sanitize_filename(f))
1137             # The sanitized filename should be the same length, though.
1138             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1139         # Special cases
1140         self.assertEqual("_", pdir.sanitize_filename(""))
1141         self.assertEqual("_", pdir.sanitize_filename("."))
1142         self.assertEqual("__", pdir.sanitize_filename(".."))
1143
1144
1145 class FuseMagicTestPDHOnly(MountTestBase):
1146     def setUp(self, api=None):
1147         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1148
1149         cw = arvados.CollectionWriter()
1150
1151         cw.start_new_file('thing1.txt')
1152         cw.write("data 1")
1153
1154         self.testcollection = cw.finish()
1155         self.test_manifest = cw.manifest_text()
1156         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1157         self.testcollectionuuid = str(created['uuid'])
1158
1159     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1160         if skip_pdh_only is True:
1161             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1162         else:
1163             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1164
1165         mount_ls = llfuse.listdir(self.mounttmp)
1166         self.assertIn('README', mount_ls)
1167         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1168                              arvados.util.uuid_pattern.match(fn)
1169                              for fn in mount_ls),
1170                          "new FUSE MagicDirectory lists Collection")
1171
1172         # look up using pdh should succeed in all cases
1173         self.assertDirContents(self.testcollection, ['thing1.txt'])
1174         self.assertDirContents(os.path.join('by_id', self.testcollection),
1175                                ['thing1.txt'])
1176         mount_ls = llfuse.listdir(self.mounttmp)
1177         self.assertIn('README', mount_ls)
1178         self.assertIn(self.testcollection, mount_ls)
1179         self.assertIn(self.testcollection,
1180                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1181
1182         files = {}
1183         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1184
1185         for k, v in viewitems(files):
1186             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1187                 self.assertEqual(v, f.read().decode())
1188
1189         # look up using uuid should fail when pdh_only is set
1190         if pdh_only is True:
1191             with self.assertRaises(OSError):
1192                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1193                                ['thing1.txt'])
1194         else:
1195             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1196                                ['thing1.txt'])
1197
1198     def test_with_pdh_only_true(self):
1199         self.verify_pdh_only(pdh_only=True)
1200
1201     def test_with_pdh_only_false(self):
1202         self.verify_pdh_only(pdh_only=False)
1203
1204     def test_with_default_by_id(self):
1205         self.verify_pdh_only(skip_pdh_only=True)
1206
1207
1208 class SlashSubstitutionTest(IntegrationTest):
1209     mnt_args = [
1210         '--read-write',
1211         '--mount-home', 'zzz',
1212     ]
1213
1214     def setUp(self):
1215         super(SlashSubstitutionTest, self).setUp()
1216         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1217         self.api.config = lambda: {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1218         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1219         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1220         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1221
1222     @IntegrationTest.mount(argv=mnt_args)
1223     @mock.patch('arvados.util.get_config_once')
1224     def test_slash_substitution_before_listing(self, get_config_once):
1225         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1226         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1227         self.checkContents()
1228     @staticmethod
1229     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1230         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1231             f.write('xxx')
1232         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1233             f.write('foo')
1234
1235     @IntegrationTest.mount(argv=mnt_args)
1236     @mock.patch('arvados.util.get_config_once')
1237     def test_slash_substitution_after_listing(self, get_config_once):
1238         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1239         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1240         self.checkContents()
1241     @staticmethod
1242     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1243         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1244             f.write('xxx')
1245         os.listdir(tmpdir)
1246         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1247             f.write('foo')
1248
1249     def checkContents(self):
1250         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1251         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1252
1253     @IntegrationTest.mount(argv=mnt_args)
1254     @mock.patch('arvados.util.get_config_once')
1255     def test_slash_substitution_conflict(self, get_config_once):
1256         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1257         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1258         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1259         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1260         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1261         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1262     @staticmethod
1263     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1264         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1265             f.write('foo')
1266
1267 class StorageClassesTest(IntegrationTest):
1268     mnt_args = [
1269         '--read-write',
1270         '--mount-home', 'homedir',
1271     ]
1272
1273     def setUp(self):
1274         super(StorageClassesTest, self).setUp()
1275         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1276
1277     @IntegrationTest.mount(argv=mnt_args)
1278     def test_collection_default_storage_classes(self):
1279         coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
1280         self.api.collections().create(body={'name':'a_collection'}).execute()
1281         self.pool_test(coll_path)
1282     @staticmethod
1283     def _test_collection_default_storage_classes(self, coll):
1284         self.assertEqual(storage_classes_desired(coll), ['default'])
1285
1286     @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
1287     def test_collection_custom_storage_classes(self):
1288         coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
1289         os.mkdir(coll_path)
1290         self.pool_test(coll_path)
1291     @staticmethod
1292     def _test_collection_custom_storage_classes(self, coll):
1293         self.assertEqual(storage_classes_desired(coll), ['foo'])