services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 from __future__ import absolute_import
   6 from future.utils import viewitems
   7 from builtins import str
   8 from builtins import object
   9 from six import assertRegex
  10 import json
  11 import llfuse
  12 import logging
  13 import mock
  14 import os
  15 import subprocess
  16 import time
  17 import unittest
  18 import tempfile
  19
  20 import arvados
  21 import arvados_fuse as fuse
  22 from . import run_test_server
  23
  24 from .integration_test import IntegrationTest
  25 from .mount_test_base import MountTestBase
  26 from .test_tmp_collection import storage_classes_desired
  27
  28 logger = logging.getLogger('arvados.arv-mount')
  29
  30
  31 class AssertWithTimeout(object):
  32     """Allow some time for an assertion to pass."""
  33
  34     def __init__(self, timeout=0):
  35         self.timeout = timeout
  36
  37     def __iter__(self):
  38         self.deadline = time.time() + self.timeout
  39         self.done = False
  40         return self
  41
  42     def __next__(self):
  43         if self.done:
  44             raise StopIteration
  45         return self.attempt
  46
  47     def attempt(self, fn, *args, **kwargs):
  48         try:
  49             fn(*args, **kwargs)
  50         except AssertionError:
  51             if time.time() > self.deadline:
  52                 raise
  53             time.sleep(0.1)
  54         else:
  55             self.done = True
  56
  57
  58 class FuseMountTest(MountTestBase):
  59     def setUp(self):
  60         super(FuseMountTest, self).setUp()
  61
  62         cw = arvados.CollectionWriter()
  63
  64         cw.start_new_file('thing1.txt')
  65         cw.write("data 1")
  66         cw.start_new_file('thing2.txt')
  67         cw.write("data 2")
  68
  69         cw.start_new_stream('dir1')
  70         cw.start_new_file('thing3.txt')
  71         cw.write("data 3")
  72         cw.start_new_file('thing4.txt')
  73         cw.write("data 4")
  74
  75         cw.start_new_stream('dir2')
  76         cw.start_new_file('thing5.txt')
  77         cw.write("data 5")
  78         cw.start_new_file('thing6.txt')
  79         cw.write("data 6")
  80
  81         cw.start_new_stream('dir2/dir3')
  82         cw.start_new_file('thing7.txt')
  83         cw.write("data 7")
  84
  85         cw.start_new_file('thing8.txt')
  86         cw.write("data 8")
  87
  88         cw.start_new_stream('edgecases')
  89         for f in ":/.../-/*/ ".split("/"):
  90             cw.start_new_file(f)
  91             cw.write('x')
  92
  93         for f in ":/.../-/*/ ".split("/"):
  94             cw.start_new_stream('edgecases/dirs/' + f)
  95             cw.start_new_file('x/x')
  96             cw.write('x')
  97
  98         self.testcollection = cw.finish()
  99         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
 100
 101     def runTest(self):
 102         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
 103
 104         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 105                                       'edgecases', 'dir1', 'dir2'])
 106         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 107         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 108         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 109         self.assertDirContents('edgecases',
 110                                "dirs/:/.../-/*/ ".split("/"))
 111         self.assertDirContents('edgecases/dirs',
 112                                ":/.../-/*/ ".split("/"))
 113
 114         files = {'thing1.txt': 'data 1',
 115                  'thing2.txt': 'data 2',
 116                  'dir1/thing3.txt': 'data 3',
 117                  'dir1/thing4.txt': 'data 4',
 118                  'dir2/thing5.txt': 'data 5',
 119                  'dir2/thing6.txt': 'data 6',
 120                  'dir2/dir3/thing7.txt': 'data 7',
 121                  'dir2/dir3/thing8.txt': 'data 8'}
 122
 123         for k, v in viewitems(files):
 124             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 125                 self.assertEqual(v, f.read().decode())
 126
 127
 128 class FuseMagicTest(MountTestBase):
 129     def setUp(self, api=None):
 130         super(FuseMagicTest, self).setUp(api=api)
 131
 132         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 133         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
 134         self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
 135         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 136         self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
 137
 138         cw = arvados.CollectionWriter()
 139
 140         cw.start_new_file('thing1.txt')
 141         cw.write("data 1")
 142
 143         self.testcollection = cw.finish()
 144         self.test_manifest = cw.manifest_text()
 145         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 146         self.test_manifest_pdh = coll['portable_data_hash']
 147
 148     def runTest(self):
 149         self.make_mount(fuse.MagicDirectory)
 150
 151         mount_ls = llfuse.listdir(self.mounttmp)
 152         self.assertIn('README', mount_ls)
 153         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 154                              arvados.util.uuid_pattern.match(fn)
 155                              for fn in mount_ls),
 156                          "new FUSE MagicDirectory has no collections or projects")
 157         self.assertDirContents(self.testcollection, ['thing1.txt'])
 158         self.assertDirContents(os.path.join('by_id', self.testcollection),
 159                                ['thing1.txt'])
 160         self.assertIn(self.collection_in_test_project,
 161                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 162         self.assertIn(self.collection_in_test_project,
 163                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 164         self.assertIn(self.collection_in_filter_group,
 165                       llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
 166         self.assertIn(self.collection_in_filter_group,
 167                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
 168
 169
 170         mount_ls = llfuse.listdir(self.mounttmp)
 171         self.assertIn('README', mount_ls)
 172         self.assertIn(self.testcollection, mount_ls)
 173         self.assertIn(self.testcollection,
 174                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 175         self.assertIn(self.test_project, mount_ls)
 176         self.assertIn(self.test_project,
 177                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 178         self.assertIn(self.filter_group,
 179                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 180
 181         with self.assertRaises(OSError):
 182             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 183
 184         files = {}
 185         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 186
 187         for k, v in viewitems(files):
 188             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 189                 self.assertEqual(v, f.read().decode())
 190
 191
 192 class FuseTagsTest(MountTestBase):
 193     def runTest(self):
 194         self.make_mount(fuse.TagsDirectory)
 195
 196         d1 = llfuse.listdir(self.mounttmp)
 197         d1.sort()
 198         self.assertEqual(['foo_tag'], d1)
 199
 200         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 201         d2.sort()
 202         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 203
 204         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 205         d3.sort()
 206         self.assertEqual(['foo'], d3)
 207
 208
 209 class FuseTagsUpdateTest(MountTestBase):
 210     def tag_collection(self, coll_uuid, tag_name):
 211         return self.api.links().create(
 212             body={'link': {'head_uuid': coll_uuid,
 213                            'link_class': 'tag',
 214                            'name': tag_name,
 215         }}).execute()
 216
 217     def runTest(self):
 218         self.make_mount(fuse.TagsDirectory, poll_time=1)
 219
 220         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 221
 222         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 223         self.tag_collection(bar_uuid, 'fuse_test_tag')
 224         for attempt in AssertWithTimeout(10):
 225             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 226         self.assertDirContents('fuse_test_tag', [bar_uuid])
 227
 228         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 229         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 230         for attempt in AssertWithTimeout(10):
 231             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 232
 233         self.api.links().delete(uuid=l['uuid']).execute()
 234         for attempt in AssertWithTimeout(10):
 235             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 236
 237
 238 def fuseSharedTestHelper(mounttmp):
 239     class Test(unittest.TestCase):
 240         def runTest(self):
 241             # Double check that we can open and read objects in this folder as a file,
 242             # and that its contents are what we expect.
 243             baz_path = os.path.join(
 244                 mounttmp,
 245                 'FUSE User',
 246                 'FUSE Test Project',
 247                 'collection in FUSE project',
 248                 'baz')
 249             with open(baz_path) as f:
 250                 self.assertEqual("baz", f.read())
 251
 252             # check mtime on collection
 253             st = os.stat(baz_path)
 254             try:
 255                 mtime = st.st_mtime_ns // 1000000000
 256             except AttributeError:
 257                 mtime = st.st_mtime
 258             self.assertEqual(mtime, 1391448174)
 259
 260             # shared_dirs is a list of the directories exposed
 261             # by fuse.SharedDirectory (i.e. any object visible
 262             # to the current user)
 263             shared_dirs = llfuse.listdir(mounttmp)
 264             shared_dirs.sort()
 265             self.assertIn('FUSE User', shared_dirs)
 266
 267             # fuse_user_objs is a list of the objects owned by the FUSE
 268             # test user (which present as files in the 'FUSE User'
 269             # directory)
 270             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 271             fuse_user_objs.sort()
 272             self.assertEqual(['FUSE Test Project',                    # project owned by user
 273                               'collection #1 owned by FUSE',          # collection owned by user
 274                               'collection #2 owned by FUSE'          # collection owned by user
 275                           ], fuse_user_objs)
 276
 277             # test_proj_files is a list of the files in the FUSE Test Project.
 278             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 279             test_proj_files.sort()
 280             self.assertEqual(['collection in FUSE project'
 281                           ], test_proj_files)
 282
 283
 284     Test().runTest()
 285
 286 class FuseSharedTest(MountTestBase):
 287     def runTest(self):
 288         self.make_mount(fuse.SharedDirectory,
 289                         exclude=self.api.users().current().execute()['uuid'])
 290         keep = arvados.keep.KeepClient()
 291         keep.put("baz".encode())
 292
 293         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 294
 295
 296 class FuseHomeTest(MountTestBase):
 297     def runTest(self):
 298         self.make_mount(fuse.ProjectDirectory,
 299                         project_object=self.api.users().current().execute())
 300
 301         d1 = llfuse.listdir(self.mounttmp)
 302         self.assertIn('Unrestricted public data', d1)
 303
 304         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 305         public_project = run_test_server.fixture('groups')[
 306             'anonymously_accessible_project']
 307         found_in = 0
 308         found_not_in = 0
 309         for name, item in viewitems(run_test_server.fixture('collections')):
 310             if 'name' not in item:
 311                 pass
 312             elif item['owner_uuid'] == public_project['uuid']:
 313                 self.assertIn(item['name'], d2)
 314                 found_in += 1
 315             else:
 316                 # Artificial assumption here: there is no public
 317                 # collection fixture with the same name as a
 318                 # non-public collection.
 319                 self.assertNotIn(item['name'], d2)
 320                 found_not_in += 1
 321         self.assertNotEqual(0, found_in)
 322         self.assertNotEqual(0, found_not_in)
 323
 324         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 325         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 326
 327
 328 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 329     class Test(unittest.TestCase):
 330         def runTest(self):
 331             d1 = llfuse.listdir(mounttmp)
 332             self.assertEqual(["file1.txt"], d1)
 333             with open(os.path.join(mounttmp, "file1.txt")) as f:
 334                 self.assertEqual("blub", f.read())
 335     Test().runTest()
 336
 337 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 338     class Test(unittest.TestCase):
 339         def runTest(self):
 340             d1 = llfuse.listdir(mounttmp)
 341             self.assertEqual(["file1.txt"], d1)
 342             with open(os.path.join(mounttmp, "file1.txt")) as f:
 343                 self.assertEqual("plnp", f.read())
 344     Test().runTest()
 345
 346 class FuseModifyFileTest(MountTestBase):
 347     def runTest(self):
 348         collection = arvados.collection.Collection(api_client=self.api)
 349         with collection.open("file1.txt", "w") as f:
 350             f.write("blub")
 351
 352         collection.save_new()
 353
 354         m = self.make_mount(fuse.CollectionDirectory)
 355         with llfuse.lock:
 356             m.new_collection(collection.api_response(), collection)
 357
 358         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 359
 360         with collection.open("file1.txt", "w") as f:
 361             f.write("plnp")
 362
 363         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 364
 365
 366 class FuseAddFileToCollectionTest(MountTestBase):
 367     def runTest(self):
 368         collection = arvados.collection.Collection(api_client=self.api)
 369         with collection.open("file1.txt", "w") as f:
 370             f.write("blub")
 371
 372         collection.save_new()
 373
 374         m = self.make_mount(fuse.CollectionDirectory)
 375         with llfuse.lock:
 376             m.new_collection(collection.api_response(), collection)
 377
 378         d1 = llfuse.listdir(self.mounttmp)
 379         self.assertEqual(["file1.txt"], d1)
 380
 381         with collection.open("file2.txt", "w") as f:
 382             f.write("plnp")
 383
 384         d1 = llfuse.listdir(self.mounttmp)
 385         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 386
 387
 388 class FuseRemoveFileFromCollectionTest(MountTestBase):
 389     def runTest(self):
 390         collection = arvados.collection.Collection(api_client=self.api)
 391         with collection.open("file1.txt", "w") as f:
 392             f.write("blub")
 393
 394         with collection.open("file2.txt", "w") as f:
 395             f.write("plnp")
 396
 397         collection.save_new()
 398
 399         m = self.make_mount(fuse.CollectionDirectory)
 400         with llfuse.lock:
 401             m.new_collection(collection.api_response(), collection)
 402
 403         d1 = llfuse.listdir(self.mounttmp)
 404         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 405
 406         collection.remove("file2.txt")
 407
 408         d1 = llfuse.listdir(self.mounttmp)
 409         self.assertEqual(["file1.txt"], d1)
 410
 411
 412 def fuseCreateFileTestHelper(mounttmp):
 413     class Test(unittest.TestCase):
 414         def runTest(self):
 415             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 416                 pass
 417     Test().runTest()
 418
 419 class FuseCreateFileTest(MountTestBase):
 420     def runTest(self):
 421         collection = arvados.collection.Collection(api_client=self.api)
 422         collection.save_new()
 423
 424         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 425         self.assertEqual(collection2["manifest_text"], "")
 426
 427         collection.save_new()
 428
 429         m = self.make_mount(fuse.CollectionDirectory)
 430         with llfuse.lock:
 431             m.new_collection(collection.api_response(), collection)
 432         self.assertTrue(m.writable())
 433
 434         self.assertNotIn("file1.txt", collection)
 435
 436         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 437
 438         self.assertIn("file1.txt", collection)
 439
 440         d1 = llfuse.listdir(self.mounttmp)
 441         self.assertEqual(["file1.txt"], d1)
 442
 443         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 444         assertRegex(self, collection2["manifest_text"],
 445             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 446
 447
 448 def fuseWriteFileTestHelperWriteFile(mounttmp):
 449     class Test(unittest.TestCase):
 450         def runTest(self):
 451             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 452                 f.write("Hello world!")
 453     Test().runTest()
 454
 455 def fuseWriteFileTestHelperReadFile(mounttmp):
 456     class Test(unittest.TestCase):
 457         def runTest(self):
 458             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 459                 self.assertEqual(f.read(), "Hello world!")
 460     Test().runTest()
 461
 462 class FuseWriteFileTest(MountTestBase):
 463     def runTest(self):
 464         collection = arvados.collection.Collection(api_client=self.api)
 465         collection.save_new()
 466
 467         m = self.make_mount(fuse.CollectionDirectory)
 468         with llfuse.lock:
 469             m.new_collection(collection.api_response(), collection)
 470         self.assertTrue(m.writable())
 471
 472         self.assertNotIn("file1.txt", collection)
 473
 474         self.assertEqual(0, self.operations.write_counter.get())
 475         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 476         self.assertEqual(12, self.operations.write_counter.get())
 477
 478         with collection.open("file1.txt") as f:
 479             self.assertEqual(f.read(), "Hello world!")
 480
 481         self.assertEqual(0, self.operations.read_counter.get())
 482         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 483         self.assertEqual(12, self.operations.read_counter.get())
 484
 485         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 486         assertRegex(self, collection2["manifest_text"],
 487             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 488
 489
 490 def fuseUpdateFileTestHelper(mounttmp):
 491     class Test(unittest.TestCase):
 492         def runTest(self):
 493             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 494                 f.write("Hello world!")
 495
 496             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 497                 fr = f.read()
 498                 self.assertEqual(fr, "Hello world!")
 499                 f.seek(0)
 500                 f.write("Hola mundo!")
 501                 f.seek(0)
 502                 fr = f.read()
 503                 self.assertEqual(fr, "Hola mundo!!")
 504
 505             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 506                 self.assertEqual(f.read(), "Hola mundo!!")
 507
 508     Test().runTest()
 509
 510 class FuseUpdateFileTest(MountTestBase):
 511     def runTest(self):
 512         collection = arvados.collection.Collection(api_client=self.api)
 513         collection.save_new()
 514
 515         m = self.make_mount(fuse.CollectionDirectory)
 516         with llfuse.lock:
 517             m.new_collection(collection.api_response(), collection)
 518         self.assertTrue(m.writable())
 519
 520         # See note in MountTestBase.setUp
 521         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 522
 523         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 524         assertRegex(self, collection2["manifest_text"],
 525             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 526
 527
 528 def fuseMkdirTestHelper(mounttmp):
 529     class Test(unittest.TestCase):
 530         def runTest(self):
 531             with self.assertRaises(IOError):
 532                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 533                     f.write("Hello world!")
 534
 535             os.mkdir(os.path.join(mounttmp, "testdir"))
 536
 537             with self.assertRaises(OSError):
 538                 os.mkdir(os.path.join(mounttmp, "testdir"))
 539
 540             d1 = llfuse.listdir(mounttmp)
 541             self.assertEqual(["testdir"], d1)
 542
 543             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 544                 f.write("Hello world!")
 545
 546             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 547             self.assertEqual(["file1.txt"], d1)
 548
 549     Test().runTest()
 550
 551 class FuseMkdirTest(MountTestBase):
 552     def runTest(self):
 553         collection = arvados.collection.Collection(api_client=self.api)
 554         collection.save_new()
 555
 556         m = self.make_mount(fuse.CollectionDirectory)
 557         with llfuse.lock:
 558             m.new_collection(collection.api_response(), collection)
 559         self.assertTrue(m.writable())
 560
 561         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 562
 563         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 564         assertRegex(self, collection2["manifest_text"],
 565             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 566
 567
 568 def fuseRmTestHelperWriteFile(mounttmp):
 569     class Test(unittest.TestCase):
 570         def runTest(self):
 571             os.mkdir(os.path.join(mounttmp, "testdir"))
 572
 573             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 574                 f.write("Hello world!")
 575
 576     Test().runTest()
 577
 578 def fuseRmTestHelperDeleteFile(mounttmp):
 579     class Test(unittest.TestCase):
 580         def runTest(self):
 581             # Can't delete because it's not empty
 582             with self.assertRaises(OSError):
 583                 os.rmdir(os.path.join(mounttmp, "testdir"))
 584
 585             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 586             self.assertEqual(["file1.txt"], d1)
 587
 588             # Delete file
 589             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 590
 591             # Make sure it's empty
 592             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 593             self.assertEqual([], d1)
 594
 595             # Try to delete it again
 596             with self.assertRaises(OSError):
 597                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 598
 599     Test().runTest()
 600
 601 def fuseRmTestHelperRmdir(mounttmp):
 602     class Test(unittest.TestCase):
 603         def runTest(self):
 604             # Should be able to delete now that it is empty
 605             os.rmdir(os.path.join(mounttmp, "testdir"))
 606
 607             # Make sure it's empty
 608             d1 = llfuse.listdir(os.path.join(mounttmp))
 609             self.assertEqual([], d1)
 610
 611             # Try to delete it again
 612             with self.assertRaises(OSError):
 613                 os.rmdir(os.path.join(mounttmp, "testdir"))
 614
 615     Test().runTest()
 616
 617 class FuseRmTest(MountTestBase):
 618     def runTest(self):
 619         collection = arvados.collection.Collection(api_client=self.api)
 620         collection.save_new()
 621
 622         m = self.make_mount(fuse.CollectionDirectory)
 623         with llfuse.lock:
 624             m.new_collection(collection.api_response(), collection)
 625         self.assertTrue(m.writable())
 626
 627         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 628
 629         # Starting manifest
 630         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 631         assertRegex(self, collection2["manifest_text"],
 632             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 633         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 634
 635         # Empty directories are represented by an empty file named "."
 636         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 637         assertRegex(self, collection2["manifest_text"],
 638                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 639
 640         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 641
 642         # manifest should be empty now.
 643         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 644         self.assertEqual(collection2["manifest_text"], "")
 645
 646
 647 def fuseMvFileTestHelperWriteFile(mounttmp):
 648     class Test(unittest.TestCase):
 649         def runTest(self):
 650             os.mkdir(os.path.join(mounttmp, "testdir"))
 651
 652             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 653                 f.write("Hello world!")
 654
 655     Test().runTest()
 656
 657 def fuseMvFileTestHelperMoveFile(mounttmp):
 658     class Test(unittest.TestCase):
 659         def runTest(self):
 660             d1 = llfuse.listdir(os.path.join(mounttmp))
 661             self.assertEqual(["testdir"], d1)
 662             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 663             self.assertEqual(["file1.txt"], d1)
 664
 665             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 666
 667             d1 = llfuse.listdir(os.path.join(mounttmp))
 668             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 669             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 670             self.assertEqual([], d1)
 671
 672     Test().runTest()
 673
 674 class FuseMvFileTest(MountTestBase):
 675     def runTest(self):
 676         collection = arvados.collection.Collection(api_client=self.api)
 677         collection.save_new()
 678
 679         m = self.make_mount(fuse.CollectionDirectory)
 680         with llfuse.lock:
 681             m.new_collection(collection.api_response(), collection)
 682         self.assertTrue(m.writable())
 683
 684         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 685
 686         # Starting manifest
 687         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 688         assertRegex(self, collection2["manifest_text"],
 689             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 690
 691         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 692
 693         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 694         assertRegex(self, collection2["manifest_text"],
 695             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 696
 697
 698 def fuseRenameTestHelper(mounttmp):
 699     class Test(unittest.TestCase):
 700         def runTest(self):
 701             os.mkdir(os.path.join(mounttmp, "testdir"))
 702
 703             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 704                 f.write("Hello world!")
 705
 706     Test().runTest()
 707
 708 class FuseRenameTest(MountTestBase):
 709     def runTest(self):
 710         collection = arvados.collection.Collection(api_client=self.api)
 711         collection.save_new()
 712
 713         m = self.make_mount(fuse.CollectionDirectory)
 714         with llfuse.lock:
 715             m.new_collection(collection.api_response(), collection)
 716         self.assertTrue(m.writable())
 717
 718         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 719
 720         # Starting manifest
 721         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 722         assertRegex(self, collection2["manifest_text"],
 723             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 724
 725         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 726         self.assertEqual(["testdir"], d1)
 727         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 728         self.assertEqual(["file1.txt"], d1)
 729
 730         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 731
 732         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 733         self.assertEqual(["testdir2"], sorted(d1))
 734         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 735         self.assertEqual(["file1.txt"], d1)
 736
 737         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 738         assertRegex(self, collection2["manifest_text"],
 739             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 740
 741
 742 class FuseUpdateFromEventTest(MountTestBase):
 743     def runTest(self):
 744         collection = arvados.collection.Collection(api_client=self.api)
 745         collection.save_new()
 746
 747         m = self.make_mount(fuse.CollectionDirectory)
 748         with llfuse.lock:
 749             m.new_collection(collection.api_response(), collection)
 750
 751         self.operations.listen_for_events()
 752
 753         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 754         self.assertEqual([], sorted(d1))
 755
 756         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 757             with collection2.open("file1.txt", "w") as f:
 758                 f.write("foo")
 759
 760         for attempt in AssertWithTimeout(10):
 761             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 762
 763
 764 class FuseDeleteProjectEventTest(MountTestBase):
 765     def runTest(self):
 766
 767         aproject = self.api.groups().create(body={
 768             "name": "aproject",
 769             "group_class": "project"
 770         }).execute()
 771
 772         bproject = self.api.groups().create(body={
 773             "name": "bproject",
 774             "group_class": "project",
 775             "owner_uuid": aproject["uuid"]
 776         }).execute()
 777
 778         self.make_mount(fuse.ProjectDirectory,
 779                         project_object=self.api.users().current().execute())
 780
 781         self.operations.listen_for_events()
 782
 783         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 784         self.assertEqual(["bproject"], sorted(d1))
 785
 786         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 787
 788         for attempt in AssertWithTimeout(10):
 789             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 790
 791
 792 def fuseFileConflictTestHelper(mounttmp, uuid, keeptmp, settings):
 793     class Test(unittest.TestCase):
 794         def runTest(self):
 795             os.environ['KEEP_LOCAL_STORE'] = keeptmp
 796
 797             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 798                 with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
 799                     with collection2.open("file1.txt", "w") as f2:
 800                         f2.write("foo")
 801                 f.write("bar")
 802
 803             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 804             self.assertEqual(len(d1), 2)
 805
 806             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 807                 self.assertEqual(f.read(), "bar")
 808
 809             assertRegex(self, d1[1],
 810                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 811
 812             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 813                 self.assertEqual(f.read(), "foo")
 814
 815     Test().runTest()
 816
 817 class FuseFileConflictTest(MountTestBase):
 818     def runTest(self):
 819         collection = arvados.collection.Collection(api_client=self.api)
 820         collection.save_new()
 821
 822         m = self.make_mount(fuse.CollectionDirectory)
 823         with llfuse.lock:
 824             m.new_collection(collection.api_response(), collection)
 825
 826         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 827         self.assertEqual([], sorted(d1))
 828
 829         # See note in MountTestBase.setUp
 830         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
 831
 832
 833 def fuseUnlinkOpenFileTest(mounttmp):
 834     class Test(unittest.TestCase):
 835         def runTest(self):
 836             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 837                 f.write("foo")
 838
 839                 d1 = llfuse.listdir(os.path.join(mounttmp))
 840                 self.assertEqual(["file1.txt"], sorted(d1))
 841
 842                 os.remove(os.path.join(mounttmp, "file1.txt"))
 843
 844                 d1 = llfuse.listdir(os.path.join(mounttmp))
 845                 self.assertEqual([], sorted(d1))
 846
 847                 f.seek(0)
 848                 self.assertEqual(f.read(), "foo")
 849                 f.write("bar")
 850
 851                 f.seek(0)
 852                 self.assertEqual(f.read(), "foobar")
 853
 854     Test().runTest()
 855
 856 class FuseUnlinkOpenFileTest(MountTestBase):
 857     def runTest(self):
 858         collection = arvados.collection.Collection(api_client=self.api)
 859         collection.save_new()
 860
 861         m = self.make_mount(fuse.CollectionDirectory)
 862         with llfuse.lock:
 863             m.new_collection(collection.api_response(), collection)
 864
 865         # See note in MountTestBase.setUp
 866         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 867
 868         self.assertEqual(collection.manifest_text(), "")
 869
 870
 871 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 872     class Test(unittest.TestCase):
 873         def runTest(self):
 874             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 875                 f.write("Hello world!")
 876
 877             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 878             self.assertEqual(["file1.txt"], sorted(d1))
 879             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 880             self.assertEqual([], sorted(d1))
 881
 882     Test().runTest()
 883
 884 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 885     class Test(unittest.TestCase):
 886         def runTest(self):
 887             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 888
 889             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 890             self.assertEqual([], sorted(d1))
 891             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 892             self.assertEqual(["file2.txt"], sorted(d1))
 893
 894     Test().runTest()
 895
 896 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 897     def runTest(self):
 898         collection1 = arvados.collection.Collection(api_client=self.api)
 899         collection1.save_new()
 900
 901         collection2 = arvados.collection.Collection(api_client=self.api)
 902         collection2.save_new()
 903
 904         m = self.make_mount(fuse.MagicDirectory)
 905
 906         # See note in MountTestBase.setUp
 907         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 908                                                   collection1.manifest_locator(),
 909                                                   collection2.manifest_locator()))
 910
 911         collection1.update()
 912         collection2.update()
 913
 914         assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 915         self.assertEqual(collection2.manifest_text(), "")
 916
 917         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 918                                                   collection1.manifest_locator(),
 919                                                   collection2.manifest_locator()))
 920
 921         collection1.update()
 922         collection2.update()
 923
 924         self.assertEqual(collection1.manifest_text(), "")
 925         assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 926
 927         collection1.stop_threads()
 928         collection2.stop_threads()
 929
 930
 931 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 932     class Test(unittest.TestCase):
 933         def runTest(self):
 934             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 935             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 936                 f.write("Hello world!")
 937
 938             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 939             self.assertEqual(["testdir"], sorted(d1))
 940             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 941             self.assertEqual(["file1.txt"], sorted(d1))
 942
 943             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 944             self.assertEqual([], sorted(d1))
 945
 946     Test().runTest()
 947
 948
 949 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 950     class Test(unittest.TestCase):
 951         def runTest(self):
 952             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 953
 954             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 955             self.assertEqual([], sorted(d1))
 956
 957             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 958             self.assertEqual(["testdir2"], sorted(d1))
 959             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 960             self.assertEqual(["file1.txt"], sorted(d1))
 961
 962             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 963                 self.assertEqual(f.read(), "Hello world!")
 964
 965     Test().runTest()
 966
 967 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 968     def runTest(self):
 969         collection1 = arvados.collection.Collection(api_client=self.api)
 970         collection1.save_new()
 971
 972         collection2 = arvados.collection.Collection(api_client=self.api)
 973         collection2.save_new()
 974
 975         m = self.make_mount(fuse.MagicDirectory)
 976
 977         # See note in MountTestBase.setUp
 978         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 979                                                   collection1.manifest_locator(),
 980                                                   collection2.manifest_locator()))
 981
 982         collection1.update()
 983         collection2.update()
 984
 985         assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 986         self.assertEqual(collection2.manifest_text(), "")
 987
 988         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 989                                                   collection1.manifest_locator(),
 990                                                   collection2.manifest_locator()))
 991
 992         collection1.update()
 993         collection2.update()
 994
 995         self.assertEqual(collection1.manifest_text(), "")
 996         assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 997
 998         collection1.stop_threads()
 999         collection2.stop_threads()
1000
1001 def fuseProjectMkdirTestHelper1(mounttmp):
1002     class Test(unittest.TestCase):
1003         def runTest(self):
1004             os.mkdir(os.path.join(mounttmp, "testcollection"))
1005             with self.assertRaises(OSError):
1006                 os.mkdir(os.path.join(mounttmp, "testcollection"))
1007     Test().runTest()
1008
1009 def fuseProjectMkdirTestHelper2(mounttmp):
1010     class Test(unittest.TestCase):
1011         def runTest(self):
1012             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1013                 f.write("Hello world!")
1014             with self.assertRaises(OSError):
1015                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1016             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1017             with self.assertRaises(OSError):
1018                 os.remove(os.path.join(mounttmp, "testcollection"))
1019             os.rmdir(os.path.join(mounttmp, "testcollection"))
1020     Test().runTest()
1021
1022 class FuseProjectMkdirRmdirTest(MountTestBase):
1023     def runTest(self):
1024         self.make_mount(fuse.ProjectDirectory,
1025                         project_object=self.api.users().current().execute())
1026
1027         d1 = llfuse.listdir(self.mounttmp)
1028         self.assertNotIn('testcollection', d1)
1029
1030         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1031
1032         d1 = llfuse.listdir(self.mounttmp)
1033         self.assertIn('testcollection', d1)
1034
1035         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1036
1037         d1 = llfuse.listdir(self.mounttmp)
1038         self.assertNotIn('testcollection', d1)
1039
1040
1041 def fuseProjectMvTestHelper1(mounttmp):
1042     class Test(unittest.TestCase):
1043         def runTest(self):
1044             d1 = llfuse.listdir(mounttmp)
1045             self.assertNotIn('testcollection', d1)
1046
1047             os.mkdir(os.path.join(mounttmp, "testcollection"))
1048
1049             d1 = llfuse.listdir(mounttmp)
1050             self.assertIn('testcollection', d1)
1051
1052             with self.assertRaises(OSError):
1053                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1054
1055             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1056
1057             d1 = llfuse.listdir(mounttmp)
1058             self.assertNotIn('testcollection', d1)
1059
1060             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1061             self.assertIn('testcollection', d1)
1062
1063     Test().runTest()
1064
1065 class FuseProjectMvTest(MountTestBase):
1066     def runTest(self):
1067         self.make_mount(fuse.ProjectDirectory,
1068                         project_object=self.api.users().current().execute())
1069
1070         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1071
1072
1073 def fuseFsyncTestHelper(mounttmp, k):
1074     class Test(unittest.TestCase):
1075         def runTest(self):
1076             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1077             os.fsync(fd)
1078             os.close(fd)
1079
1080     Test().runTest()
1081
1082 class FuseFsyncTest(FuseMagicTest):
1083     def runTest(self):
1084         self.make_mount(fuse.MagicDirectory)
1085         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1086
1087
1088 class MagicDirApiError(FuseMagicTest):
1089     def setUp(self):
1090         api = mock.MagicMock()
1091         super(MagicDirApiError, self).setUp(api=api)
1092         api.collections().get().execute.side_effect = iter([
1093             Exception('API fail'),
1094             {
1095                 "manifest_text": self.test_manifest,
1096                 "portable_data_hash": self.test_manifest_pdh,
1097             },
1098         ])
1099         api.keep.get.side_effect = Exception('Keep fail')
1100
1101     def runTest(self):
1102         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1103             self.make_mount(fuse.MagicDirectory)
1104
1105             self.operations.inodes.inode_cache.cap = 1
1106             self.operations.inodes.inode_cache.min_entries = 2
1107
1108             with self.assertRaises(OSError):
1109                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1110
1111             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1112
1113
1114 class SanitizeFilenameTest(MountTestBase):
1115     def test_sanitize_filename(self):
1116         pdir = fuse.ProjectDirectory(1, {}, self.api, 0, False, project_object=self.api.users().current().execute())
1117         acceptable = [
1118             "foo.txt",
1119             ".foo",
1120             "..foo",
1121             "...",
1122             "foo...",
1123             "foo..",
1124             "foo.",
1125             "-",
1126             "\x01\x02\x03",
1127             ]
1128         unacceptable = [
1129             "f\00",
1130             "\00\00",
1131             "/foo",
1132             "foo/",
1133             "//",
1134             ]
1135         for f in acceptable:
1136             self.assertEqual(f, pdir.sanitize_filename(f))
1137         for f in unacceptable:
1138             self.assertNotEqual(f, pdir.sanitize_filename(f))
1139             # The sanitized filename should be the same length, though.
1140             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1141         # Special cases
1142         self.assertEqual("_", pdir.sanitize_filename(""))
1143         self.assertEqual("_", pdir.sanitize_filename("."))
1144         self.assertEqual("__", pdir.sanitize_filename(".."))
1145
1146
1147 class FuseMagicTestPDHOnly(MountTestBase):
1148     def setUp(self, api=None):
1149         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1150
1151         cw = arvados.CollectionWriter()
1152
1153         cw.start_new_file('thing1.txt')
1154         cw.write("data 1")
1155
1156         self.testcollection = cw.finish()
1157         self.test_manifest = cw.manifest_text()
1158         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1159         self.testcollectionuuid = str(created['uuid'])
1160
1161     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1162         if skip_pdh_only is True:
1163             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1164         else:
1165             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1166
1167         mount_ls = llfuse.listdir(self.mounttmp)
1168         self.assertIn('README', mount_ls)
1169         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1170                              arvados.util.uuid_pattern.match(fn)
1171                              for fn in mount_ls),
1172                          "new FUSE MagicDirectory lists Collection")
1173
1174         # look up using pdh should succeed in all cases
1175         self.assertDirContents(self.testcollection, ['thing1.txt'])
1176         self.assertDirContents(os.path.join('by_id', self.testcollection),
1177                                ['thing1.txt'])
1178         mount_ls = llfuse.listdir(self.mounttmp)
1179         self.assertIn('README', mount_ls)
1180         self.assertIn(self.testcollection, mount_ls)
1181         self.assertIn(self.testcollection,
1182                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1183
1184         files = {}
1185         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1186
1187         for k, v in viewitems(files):
1188             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1189                 self.assertEqual(v, f.read().decode())
1190
1191         # look up using uuid should fail when pdh_only is set
1192         if pdh_only is True:
1193             with self.assertRaises(OSError):
1194                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1195                                ['thing1.txt'])
1196         else:
1197             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1198                                ['thing1.txt'])
1199
1200     def test_with_pdh_only_true(self):
1201         self.verify_pdh_only(pdh_only=True)
1202
1203     def test_with_pdh_only_false(self):
1204         self.verify_pdh_only(pdh_only=False)
1205
1206     def test_with_default_by_id(self):
1207         self.verify_pdh_only(skip_pdh_only=True)
1208
1209
1210 class SlashSubstitutionTest(IntegrationTest):
1211     mnt_args = [
1212         '--read-write',
1213         '--mount-home', 'zzz',
1214     ]
1215
1216     def setUp(self):
1217         super(SlashSubstitutionTest, self).setUp()
1218         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1219         self.api.config = lambda: {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1220         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1221         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1222         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1223
1224     @IntegrationTest.mount(argv=mnt_args)
1225     @mock.patch('arvados.util.get_config_once')
1226     def test_slash_substitution_before_listing(self, get_config_once):
1227         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1228         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1229         self.checkContents()
1230     @staticmethod
1231     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1232         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1233             f.write('xxx')
1234         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1235             f.write('foo')
1236
1237     @IntegrationTest.mount(argv=mnt_args)
1238     @mock.patch('arvados.util.get_config_once')
1239     def test_slash_substitution_after_listing(self, get_config_once):
1240         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1241         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1242         self.checkContents()
1243     @staticmethod
1244     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1245         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1246             f.write('xxx')
1247         os.listdir(tmpdir)
1248         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1249             f.write('foo')
1250
1251     def checkContents(self):
1252         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1253         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1254
1255     @IntegrationTest.mount(argv=mnt_args)
1256     @mock.patch('arvados.util.get_config_once')
1257     def test_slash_substitution_conflict(self, get_config_once):
1258         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1259         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1260         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1261         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1262         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1263         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1264     @staticmethod
1265     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1266         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1267             f.write('foo')
1268
1269 class StorageClassesTest(IntegrationTest):
1270     mnt_args = [
1271         '--read-write',
1272         '--mount-home', 'homedir',
1273     ]
1274
1275     def setUp(self):
1276         super(StorageClassesTest, self).setUp()
1277         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1278
1279     @IntegrationTest.mount(argv=mnt_args)
1280     def test_collection_default_storage_classes(self):
1281         coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
1282         self.api.collections().create(body={'name':'a_collection'}).execute()
1283         self.pool_test(coll_path)
1284     @staticmethod
1285     def _test_collection_default_storage_classes(self, coll):
1286         self.assertEqual(storage_classes_desired(coll), ['default'])
1287
1288     @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
1289     def test_collection_custom_storage_classes(self):
1290         coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
1291         os.mkdir(coll_path)
1292         self.pool_test(coll_path)
1293     @staticmethod
1294     def _test_collection_custom_storage_classes(self, coll):
1295         self.assertEqual(storage_classes_desired(coll), ['foo'])
1296
1297 def _readonlyCollectionTestHelper(mounttmp):
1298     f = open(os.path.join(mounttmp, 'thing1.txt'), 'rt')
1299     # Testing that close() doesn't raise an error.
1300     f.close()
1301
1302 class ReadonlyCollectionTest(MountTestBase):
1303     def setUp(self):
1304         super(ReadonlyCollectionTest, self).setUp()
1305         cw = arvados.collection.Collection()
1306         with cw.open('thing1.txt', 'wt') as f:
1307             f.write("data 1")
1308         cw.save_new(owner_uuid=run_test_server.fixture("groups")["aproject"]["uuid"])
1309         self.testcollection = cw.api_response()
1310
1311     def runTest(self):
1312         settings = arvados.config.settings().copy()
1313         settings["ARVADOS_API_TOKEN"] = run_test_server.fixture("api_client_authorizations")["project_viewer"]["api_token"]
1314         self.api = arvados.safeapi.ThreadSafeApiCache(settings)
1315         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection, enable_write=False)
1316
1317         self.pool.apply(_readonlyCollectionTestHelper, (self.mounttmp,))