services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 from __future__ import absolute_import
   6 from future.utils import viewitems
   7 from builtins import str
   8 from builtins import object
   9 from six import assertRegex
  10 import json
  11 import llfuse
  12 import logging
  13 import mock
  14 import os
  15 import subprocess
  16 import time
  17 import unittest
  18 import tempfile
  19 import parameterized
  20
  21 import arvados
  22 import arvados_fuse as fuse
  23 from . import run_test_server
  24
  25 from .integration_test import IntegrationTest
  26 from .mount_test_base import MountTestBase
  27 from .test_tmp_collection import storage_classes_desired
  28
  29 logger = logging.getLogger('arvados.arv-mount')
  30
  31
  32 class AssertWithTimeout(object):
  33     """Allow some time for an assertion to pass."""
  34
  35     def __init__(self, timeout=0):
  36         self.timeout = timeout
  37
  38     def __iter__(self):
  39         self.deadline = time.time() + self.timeout
  40         self.done = False
  41         return self
  42
  43     def __next__(self):
  44         if self.done:
  45             raise StopIteration
  46         return self.attempt
  47
  48     def attempt(self, fn, *args, **kwargs):
  49         try:
  50             fn(*args, **kwargs)
  51         except AssertionError:
  52             if time.time() > self.deadline:
  53                 raise
  54             time.sleep(0.1)
  55         else:
  56             self.done = True
  57
  58 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
  59 class FuseMountTest(MountTestBase):
  60     def setUp(self):
  61         super(FuseMountTest, self).setUp()
  62
  63         cw = arvados.CollectionWriter()
  64
  65         cw.start_new_file('thing1.txt')
  66         cw.write("data 1")
  67         cw.start_new_file('thing2.txt')
  68         cw.write("data 2")
  69
  70         cw.start_new_stream('dir1')
  71         cw.start_new_file('thing3.txt')
  72         cw.write("data 3")
  73         cw.start_new_file('thing4.txt')
  74         cw.write("data 4")
  75
  76         cw.start_new_stream('dir2')
  77         cw.start_new_file('thing5.txt')
  78         cw.write("data 5")
  79         cw.start_new_file('thing6.txt')
  80         cw.write("data 6")
  81
  82         cw.start_new_stream('dir2/dir3')
  83         cw.start_new_file('thing7.txt')
  84         cw.write("data 7")
  85
  86         cw.start_new_file('thing8.txt')
  87         cw.write("data 8")
  88
  89         cw.start_new_stream('edgecases')
  90         for f in ":/.../-/*/ ".split("/"):
  91             cw.start_new_file(f)
  92             cw.write('x')
  93
  94         for f in ":/.../-/*/ ".split("/"):
  95             cw.start_new_stream('edgecases/dirs/' + f)
  96             cw.start_new_file('x/x')
  97             cw.write('x')
  98
  99         self.testcollection = cw.finish()
 100         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
 101
 102     def runTest(self):
 103         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
 104
 105         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 106                                       'edgecases', 'dir1', 'dir2'])
 107         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 108         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 109         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 110         self.assertDirContents('edgecases',
 111                                "dirs/:/.../-/*/ ".split("/"))
 112         self.assertDirContents('edgecases/dirs',
 113                                ":/.../-/*/ ".split("/"))
 114
 115         files = {'thing1.txt': 'data 1',
 116                  'thing2.txt': 'data 2',
 117                  'dir1/thing3.txt': 'data 3',
 118                  'dir1/thing4.txt': 'data 4',
 119                  'dir2/thing5.txt': 'data 5',
 120                  'dir2/thing6.txt': 'data 6',
 121                  'dir2/dir3/thing7.txt': 'data 7',
 122                  'dir2/dir3/thing8.txt': 'data 8'}
 123
 124         for k, v in viewitems(files):
 125             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 126                 self.assertEqual(v, f.read().decode())
 127
 128
 129 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 130 class FuseMagicTest(MountTestBase):
 131     def setUp(self, api=None):
 132         super(FuseMagicTest, self).setUp(api=api)
 133
 134         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 135         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
 136         self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
 137         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 138         self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
 139
 140         cw = arvados.CollectionWriter()
 141
 142         cw.start_new_file('thing1.txt')
 143         cw.write("data 1")
 144
 145         self.testcollection = cw.finish()
 146         self.test_manifest = cw.manifest_text()
 147         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 148         self.test_manifest_pdh = coll['portable_data_hash']
 149
 150     def runTest(self):
 151         self.make_mount(fuse.MagicDirectory)
 152
 153         mount_ls = llfuse.listdir(self.mounttmp)
 154         self.assertIn('README', mount_ls)
 155         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 156                              arvados.util.uuid_pattern.match(fn)
 157                              for fn in mount_ls),
 158                          "new FUSE MagicDirectory has no collections or projects")
 159         self.assertDirContents(self.testcollection, ['thing1.txt'])
 160         self.assertDirContents(os.path.join('by_id', self.testcollection),
 161                                ['thing1.txt'])
 162         self.assertIn(self.collection_in_test_project,
 163                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 164         self.assertIn(self.collection_in_test_project,
 165                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 166         self.assertIn(self.collection_in_filter_group,
 167                       llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
 168         self.assertIn(self.collection_in_filter_group,
 169                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
 170
 171
 172         mount_ls = llfuse.listdir(self.mounttmp)
 173         self.assertIn('README', mount_ls)
 174         self.assertIn(self.testcollection, mount_ls)
 175         self.assertIn(self.testcollection,
 176                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 177         self.assertIn(self.test_project, mount_ls)
 178         self.assertIn(self.test_project,
 179                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 180         self.assertIn(self.filter_group,
 181                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 182
 183         with self.assertRaises(OSError):
 184             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 185
 186         files = {}
 187         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 188
 189         for k, v in viewitems(files):
 190             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 191                 self.assertEqual(v, f.read().decode())
 192
 193
 194 class FuseTagsTest(MountTestBase):
 195     def runTest(self):
 196         self.make_mount(fuse.TagsDirectory)
 197
 198         d1 = llfuse.listdir(self.mounttmp)
 199         d1.sort()
 200         self.assertEqual(['foo_tag'], d1)
 201
 202         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 203         d2.sort()
 204         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 205
 206         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 207         d3.sort()
 208         self.assertEqual(['foo'], d3)
 209
 210
 211 class FuseTagsUpdateTest(MountTestBase):
 212     def tag_collection(self, coll_uuid, tag_name):
 213         return self.api.links().create(
 214             body={'link': {'head_uuid': coll_uuid,
 215                            'link_class': 'tag',
 216                            'name': tag_name,
 217         }}).execute()
 218
 219     def runTest(self):
 220         self.make_mount(fuse.TagsDirectory, poll_time=1)
 221
 222         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 223
 224         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 225         self.tag_collection(bar_uuid, 'fuse_test_tag')
 226         for attempt in AssertWithTimeout(10):
 227             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 228         self.assertDirContents('fuse_test_tag', [bar_uuid])
 229
 230         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 231         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 232         for attempt in AssertWithTimeout(10):
 233             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 234
 235         self.api.links().delete(uuid=l['uuid']).execute()
 236         for attempt in AssertWithTimeout(10):
 237             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 238
 239
 240 def fuseSharedTestHelper(mounttmp):
 241     class Test(unittest.TestCase):
 242         def runTest(self):
 243             # Double check that we can open and read objects in this folder as a file,
 244             # and that its contents are what we expect.
 245             baz_path = os.path.join(
 246                 mounttmp,
 247                 'FUSE User',
 248                 'FUSE Test Project',
 249                 'collection in FUSE project',
 250                 'baz')
 251             with open(baz_path) as f:
 252                 self.assertEqual("baz", f.read())
 253
 254             # check mtime on collection
 255             st = os.stat(baz_path)
 256             try:
 257                 mtime = st.st_mtime_ns // 1000000000
 258             except AttributeError:
 259                 mtime = st.st_mtime
 260             self.assertEqual(mtime, 1391448174)
 261
 262             # shared_dirs is a list of the directories exposed
 263             # by fuse.SharedDirectory (i.e. any object visible
 264             # to the current user)
 265             shared_dirs = llfuse.listdir(mounttmp)
 266             shared_dirs.sort()
 267             self.assertIn('FUSE User', shared_dirs)
 268
 269             # fuse_user_objs is a list of the objects owned by the FUSE
 270             # test user (which present as files in the 'FUSE User'
 271             # directory)
 272             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 273             fuse_user_objs.sort()
 274             self.assertEqual(['FUSE Test Project',                    # project owned by user
 275                               'collection #1 owned by FUSE',          # collection owned by user
 276                               'collection #2 owned by FUSE'          # collection owned by user
 277                           ], fuse_user_objs)
 278
 279             # test_proj_files is a list of the files in the FUSE Test Project.
 280             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 281             test_proj_files.sort()
 282             self.assertEqual(['collection in FUSE project'
 283                           ], test_proj_files)
 284
 285
 286     Test().runTest()
 287
 288 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 289 class FuseSharedTest(MountTestBase):
 290     def runTest(self):
 291         self.make_mount(fuse.SharedDirectory,
 292                         exclude=self.api.users().current().execute()['uuid'])
 293         keep = arvados.keep.KeepClient()
 294         keep.put("baz".encode())
 295
 296         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 297
 298
 299 class FuseHomeTest(MountTestBase):
 300     def runTest(self):
 301         self.make_mount(fuse.ProjectDirectory,
 302                         project_object=self.api.users().current().execute())
 303
 304         d1 = llfuse.listdir(self.mounttmp)
 305         self.assertIn('Unrestricted public data', d1)
 306
 307         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 308         public_project = run_test_server.fixture('groups')[
 309             'anonymously_accessible_project']
 310         found_in = 0
 311         found_not_in = 0
 312         for name, item in viewitems(run_test_server.fixture('collections')):
 313             if 'name' not in item:
 314                 pass
 315             elif item['owner_uuid'] == public_project['uuid']:
 316                 self.assertIn(item['name'], d2)
 317                 found_in += 1
 318             else:
 319                 # Artificial assumption here: there is no public
 320                 # collection fixture with the same name as a
 321                 # non-public collection.
 322                 self.assertNotIn(item['name'], d2)
 323                 found_not_in += 1
 324         self.assertNotEqual(0, found_in)
 325         self.assertNotEqual(0, found_not_in)
 326
 327         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 328         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 329
 330
 331 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 332     class Test(unittest.TestCase):
 333         def runTest(self):
 334             d1 = llfuse.listdir(mounttmp)
 335             self.assertEqual(["file1.txt"], d1)
 336             with open(os.path.join(mounttmp, "file1.txt")) as f:
 337                 self.assertEqual("blub", f.read())
 338     Test().runTest()
 339
 340 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 341     class Test(unittest.TestCase):
 342         def runTest(self):
 343             d1 = llfuse.listdir(mounttmp)
 344             self.assertEqual(["file1.txt"], d1)
 345             with open(os.path.join(mounttmp, "file1.txt")) as f:
 346                 self.assertEqual("plnp", f.read())
 347     Test().runTest()
 348
 349 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 350 class FuseModifyFileTest(MountTestBase):
 351     def runTest(self):
 352         collection = arvados.collection.Collection(api_client=self.api)
 353         with collection.open("file1.txt", "w") as f:
 354             f.write("blub")
 355
 356         collection.save_new()
 357
 358         m = self.make_mount(fuse.CollectionDirectory)
 359         with llfuse.lock:
 360             m.new_collection(collection.api_response(), collection)
 361
 362         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 363
 364         with collection.open("file1.txt", "w") as f:
 365             f.write("plnp")
 366
 367         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 368
 369
 370 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 371 class FuseAddFileToCollectionTest(MountTestBase):
 372     def runTest(self):
 373         collection = arvados.collection.Collection(api_client=self.api)
 374         with collection.open("file1.txt", "w") as f:
 375             f.write("blub")
 376
 377         collection.save_new()
 378
 379         m = self.make_mount(fuse.CollectionDirectory)
 380         with llfuse.lock:
 381             m.new_collection(collection.api_response(), collection)
 382
 383         d1 = llfuse.listdir(self.mounttmp)
 384         self.assertEqual(["file1.txt"], d1)
 385
 386         with collection.open("file2.txt", "w") as f:
 387             f.write("plnp")
 388
 389         d1 = llfuse.listdir(self.mounttmp)
 390         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 391
 392
 393 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 394 class FuseRemoveFileFromCollectionTest(MountTestBase):
 395     def runTest(self):
 396         collection = arvados.collection.Collection(api_client=self.api)
 397         with collection.open("file1.txt", "w") as f:
 398             f.write("blub")
 399
 400         with collection.open("file2.txt", "w") as f:
 401             f.write("plnp")
 402
 403         collection.save_new()
 404
 405         m = self.make_mount(fuse.CollectionDirectory)
 406         with llfuse.lock:
 407             m.new_collection(collection.api_response(), collection)
 408
 409         d1 = llfuse.listdir(self.mounttmp)
 410         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 411
 412         collection.remove("file2.txt")
 413
 414         d1 = llfuse.listdir(self.mounttmp)
 415         self.assertEqual(["file1.txt"], d1)
 416
 417
 418 def fuseCreateFileTestHelper(mounttmp):
 419     class Test(unittest.TestCase):
 420         def runTest(self):
 421             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 422                 pass
 423     Test().runTest()
 424
 425 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 426 class FuseCreateFileTest(MountTestBase):
 427     def runTest(self):
 428         collection = arvados.collection.Collection(api_client=self.api)
 429         collection.save_new()
 430
 431         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 432         self.assertEqual(collection2["manifest_text"], "")
 433
 434         collection.save_new()
 435
 436         m = self.make_mount(fuse.CollectionDirectory)
 437         with llfuse.lock:
 438             m.new_collection(collection.api_response(), collection)
 439         self.assertTrue(m.writable())
 440
 441         self.assertNotIn("file1.txt", collection)
 442
 443         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 444
 445         self.assertIn("file1.txt", collection)
 446
 447         d1 = llfuse.listdir(self.mounttmp)
 448         self.assertEqual(["file1.txt"], d1)
 449
 450         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 451         assertRegex(self, collection2["manifest_text"],
 452             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 453
 454
 455 def fuseWriteFileTestHelperWriteFile(mounttmp):
 456     class Test(unittest.TestCase):
 457         def runTest(self):
 458             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 459                 f.write("Hello world!")
 460     Test().runTest()
 461
 462 def fuseWriteFileTestHelperReadFile(mounttmp):
 463     class Test(unittest.TestCase):
 464         def runTest(self):
 465             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 466                 self.assertEqual(f.read(), "Hello world!")
 467     Test().runTest()
 468
 469 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 470 class FuseWriteFileTest(MountTestBase):
 471     def runTest(self):
 472         collection = arvados.collection.Collection(api_client=self.api)
 473         collection.save_new()
 474
 475         m = self.make_mount(fuse.CollectionDirectory)
 476         with llfuse.lock:
 477             m.new_collection(collection.api_response(), collection)
 478         self.assertTrue(m.writable())
 479
 480         self.assertNotIn("file1.txt", collection)
 481
 482         self.assertEqual(0, self.operations.write_counter.get())
 483         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 484         self.assertEqual(12, self.operations.write_counter.get())
 485
 486         with collection.open("file1.txt") as f:
 487             self.assertEqual(f.read(), "Hello world!")
 488
 489         self.assertEqual(0, self.operations.read_counter.get())
 490         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 491         self.assertEqual(12, self.operations.read_counter.get())
 492
 493         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 494         assertRegex(self, collection2["manifest_text"],
 495             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 496
 497
 498 def fuseUpdateFileTestHelper(mounttmp):
 499     class Test(unittest.TestCase):
 500         def runTest(self):
 501             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 502                 f.write("Hello world!")
 503
 504             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 505                 fr = f.read()
 506                 self.assertEqual(fr, "Hello world!")
 507                 f.seek(0)
 508                 f.write("Hola mundo!")
 509                 f.seek(0)
 510                 fr = f.read()
 511                 self.assertEqual(fr, "Hola mundo!!")
 512
 513             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 514                 self.assertEqual(f.read(), "Hola mundo!!")
 515
 516     Test().runTest()
 517
 518 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 519 class FuseUpdateFileTest(MountTestBase):
 520     def runTest(self):
 521         collection = arvados.collection.Collection(api_client=self.api)
 522         collection.save_new()
 523
 524         m = self.make_mount(fuse.CollectionDirectory)
 525         with llfuse.lock:
 526             m.new_collection(collection.api_response(), collection)
 527         self.assertTrue(m.writable())
 528
 529         # See note in MountTestBase.setUp
 530         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 531
 532         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 533         assertRegex(self, collection2["manifest_text"],
 534             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 535
 536
 537 def fuseMkdirTestHelper(mounttmp):
 538     class Test(unittest.TestCase):
 539         def runTest(self):
 540             with self.assertRaises(IOError):
 541                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 542                     f.write("Hello world!")
 543
 544             os.mkdir(os.path.join(mounttmp, "testdir"))
 545
 546             with self.assertRaises(OSError):
 547                 os.mkdir(os.path.join(mounttmp, "testdir"))
 548
 549             d1 = llfuse.listdir(mounttmp)
 550             self.assertEqual(["testdir"], d1)
 551
 552             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 553                 f.write("Hello world!")
 554
 555             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 556             self.assertEqual(["file1.txt"], d1)
 557
 558     Test().runTest()
 559
 560 class FuseMkdirTest(MountTestBase):
 561     def runTest(self):
 562         collection = arvados.collection.Collection(api_client=self.api)
 563         collection.save_new()
 564
 565         m = self.make_mount(fuse.CollectionDirectory)
 566         with llfuse.lock:
 567             m.new_collection(collection.api_response(), collection)
 568         self.assertTrue(m.writable())
 569
 570         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 571
 572         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 573         assertRegex(self, collection2["manifest_text"],
 574             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 575
 576
 577 def fuseRmTestHelperWriteFile(mounttmp):
 578     class Test(unittest.TestCase):
 579         def runTest(self):
 580             os.mkdir(os.path.join(mounttmp, "testdir"))
 581
 582             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 583                 f.write("Hello world!")
 584
 585     Test().runTest()
 586
 587 def fuseRmTestHelperDeleteFile(mounttmp):
 588     class Test(unittest.TestCase):
 589         def runTest(self):
 590             # Can't delete because it's not empty
 591             with self.assertRaises(OSError):
 592                 os.rmdir(os.path.join(mounttmp, "testdir"))
 593
 594             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 595             self.assertEqual(["file1.txt"], d1)
 596
 597             # Delete file
 598             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 599
 600             # Make sure it's empty
 601             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 602             self.assertEqual([], d1)
 603
 604             # Try to delete it again
 605             with self.assertRaises(OSError):
 606                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 607
 608     Test().runTest()
 609
 610 def fuseRmTestHelperRmdir(mounttmp):
 611     class Test(unittest.TestCase):
 612         def runTest(self):
 613             # Should be able to delete now that it is empty
 614             os.rmdir(os.path.join(mounttmp, "testdir"))
 615
 616             # Make sure it's empty
 617             d1 = llfuse.listdir(os.path.join(mounttmp))
 618             self.assertEqual([], d1)
 619
 620             # Try to delete it again
 621             with self.assertRaises(OSError):
 622                 os.rmdir(os.path.join(mounttmp, "testdir"))
 623
 624     Test().runTest()
 625
 626 class FuseRmTest(MountTestBase):
 627     def runTest(self):
 628         collection = arvados.collection.Collection(api_client=self.api)
 629         collection.save_new()
 630
 631         m = self.make_mount(fuse.CollectionDirectory)
 632         with llfuse.lock:
 633             m.new_collection(collection.api_response(), collection)
 634         self.assertTrue(m.writable())
 635
 636         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 637
 638         # Starting manifest
 639         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 640         assertRegex(self, collection2["manifest_text"],
 641             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 642         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 643
 644         # Empty directories are represented by an empty file named "."
 645         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 646         assertRegex(self, collection2["manifest_text"],
 647                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 648
 649         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 650
 651         # manifest should be empty now.
 652         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 653         self.assertEqual(collection2["manifest_text"], "")
 654
 655
 656 def fuseMvFileTestHelperWriteFile(mounttmp):
 657     class Test(unittest.TestCase):
 658         def runTest(self):
 659             os.mkdir(os.path.join(mounttmp, "testdir"))
 660
 661             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 662                 f.write("Hello world!")
 663
 664     Test().runTest()
 665
 666 def fuseMvFileTestHelperMoveFile(mounttmp):
 667     class Test(unittest.TestCase):
 668         def runTest(self):
 669             d1 = llfuse.listdir(os.path.join(mounttmp))
 670             self.assertEqual(["testdir"], d1)
 671             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 672             self.assertEqual(["file1.txt"], d1)
 673
 674             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 675
 676             d1 = llfuse.listdir(os.path.join(mounttmp))
 677             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 678             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 679             self.assertEqual([], d1)
 680
 681     Test().runTest()
 682
 683 class FuseMvFileTest(MountTestBase):
 684     def runTest(self):
 685         collection = arvados.collection.Collection(api_client=self.api)
 686         collection.save_new()
 687
 688         m = self.make_mount(fuse.CollectionDirectory)
 689         with llfuse.lock:
 690             m.new_collection(collection.api_response(), collection)
 691         self.assertTrue(m.writable())
 692
 693         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 694
 695         # Starting manifest
 696         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 697         assertRegex(self, collection2["manifest_text"],
 698             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 699
 700         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 701
 702         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 703         assertRegex(self, collection2["manifest_text"],
 704             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 705
 706
 707 def fuseRenameTestHelper(mounttmp):
 708     class Test(unittest.TestCase):
 709         def runTest(self):
 710             os.mkdir(os.path.join(mounttmp, "testdir"))
 711
 712             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 713                 f.write("Hello world!")
 714
 715     Test().runTest()
 716
 717 class FuseRenameTest(MountTestBase):
 718     def runTest(self):
 719         collection = arvados.collection.Collection(api_client=self.api)
 720         collection.save_new()
 721
 722         m = self.make_mount(fuse.CollectionDirectory)
 723         with llfuse.lock:
 724             m.new_collection(collection.api_response(), collection)
 725         self.assertTrue(m.writable())
 726
 727         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 728
 729         # Starting manifest
 730         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 731         assertRegex(self, collection2["manifest_text"],
 732             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 733
 734         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 735         self.assertEqual(["testdir"], d1)
 736         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 737         self.assertEqual(["file1.txt"], d1)
 738
 739         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 740
 741         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 742         self.assertEqual(["testdir2"], sorted(d1))
 743         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 744         self.assertEqual(["file1.txt"], d1)
 745
 746         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 747         assertRegex(self, collection2["manifest_text"],
 748             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 749
 750
 751 class FuseUpdateFromEventTest(MountTestBase):
 752     def runTest(self):
 753         collection = arvados.collection.Collection(api_client=self.api)
 754         collection.save_new()
 755
 756         m = self.make_mount(fuse.CollectionDirectory)
 757         with llfuse.lock:
 758             m.new_collection(collection.api_response(), collection)
 759
 760         self.operations.listen_for_events()
 761
 762         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 763         self.assertEqual([], sorted(d1))
 764
 765         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 766             with collection2.open("file1.txt", "w") as f:
 767                 f.write("foo")
 768
 769         for attempt in AssertWithTimeout(10):
 770             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 771
 772
 773 class FuseDeleteProjectEventTest(MountTestBase):
 774     def runTest(self):
 775
 776         aproject = self.api.groups().create(body={
 777             "name": "aproject",
 778             "group_class": "project"
 779         }).execute()
 780
 781         bproject = self.api.groups().create(body={
 782             "name": "bproject",
 783             "group_class": "project",
 784             "owner_uuid": aproject["uuid"]
 785         }).execute()
 786
 787         self.make_mount(fuse.ProjectDirectory,
 788                         project_object=self.api.users().current().execute())
 789
 790         self.operations.listen_for_events()
 791
 792         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 793         self.assertEqual(["bproject"], sorted(d1))
 794
 795         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 796
 797         for attempt in AssertWithTimeout(10):
 798             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 799
 800
 801 def fuseFileConflictTestHelper(mounttmp, uuid, keeptmp, settings):
 802     class Test(unittest.TestCase):
 803         def runTest(self):
 804             os.environ['KEEP_LOCAL_STORE'] = keeptmp
 805
 806             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 807                 with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
 808                     with collection2.open("file1.txt", "w") as f2:
 809                         f2.write("foo")
 810                 f.write("bar")
 811
 812             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 813             self.assertEqual(len(d1), 2)
 814
 815             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 816                 self.assertEqual(f.read(), "bar")
 817
 818             assertRegex(self, d1[1],
 819                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 820
 821             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 822                 self.assertEqual(f.read(), "foo")
 823
 824     Test().runTest()
 825
 826 class FuseFileConflictTest(MountTestBase):
 827     def runTest(self):
 828         collection = arvados.collection.Collection(api_client=self.api)
 829         collection.save_new()
 830
 831         m = self.make_mount(fuse.CollectionDirectory)
 832         with llfuse.lock:
 833             m.new_collection(collection.api_response(), collection)
 834
 835         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 836         self.assertEqual([], sorted(d1))
 837
 838         # See note in MountTestBase.setUp
 839         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
 840
 841
 842 def fuseUnlinkOpenFileTest(mounttmp):
 843     class Test(unittest.TestCase):
 844         def runTest(self):
 845             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 846                 f.write("foo")
 847
 848                 d1 = llfuse.listdir(os.path.join(mounttmp))
 849                 self.assertEqual(["file1.txt"], sorted(d1))
 850
 851                 os.remove(os.path.join(mounttmp, "file1.txt"))
 852
 853                 d1 = llfuse.listdir(os.path.join(mounttmp))
 854                 self.assertEqual([], sorted(d1))
 855
 856                 f.seek(0)
 857                 self.assertEqual(f.read(), "foo")
 858                 f.write("bar")
 859
 860                 f.seek(0)
 861                 self.assertEqual(f.read(), "foobar")
 862
 863     Test().runTest()
 864
 865 class FuseUnlinkOpenFileTest(MountTestBase):
 866     def runTest(self):
 867         collection = arvados.collection.Collection(api_client=self.api)
 868         collection.save_new()
 869
 870         m = self.make_mount(fuse.CollectionDirectory)
 871         with llfuse.lock:
 872             m.new_collection(collection.api_response(), collection)
 873
 874         # See note in MountTestBase.setUp
 875         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 876
 877         self.assertEqual(collection.manifest_text(), "")
 878
 879
 880 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 881     class Test(unittest.TestCase):
 882         def runTest(self):
 883             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 884                 f.write("Hello world!")
 885
 886             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 887             self.assertEqual(["file1.txt"], sorted(d1))
 888             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 889             self.assertEqual([], sorted(d1))
 890
 891     Test().runTest()
 892
 893 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 894     class Test(unittest.TestCase):
 895         def runTest(self):
 896             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 897
 898             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 899             self.assertEqual([], sorted(d1))
 900             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 901             self.assertEqual(["file2.txt"], sorted(d1))
 902
 903     Test().runTest()
 904
 905 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 906     def runTest(self):
 907         collection1 = arvados.collection.Collection(api_client=self.api)
 908         collection1.save_new()
 909
 910         collection2 = arvados.collection.Collection(api_client=self.api)
 911         collection2.save_new()
 912
 913         m = self.make_mount(fuse.MagicDirectory)
 914
 915         # See note in MountTestBase.setUp
 916         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 917                                                   collection1.manifest_locator(),
 918                                                   collection2.manifest_locator()))
 919
 920         collection1.update()
 921         collection2.update()
 922
 923         assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 924         self.assertEqual(collection2.manifest_text(), "")
 925
 926         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 927                                                   collection1.manifest_locator(),
 928                                                   collection2.manifest_locator()))
 929
 930         collection1.update()
 931         collection2.update()
 932
 933         self.assertEqual(collection1.manifest_text(), "")
 934         assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 935
 936         collection1.stop_threads()
 937         collection2.stop_threads()
 938
 939
 940 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 941     class Test(unittest.TestCase):
 942         def runTest(self):
 943             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 944             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 945                 f.write("Hello world!")
 946
 947             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 948             self.assertEqual(["testdir"], sorted(d1))
 949             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 950             self.assertEqual(["file1.txt"], sorted(d1))
 951
 952             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 953             self.assertEqual([], sorted(d1))
 954
 955     Test().runTest()
 956
 957
 958 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 959     class Test(unittest.TestCase):
 960         def runTest(self):
 961             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 962
 963             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 964             self.assertEqual([], sorted(d1))
 965
 966             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 967             self.assertEqual(["testdir2"], sorted(d1))
 968             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 969             self.assertEqual(["file1.txt"], sorted(d1))
 970
 971             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 972                 self.assertEqual(f.read(), "Hello world!")
 973
 974     Test().runTest()
 975
 976 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 977     def runTest(self):
 978         collection1 = arvados.collection.Collection(api_client=self.api)
 979         collection1.save_new()
 980
 981         collection2 = arvados.collection.Collection(api_client=self.api)
 982         collection2.save_new()
 983
 984         m = self.make_mount(fuse.MagicDirectory)
 985
 986         # See note in MountTestBase.setUp
 987         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 988                                                   collection1.manifest_locator(),
 989                                                   collection2.manifest_locator()))
 990
 991         collection1.update()
 992         collection2.update()
 993
 994         assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 995         self.assertEqual(collection2.manifest_text(), "")
 996
 997         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 998                                                   collection1.manifest_locator(),
 999                                                   collection2.manifest_locator()))
1000
1001         collection1.update()
1002         collection2.update()
1003
1004         self.assertEqual(collection1.manifest_text(), "")
1005         assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
1006
1007         collection1.stop_threads()
1008         collection2.stop_threads()
1009
1010 def fuseProjectMkdirTestHelper1(mounttmp):
1011     class Test(unittest.TestCase):
1012         def runTest(self):
1013             os.mkdir(os.path.join(mounttmp, "testcollection"))
1014             with self.assertRaises(OSError):
1015                 os.mkdir(os.path.join(mounttmp, "testcollection"))
1016     Test().runTest()
1017
1018 def fuseProjectMkdirTestHelper2(mounttmp):
1019     class Test(unittest.TestCase):
1020         def runTest(self):
1021             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1022                 f.write("Hello world!")
1023             with self.assertRaises(OSError):
1024                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1025             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1026             with self.assertRaises(OSError):
1027                 os.remove(os.path.join(mounttmp, "testcollection"))
1028             os.rmdir(os.path.join(mounttmp, "testcollection"))
1029     Test().runTest()
1030
1031 class FuseProjectMkdirRmdirTest(MountTestBase):
1032     def runTest(self):
1033         self.make_mount(fuse.ProjectDirectory,
1034                         project_object=self.api.users().current().execute())
1035
1036         d1 = llfuse.listdir(self.mounttmp)
1037         self.assertNotIn('testcollection', d1)
1038
1039         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1040
1041         d1 = llfuse.listdir(self.mounttmp)
1042         self.assertIn('testcollection', d1)
1043
1044         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1045
1046         d1 = llfuse.listdir(self.mounttmp)
1047         self.assertNotIn('testcollection', d1)
1048
1049
1050 def fuseProjectMvTestHelper1(mounttmp):
1051     class Test(unittest.TestCase):
1052         def runTest(self):
1053             d1 = llfuse.listdir(mounttmp)
1054             self.assertNotIn('testcollection', d1)
1055
1056             os.mkdir(os.path.join(mounttmp, "testcollection"))
1057
1058             d1 = llfuse.listdir(mounttmp)
1059             self.assertIn('testcollection', d1)
1060
1061             with self.assertRaises(OSError):
1062                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1063
1064             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1065
1066             d1 = llfuse.listdir(mounttmp)
1067             self.assertNotIn('testcollection', d1)
1068
1069             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1070             self.assertIn('testcollection', d1)
1071
1072     Test().runTest()
1073
1074 class FuseProjectMvTest(MountTestBase):
1075     def runTest(self):
1076         self.make_mount(fuse.ProjectDirectory,
1077                         project_object=self.api.users().current().execute())
1078
1079         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1080
1081
1082 def fuseFsyncTestHelper(mounttmp, k):
1083     class Test(unittest.TestCase):
1084         def runTest(self):
1085             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1086             os.fsync(fd)
1087             os.close(fd)
1088
1089     Test().runTest()
1090
1091 class FuseFsyncTest(FuseMagicTest):
1092     def runTest(self):
1093         self.make_mount(fuse.MagicDirectory)
1094         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1095
1096
1097 class MagicDirApiError(FuseMagicTest):
1098     def setUp(self):
1099         api = mock.MagicMock()
1100         api.keep.block_cache = mock.MagicMock(cache_max=1)
1101         super(MagicDirApiError, self).setUp(api=api)
1102         api.collections().get().execute.side_effect = iter([
1103             Exception('API fail'),
1104             {
1105                 "manifest_text": self.test_manifest,
1106                 "portable_data_hash": self.test_manifest_pdh,
1107             },
1108         ])
1109         api.keep.get.side_effect = Exception('Keep fail')
1110
1111     def runTest(self):
1112         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1113             self.make_mount(fuse.MagicDirectory)
1114
1115             self.operations.inodes.inode_cache.cap = 1
1116             self.operations.inodes.inode_cache.min_entries = 2
1117
1118             with self.assertRaises(OSError):
1119                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1120
1121             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1122
1123
1124 class SanitizeFilenameTest(MountTestBase):
1125     def test_sanitize_filename(self):
1126         pdir = fuse.ProjectDirectory(1, {}, self.api, 0, False, project_object=self.api.users().current().execute())
1127         acceptable = [
1128             "foo.txt",
1129             ".foo",
1130             "..foo",
1131             "...",
1132             "foo...",
1133             "foo..",
1134             "foo.",
1135             "-",
1136             "\x01\x02\x03",
1137             ]
1138         unacceptable = [
1139             "f\00",
1140             "\00\00",
1141             "/foo",
1142             "foo/",
1143             "//",
1144             ]
1145         for f in acceptable:
1146             self.assertEqual(f, pdir.sanitize_filename(f))
1147         for f in unacceptable:
1148             self.assertNotEqual(f, pdir.sanitize_filename(f))
1149             # The sanitized filename should be the same length, though.
1150             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1151         # Special cases
1152         self.assertEqual("_", pdir.sanitize_filename(""))
1153         self.assertEqual("_", pdir.sanitize_filename("."))
1154         self.assertEqual("__", pdir.sanitize_filename(".."))
1155
1156
1157 class FuseMagicTestPDHOnly(MountTestBase):
1158     def setUp(self, api=None):
1159         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1160
1161         cw = arvados.CollectionWriter()
1162
1163         cw.start_new_file('thing1.txt')
1164         cw.write("data 1")
1165
1166         self.testcollection = cw.finish()
1167         self.test_manifest = cw.manifest_text()
1168         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1169         self.testcollectionuuid = str(created['uuid'])
1170
1171     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1172         if skip_pdh_only is True:
1173             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1174         else:
1175             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1176
1177         mount_ls = llfuse.listdir(self.mounttmp)
1178         self.assertIn('README', mount_ls)
1179         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1180                              arvados.util.uuid_pattern.match(fn)
1181                              for fn in mount_ls),
1182                          "new FUSE MagicDirectory lists Collection")
1183
1184         # look up using pdh should succeed in all cases
1185         self.assertDirContents(self.testcollection, ['thing1.txt'])
1186         self.assertDirContents(os.path.join('by_id', self.testcollection),
1187                                ['thing1.txt'])
1188         mount_ls = llfuse.listdir(self.mounttmp)
1189         self.assertIn('README', mount_ls)
1190         self.assertIn(self.testcollection, mount_ls)
1191         self.assertIn(self.testcollection,
1192                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1193
1194         files = {}
1195         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1196
1197         for k, v in viewitems(files):
1198             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1199                 self.assertEqual(v, f.read().decode())
1200
1201         # look up using uuid should fail when pdh_only is set
1202         if pdh_only is True:
1203             with self.assertRaises(OSError):
1204                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1205                                ['thing1.txt'])
1206         else:
1207             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1208                                ['thing1.txt'])
1209
1210     def test_with_pdh_only_true(self):
1211         self.verify_pdh_only(pdh_only=True)
1212
1213     def test_with_pdh_only_false(self):
1214         self.verify_pdh_only(pdh_only=False)
1215
1216     def test_with_default_by_id(self):
1217         self.verify_pdh_only(skip_pdh_only=True)
1218
1219
1220 class SlashSubstitutionTest(IntegrationTest):
1221     mnt_args = [
1222         '--read-write',
1223         '--mount-home', 'zzz',
1224     ]
1225
1226     def setUp(self):
1227         super(SlashSubstitutionTest, self).setUp()
1228         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1229         self.api.config = lambda: {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1230         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1231         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1232         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1233
1234     @IntegrationTest.mount(argv=mnt_args)
1235     @mock.patch('arvados.util.get_config_once')
1236     def test_slash_substitution_before_listing(self, get_config_once):
1237         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1238         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1239         self.checkContents()
1240     @staticmethod
1241     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1242         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1243             f.write('xxx')
1244         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1245             f.write('foo')
1246
1247     @IntegrationTest.mount(argv=mnt_args)
1248     @mock.patch('arvados.util.get_config_once')
1249     def test_slash_substitution_after_listing(self, get_config_once):
1250         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1251         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1252         self.checkContents()
1253     @staticmethod
1254     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1255         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1256             f.write('xxx')
1257         os.listdir(tmpdir)
1258         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1259             f.write('foo')
1260
1261     def checkContents(self):
1262         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1263         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1264
1265     @IntegrationTest.mount(argv=mnt_args)
1266     @mock.patch('arvados.util.get_config_once')
1267     def test_slash_substitution_conflict(self, get_config_once):
1268         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1269         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1270         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1271         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1272         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1273         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1274     @staticmethod
1275     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1276         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1277             f.write('foo')
1278
1279 class StorageClassesTest(IntegrationTest):
1280     mnt_args = [
1281         '--read-write',
1282         '--mount-home', 'homedir',
1283     ]
1284
1285     def setUp(self):
1286         super(StorageClassesTest, self).setUp()
1287         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
1288
1289     @IntegrationTest.mount(argv=mnt_args)
1290     def test_collection_default_storage_classes(self):
1291         coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
1292         self.api.collections().create(body={'name':'a_collection'}).execute()
1293         self.pool_test(coll_path)
1294     @staticmethod
1295     def _test_collection_default_storage_classes(self, coll):
1296         self.assertEqual(storage_classes_desired(coll), ['default'])
1297
1298     @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
1299     def test_collection_custom_storage_classes(self):
1300         coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
1301         os.mkdir(coll_path)
1302         self.pool_test(coll_path)
1303     @staticmethod
1304     def _test_collection_custom_storage_classes(self, coll):
1305         self.assertEqual(storage_classes_desired(coll), ['foo'])
1306
1307 def _readonlyCollectionTestHelper(mounttmp):
1308     f = open(os.path.join(mounttmp, 'thing1.txt'), 'rt')
1309     # Testing that close() doesn't raise an error.
1310     f.close()
1311
1312 class ReadonlyCollectionTest(MountTestBase):
1313     def setUp(self):
1314         super(ReadonlyCollectionTest, self).setUp()
1315         cw = arvados.collection.Collection()
1316         with cw.open('thing1.txt', 'wt') as f:
1317             f.write("data 1")
1318         cw.save_new(owner_uuid=run_test_server.fixture("groups")["aproject"]["uuid"])
1319         self.testcollection = cw.api_response()
1320
1321     def runTest(self):
1322         settings = arvados.config.settings().copy()
1323         settings["ARVADOS_API_TOKEN"] = run_test_server.fixture("api_client_authorizations")["project_viewer"]["api_token"]
1324         self.api = arvados.safeapi.ThreadSafeApiCache(settings)
1325         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection, enable_write=False)
1326
1327         self.pool.apply(_readonlyCollectionTestHelper, (self.mounttmp,))