services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 import errno
   6 import json
   7 import llfuse
   8 import logging
   9 import os
  10 import subprocess
  11 import time
  12 import unittest
  13 import tempfile
  14
  15 from pathlib import Path
  16 from unittest import mock
  17
  18 import arvados
  19 import arvados_fuse as fuse
  20 import parameterized
  21
  22 from arvados_fuse import fusedir
  23
  24 from . import run_test_server
  25 from .integration_test import IntegrationTest
  26 from .mount_test_base import MountTestBase
  27 from .test_tmp_collection import storage_classes_desired
  28
  29 logger = logging.getLogger('arvados.arv-mount')
  30
  31 class AssertWithTimeout(object):
  32     """Allow some time for an assertion to pass."""
  33
  34     def __init__(self, timeout=0):
  35         self.timeout = timeout
  36
  37     def __iter__(self):
  38         self.deadline = time.time() + self.timeout
  39         self.done = False
  40         return self
  41
  42     def __next__(self):
  43         if self.done:
  44             raise StopIteration
  45         return self.attempt
  46
  47     def attempt(self, fn, *args, **kwargs):
  48         try:
  49             fn(*args, **kwargs)
  50         except AssertionError:
  51             if time.time() > self.deadline:
  52                 raise
  53             time.sleep(0.1)
  54         else:
  55             self.done = True
  56
  57 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
  58 class FuseMountTest(MountTestBase):
  59     def setUp(self):
  60         super(FuseMountTest, self).setUp()
  61
  62         cw = arvados.CollectionWriter()
  63
  64         cw.start_new_file('thing1.txt')
  65         cw.write("data 1")
  66         cw.start_new_file('thing2.txt')
  67         cw.write("data 2")
  68
  69         cw.start_new_stream('dir1')
  70         cw.start_new_file('thing3.txt')
  71         cw.write("data 3")
  72         cw.start_new_file('thing4.txt')
  73         cw.write("data 4")
  74
  75         cw.start_new_stream('dir2')
  76         cw.start_new_file('thing5.txt')
  77         cw.write("data 5")
  78         cw.start_new_file('thing6.txt')
  79         cw.write("data 6")
  80
  81         cw.start_new_stream('dir2/dir3')
  82         cw.start_new_file('thing7.txt')
  83         cw.write("data 7")
  84
  85         cw.start_new_file('thing8.txt')
  86         cw.write("data 8")
  87
  88         cw.start_new_stream('edgecases')
  89         for f in ":/.../-/*/ ".split("/"):
  90             cw.start_new_file(f)
  91             cw.write('x')
  92
  93         for f in ":/.../-/*/ ".split("/"):
  94             cw.start_new_stream('edgecases/dirs/' + f)
  95             cw.start_new_file('x/x')
  96             cw.write('x')
  97
  98         self.testcollection = cw.finish()
  99         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
 100
 101     def runTest(self):
 102         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
 103
 104         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 105                                       'edgecases', 'dir1', 'dir2'])
 106         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 107         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 108         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 109         self.assertDirContents('edgecases',
 110                                "dirs/:/.../-/*/ ".split("/"))
 111         self.assertDirContents('edgecases/dirs',
 112                                ":/.../-/*/ ".split("/"))
 113
 114         files = {'thing1.txt': 'data 1',
 115                  'thing2.txt': 'data 2',
 116                  'dir1/thing3.txt': 'data 3',
 117                  'dir1/thing4.txt': 'data 4',
 118                  'dir2/thing5.txt': 'data 5',
 119                  'dir2/thing6.txt': 'data 6',
 120                  'dir2/dir3/thing7.txt': 'data 7',
 121                  'dir2/dir3/thing8.txt': 'data 8'}
 122
 123         for k, v in files.items():
 124             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 125                 self.assertEqual(v, f.read().decode())
 126
 127
 128 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 129 class FuseMagicTest(MountTestBase):
 130     def setUp(self, api=None):
 131         super(FuseMagicTest, self).setUp(api=api)
 132
 133         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 134         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
 135         self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
 136         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 137         self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
 138
 139         cw = arvados.CollectionWriter()
 140
 141         cw.start_new_file('thing1.txt')
 142         cw.write("data 1")
 143
 144         self.testcollection = cw.finish()
 145         self.test_manifest = cw.manifest_text()
 146         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 147         self.test_manifest_pdh = coll['portable_data_hash']
 148
 149     def runTest(self):
 150         self.make_mount(fuse.MagicDirectory)
 151
 152         mount_ls = llfuse.listdir(self.mounttmp)
 153         self.assertIn('README', mount_ls)
 154         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 155                              arvados.util.uuid_pattern.match(fn)
 156                              for fn in mount_ls),
 157                          "new FUSE MagicDirectory has no collections or projects")
 158         self.assertDirContents(self.testcollection, ['thing1.txt'])
 159         self.assertDirContents(os.path.join('by_id', self.testcollection),
 160                                ['thing1.txt'])
 161         self.assertIn(self.collection_in_test_project,
 162                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 163         self.assertIn(self.collection_in_test_project,
 164                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 165         self.assertIn(self.collection_in_filter_group,
 166                       llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
 167         self.assertIn(self.collection_in_filter_group,
 168                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
 169
 170
 171         mount_ls = llfuse.listdir(self.mounttmp)
 172         self.assertIn('README', mount_ls)
 173         self.assertIn(self.testcollection, mount_ls)
 174         self.assertIn(self.testcollection,
 175                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 176         self.assertIn(self.test_project, mount_ls)
 177         self.assertIn(self.test_project,
 178                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 179         self.assertIn(self.filter_group,
 180                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 181
 182         with self.assertRaises(OSError):
 183             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 184
 185         files = {}
 186         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 187
 188         for k, v in files.items():
 189             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 190                 self.assertEqual(v, f.read().decode())
 191
 192
 193 class FuseTagsTest(MountTestBase):
 194     def runTest(self):
 195         self.make_mount(fuse.TagsDirectory)
 196
 197         d1 = llfuse.listdir(self.mounttmp)
 198         d1.sort()
 199         self.assertEqual(['foo_tag'], d1)
 200
 201         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 202         d2.sort()
 203         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 204
 205         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 206         d3.sort()
 207         self.assertEqual(['foo'], d3)
 208
 209
 210 class FuseTagsUpdateTest(MountTestBase):
 211     def tag_collection(self, coll_uuid, tag_name):
 212         return self.api.links().create(
 213             body={'link': {'head_uuid': coll_uuid,
 214                            'link_class': 'tag',
 215                            'name': tag_name,
 216         }}).execute()
 217
 218     def runTest(self):
 219         self.make_mount(fuse.TagsDirectory, poll_time=1)
 220
 221         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 222
 223         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 224         self.tag_collection(bar_uuid, 'fuse_test_tag')
 225         for attempt in AssertWithTimeout(10):
 226             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 227         self.assertDirContents('fuse_test_tag', [bar_uuid])
 228
 229         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 230         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 231         for attempt in AssertWithTimeout(10):
 232             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 233
 234         self.api.links().delete(uuid=l['uuid']).execute()
 235         for attempt in AssertWithTimeout(10):
 236             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 237
 238
 239 def fuseSharedTestHelper(mounttmp):
 240     class Test(unittest.TestCase):
 241         def runTest(self):
 242             # Double check that we can open and read objects in this folder as a file,
 243             # and that its contents are what we expect.
 244             baz_path = os.path.join(
 245                 mounttmp,
 246                 'FUSE User',
 247                 'FUSE Test Project',
 248                 'collection in FUSE project',
 249                 'baz')
 250             with open(baz_path) as f:
 251                 self.assertEqual("baz", f.read())
 252
 253             # check mtime on collection
 254             st = os.stat(baz_path)
 255             try:
 256                 mtime = st.st_mtime_ns // 1000000000
 257             except AttributeError:
 258                 mtime = st.st_mtime
 259             self.assertEqual(mtime, 1391448174)
 260
 261             # shared_dirs is a list of the directories exposed
 262             # by fuse.SharedDirectory (i.e. any object visible
 263             # to the current user)
 264             shared_dirs = llfuse.listdir(mounttmp)
 265             shared_dirs.sort()
 266             self.assertIn('FUSE User', shared_dirs)
 267
 268             # fuse_user_objs is a list of the objects owned by the FUSE
 269             # test user (which present as files in the 'FUSE User'
 270             # directory)
 271             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 272             fuse_user_objs.sort()
 273             self.assertEqual(['FUSE Test Project',                    # project owned by user
 274                               'collection #1 owned by FUSE',          # collection owned by user
 275                               'collection #2 owned by FUSE'          # collection owned by user
 276                           ], fuse_user_objs)
 277
 278             # test_proj_files is a list of the files in the FUSE Test Project.
 279             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 280             test_proj_files.sort()
 281             self.assertEqual(['collection in FUSE project'
 282                           ], test_proj_files)
 283
 284
 285     Test().runTest()
 286
 287 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 288 class FuseSharedTest(MountTestBase):
 289     def runTest(self):
 290         self.make_mount(fuse.SharedDirectory,
 291                         exclude=self.api.users().current().execute()['uuid'])
 292         keep = arvados.keep.KeepClient()
 293         keep.put("baz".encode())
 294
 295         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 296
 297
 298 class FuseHomeTest(MountTestBase):
 299     def runTest(self):
 300         self.make_mount(fuse.ProjectDirectory,
 301                         project_object=self.api.users().current().execute())
 302
 303         d1 = llfuse.listdir(self.mounttmp)
 304         self.assertIn('Unrestricted public data', d1)
 305
 306         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 307         public_project = run_test_server.fixture('groups')[
 308             'anonymously_accessible_project']
 309         found_in = 0
 310         found_not_in = 0
 311         for name, item in run_test_server.fixture('collections').items():
 312             if 'name' not in item:
 313                 pass
 314             elif item['owner_uuid'] == public_project['uuid']:
 315                 self.assertIn(item['name'], d2)
 316                 found_in += 1
 317             else:
 318                 # Artificial assumption here: there is no public
 319                 # collection fixture with the same name as a
 320                 # non-public collection.
 321                 self.assertNotIn(item['name'], d2)
 322                 found_not_in += 1
 323         self.assertNotEqual(0, found_in)
 324         self.assertNotEqual(0, found_not_in)
 325
 326         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 327         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 328
 329
 330 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 331     class Test(unittest.TestCase):
 332         def runTest(self):
 333             d1 = llfuse.listdir(mounttmp)
 334             self.assertEqual(["file1.txt"], d1)
 335             with open(os.path.join(mounttmp, "file1.txt")) as f:
 336                 self.assertEqual("blub", f.read())
 337     Test().runTest()
 338
 339 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 340     class Test(unittest.TestCase):
 341         def runTest(self):
 342             d1 = llfuse.listdir(mounttmp)
 343             self.assertEqual(["file1.txt"], d1)
 344             with open(os.path.join(mounttmp, "file1.txt")) as f:
 345                 self.assertEqual("plnp", f.read())
 346     Test().runTest()
 347
 348 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 349 class FuseModifyFileTest(MountTestBase):
 350     def runTest(self):
 351         collection = arvados.collection.Collection(api_client=self.api)
 352         with collection.open("file1.txt", "w") as f:
 353             f.write("blub")
 354
 355         collection.save_new()
 356
 357         m = self.make_mount(fuse.CollectionDirectory)
 358         with llfuse.lock:
 359             m.new_collection(collection.api_response(), collection)
 360
 361         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 362
 363         with collection.open("file1.txt", "w") as f:
 364             f.write("plnp")
 365
 366         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 367
 368
 369 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 370 class FuseAddFileToCollectionTest(MountTestBase):
 371     def runTest(self):
 372         collection = arvados.collection.Collection(api_client=self.api)
 373         with collection.open("file1.txt", "w") as f:
 374             f.write("blub")
 375
 376         collection.save_new()
 377
 378         m = self.make_mount(fuse.CollectionDirectory)
 379         with llfuse.lock:
 380             m.new_collection(collection.api_response(), collection)
 381
 382         d1 = llfuse.listdir(self.mounttmp)
 383         self.assertEqual(["file1.txt"], d1)
 384
 385         with collection.open("file2.txt", "w") as f:
 386             f.write("plnp")
 387
 388         d1 = llfuse.listdir(self.mounttmp)
 389         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 390
 391
 392 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 393 class FuseRemoveFileFromCollectionTest(MountTestBase):
 394     def runTest(self):
 395         collection = arvados.collection.Collection(api_client=self.api)
 396         with collection.open("file1.txt", "w") as f:
 397             f.write("blub")
 398
 399         with collection.open("file2.txt", "w") as f:
 400             f.write("plnp")
 401
 402         collection.save_new()
 403
 404         m = self.make_mount(fuse.CollectionDirectory)
 405         with llfuse.lock:
 406             m.new_collection(collection.api_response(), collection)
 407
 408         d1 = llfuse.listdir(self.mounttmp)
 409         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 410
 411         collection.remove("file2.txt")
 412
 413         d1 = llfuse.listdir(self.mounttmp)
 414         self.assertEqual(["file1.txt"], d1)
 415
 416
 417 def fuseCreateFileTestHelper(mounttmp):
 418     class Test(unittest.TestCase):
 419         def runTest(self):
 420             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 421                 pass
 422     Test().runTest()
 423
 424 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 425 class FuseCreateFileTest(MountTestBase):
 426     def runTest(self):
 427         collection = arvados.collection.Collection(api_client=self.api)
 428         collection.save_new()
 429
 430         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 431         self.assertEqual(collection2["manifest_text"], "")
 432
 433         collection.save_new()
 434
 435         m = self.make_mount(fuse.CollectionDirectory)
 436         with llfuse.lock:
 437             m.new_collection(collection.api_response(), collection)
 438         self.assertTrue(m.writable())
 439
 440         self.assertNotIn("file1.txt", collection)
 441
 442         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 443
 444         self.assertIn("file1.txt", collection)
 445
 446         d1 = llfuse.listdir(self.mounttmp)
 447         self.assertEqual(["file1.txt"], d1)
 448
 449         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 450         self.assertRegex(collection2["manifest_text"],
 451             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 452
 453
 454 def fuseWriteFileTestHelperWriteFile(mounttmp):
 455     class Test(unittest.TestCase):
 456         def runTest(self):
 457             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 458                 f.write("Hello world!")
 459     Test().runTest()
 460
 461 def fuseWriteFileTestHelperReadFile(mounttmp):
 462     class Test(unittest.TestCase):
 463         def runTest(self):
 464             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 465                 self.assertEqual(f.read(), "Hello world!")
 466     Test().runTest()
 467
 468 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 469 class FuseWriteFileTest(MountTestBase):
 470     def runTest(self):
 471         collection = arvados.collection.Collection(api_client=self.api)
 472         collection.save_new()
 473
 474         m = self.make_mount(fuse.CollectionDirectory)
 475         with llfuse.lock:
 476             m.new_collection(collection.api_response(), collection)
 477         self.assertTrue(m.writable())
 478
 479         self.assertNotIn("file1.txt", collection)
 480
 481         self.assertEqual(0, self.operations.write_counter.get())
 482         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 483         self.assertEqual(12, self.operations.write_counter.get())
 484
 485         with collection.open("file1.txt") as f:
 486             self.assertEqual(f.read(), "Hello world!")
 487
 488         self.assertEqual(0, self.operations.read_counter.get())
 489         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 490         self.assertEqual(12, self.operations.read_counter.get())
 491
 492         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 493         self.assertRegex(collection2["manifest_text"],
 494             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 495
 496
 497 def fuseUpdateFileTestHelper(mounttmp):
 498     class Test(unittest.TestCase):
 499         def runTest(self):
 500             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 501                 f.write("Hello world!")
 502
 503             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 504                 fr = f.read()
 505                 self.assertEqual(fr, "Hello world!")
 506                 f.seek(0)
 507                 f.write("Hola mundo!")
 508                 f.seek(0)
 509                 fr = f.read()
 510                 self.assertEqual(fr, "Hola mundo!!")
 511
 512             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 513                 self.assertEqual(f.read(), "Hola mundo!!")
 514
 515     Test().runTest()
 516
 517 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 518 class FuseUpdateFileTest(MountTestBase):
 519     def runTest(self):
 520         collection = arvados.collection.Collection(api_client=self.api)
 521         collection.save_new()
 522
 523         m = self.make_mount(fuse.CollectionDirectory)
 524         with llfuse.lock:
 525             m.new_collection(collection.api_response(), collection)
 526         self.assertTrue(m.writable())
 527
 528         # See note in MountTestBase.setUp
 529         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 530
 531         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 532         self.assertRegex(collection2["manifest_text"],
 533             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 534
 535
 536 def fuseMkdirTestHelper(mounttmp):
 537     class Test(unittest.TestCase):
 538         def runTest(self):
 539             with self.assertRaises(IOError):
 540                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 541                     f.write("Hello world!")
 542
 543             os.mkdir(os.path.join(mounttmp, "testdir"))
 544
 545             with self.assertRaises(OSError):
 546                 os.mkdir(os.path.join(mounttmp, "testdir"))
 547
 548             d1 = llfuse.listdir(mounttmp)
 549             self.assertEqual(["testdir"], d1)
 550
 551             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 552                 f.write("Hello world!")
 553
 554             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 555             self.assertEqual(["file1.txt"], d1)
 556
 557     Test().runTest()
 558
 559 class FuseMkdirTest(MountTestBase):
 560     def runTest(self):
 561         collection = arvados.collection.Collection(api_client=self.api)
 562         collection.save_new()
 563
 564         m = self.make_mount(fuse.CollectionDirectory)
 565         with llfuse.lock:
 566             m.new_collection(collection.api_response(), collection)
 567         self.assertTrue(m.writable())
 568
 569         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 570
 571         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 572         self.assertRegex(collection2["manifest_text"],
 573             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 574
 575
 576 def fuseRmTestHelperWriteFile(mounttmp):
 577     class Test(unittest.TestCase):
 578         def runTest(self):
 579             os.mkdir(os.path.join(mounttmp, "testdir"))
 580
 581             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 582                 f.write("Hello world!")
 583
 584     Test().runTest()
 585
 586 def fuseRmTestHelperDeleteFile(mounttmp):
 587     class Test(unittest.TestCase):
 588         def runTest(self):
 589             # Can't delete because it's not empty
 590             with self.assertRaises(OSError):
 591                 os.rmdir(os.path.join(mounttmp, "testdir"))
 592
 593             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 594             self.assertEqual(["file1.txt"], d1)
 595
 596             # Delete file
 597             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 598
 599             # Make sure it's empty
 600             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 601             self.assertEqual([], d1)
 602
 603             # Try to delete it again
 604             with self.assertRaises(OSError):
 605                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 606
 607     Test().runTest()
 608
 609 def fuseRmTestHelperRmdir(mounttmp):
 610     class Test(unittest.TestCase):
 611         def runTest(self):
 612             # Should be able to delete now that it is empty
 613             os.rmdir(os.path.join(mounttmp, "testdir"))
 614
 615             # Make sure it's empty
 616             d1 = llfuse.listdir(os.path.join(mounttmp))
 617             self.assertEqual([], d1)
 618
 619             # Try to delete it again
 620             with self.assertRaises(OSError):
 621                 os.rmdir(os.path.join(mounttmp, "testdir"))
 622
 623     Test().runTest()
 624
 625 class FuseRmTest(MountTestBase):
 626     def runTest(self):
 627         collection = arvados.collection.Collection(api_client=self.api)
 628         collection.save_new()
 629
 630         m = self.make_mount(fuse.CollectionDirectory)
 631         with llfuse.lock:
 632             m.new_collection(collection.api_response(), collection)
 633         self.assertTrue(m.writable())
 634
 635         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 636
 637         # Starting manifest
 638         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 639         self.assertRegex(collection2["manifest_text"],
 640             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 641         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 642
 643         # Empty directories are represented by an empty file named "."
 644         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 645         self.assertRegex(collection2["manifest_text"],
 646                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 647
 648         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 649
 650         # manifest should be empty now.
 651         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 652         self.assertEqual(collection2["manifest_text"], "")
 653
 654
 655 def fuseMvFileTestHelperWriteFile(mounttmp):
 656     class Test(unittest.TestCase):
 657         def runTest(self):
 658             os.mkdir(os.path.join(mounttmp, "testdir"))
 659
 660             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 661                 f.write("Hello world!")
 662
 663     Test().runTest()
 664
 665 def fuseMvFileTestHelperMoveFile(mounttmp):
 666     class Test(unittest.TestCase):
 667         def runTest(self):
 668             d1 = llfuse.listdir(os.path.join(mounttmp))
 669             self.assertEqual(["testdir"], d1)
 670             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 671             self.assertEqual(["file1.txt"], d1)
 672
 673             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 674
 675             d1 = llfuse.listdir(os.path.join(mounttmp))
 676             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 677             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 678             self.assertEqual([], d1)
 679
 680     Test().runTest()
 681
 682 class FuseMvFileTest(MountTestBase):
 683     def runTest(self):
 684         collection = arvados.collection.Collection(api_client=self.api)
 685         collection.save_new()
 686
 687         m = self.make_mount(fuse.CollectionDirectory)
 688         with llfuse.lock:
 689             m.new_collection(collection.api_response(), collection)
 690         self.assertTrue(m.writable())
 691
 692         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 693
 694         # Starting manifest
 695         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 696         self.assertRegex(collection2["manifest_text"],
 697             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 698
 699         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 700
 701         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 702         self.assertRegex(collection2["manifest_text"],
 703             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 704
 705
 706 def fuseRenameTestHelper(mounttmp):
 707     class Test(unittest.TestCase):
 708         def runTest(self):
 709             os.mkdir(os.path.join(mounttmp, "testdir"))
 710
 711             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 712                 f.write("Hello world!")
 713
 714     Test().runTest()
 715
 716 class FuseRenameTest(MountTestBase):
 717     def runTest(self):
 718         collection = arvados.collection.Collection(api_client=self.api)
 719         collection.save_new()
 720
 721         m = self.make_mount(fuse.CollectionDirectory)
 722         with llfuse.lock:
 723             m.new_collection(collection.api_response(), collection)
 724         self.assertTrue(m.writable())
 725
 726         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 727
 728         # Starting manifest
 729         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 730         self.assertRegex(collection2["manifest_text"],
 731             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 732
 733         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 734         self.assertEqual(["testdir"], d1)
 735         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 736         self.assertEqual(["file1.txt"], d1)
 737
 738         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 739
 740         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 741         self.assertEqual(["testdir2"], sorted(d1))
 742         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 743         self.assertEqual(["file1.txt"], d1)
 744
 745         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 746         self.assertRegex(collection2["manifest_text"],
 747             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 748
 749
 750 class FuseUpdateFromEventTest(MountTestBase):
 751     def runTest(self):
 752         collection = arvados.collection.Collection(api_client=self.api)
 753         collection.save_new()
 754
 755         m = self.make_mount(fuse.CollectionDirectory)
 756         with llfuse.lock:
 757             m.new_collection(collection.api_response(), collection)
 758
 759         self.operations.listen_for_events()
 760
 761         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 762         self.assertEqual([], sorted(d1))
 763
 764         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 765             with collection2.open("file1.txt", "w") as f:
 766                 f.write("foo")
 767
 768         for attempt in AssertWithTimeout(10):
 769             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 770
 771
 772 class FuseDeleteProjectEventTest(MountTestBase):
 773     def runTest(self):
 774
 775         aproject = self.api.groups().create(body={
 776             "name": "aproject",
 777             "group_class": "project"
 778         }).execute()
 779
 780         bproject = self.api.groups().create(body={
 781             "name": "bproject",
 782             "group_class": "project",
 783             "owner_uuid": aproject["uuid"]
 784         }).execute()
 785
 786         self.make_mount(fuse.ProjectDirectory,
 787                         project_object=self.api.users().current().execute())
 788
 789         self.operations.listen_for_events()
 790
 791         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 792         self.assertEqual(["bproject"], sorted(d1))
 793
 794         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 795
 796         for attempt in AssertWithTimeout(10):
 797             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 798
 799
 800 def fuseFileConflictTestHelper(mounttmp, uuid, keeptmp, settings):
 801     class Test(unittest.TestCase):
 802         def runTest(self):
 803             os.environ['KEEP_LOCAL_STORE'] = keeptmp
 804
 805             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 806                 with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
 807                     with collection2.open("file1.txt", "w") as f2:
 808                         f2.write("foo")
 809                 f.write("bar")
 810
 811             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 812             self.assertEqual(len(d1), 2)
 813
 814             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 815                 self.assertEqual(f.read(), "bar")
 816
 817             self.assertRegex(d1[1],
 818                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 819
 820             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 821                 self.assertEqual(f.read(), "foo")
 822
 823     Test().runTest()
 824
 825 class FuseFileConflictTest(MountTestBase):
 826     def runTest(self):
 827         collection = arvados.collection.Collection(api_client=self.api)
 828         collection.save_new()
 829
 830         m = self.make_mount(fuse.CollectionDirectory)
 831         with llfuse.lock:
 832             m.new_collection(collection.api_response(), collection)
 833
 834         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 835         self.assertEqual([], sorted(d1))
 836
 837         # See note in MountTestBase.setUp
 838         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
 839
 840
 841 def fuseUnlinkOpenFileTest(mounttmp):
 842     class Test(unittest.TestCase):
 843         def runTest(self):
 844             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 845                 f.write("foo")
 846
 847                 d1 = llfuse.listdir(os.path.join(mounttmp))
 848                 self.assertEqual(["file1.txt"], sorted(d1))
 849
 850                 os.remove(os.path.join(mounttmp, "file1.txt"))
 851
 852                 d1 = llfuse.listdir(os.path.join(mounttmp))
 853                 self.assertEqual([], sorted(d1))
 854
 855                 f.seek(0)
 856                 self.assertEqual(f.read(), "foo")
 857                 f.write("bar")
 858
 859                 f.seek(0)
 860                 self.assertEqual(f.read(), "foobar")
 861
 862     Test().runTest()
 863
 864 class FuseUnlinkOpenFileTest(MountTestBase):
 865     def runTest(self):
 866         collection = arvados.collection.Collection(api_client=self.api)
 867         collection.save_new()
 868
 869         m = self.make_mount(fuse.CollectionDirectory)
 870         with llfuse.lock:
 871             m.new_collection(collection.api_response(), collection)
 872
 873         # See note in MountTestBase.setUp
 874         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 875
 876         self.assertEqual(collection.manifest_text(), "")
 877
 878
 879 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 880     class Test(unittest.TestCase):
 881         def runTest(self):
 882             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 883                 f.write("Hello world!")
 884
 885             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 886             self.assertEqual(["file1.txt"], sorted(d1))
 887             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 888             self.assertEqual([], sorted(d1))
 889
 890     Test().runTest()
 891
 892 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 893     class Test(unittest.TestCase):
 894         def runTest(self):
 895             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 896
 897             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 898             self.assertEqual([], sorted(d1))
 899             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 900             self.assertEqual(["file2.txt"], sorted(d1))
 901
 902     Test().runTest()
 903
 904 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 905     def runTest(self):
 906         collection1 = arvados.collection.Collection(api_client=self.api)
 907         collection1.save_new()
 908
 909         collection2 = arvados.collection.Collection(api_client=self.api)
 910         collection2.save_new()
 911
 912         m = self.make_mount(fuse.MagicDirectory)
 913
 914         # See note in MountTestBase.setUp
 915         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 916                                                   collection1.manifest_locator(),
 917                                                   collection2.manifest_locator()))
 918
 919         collection1.update()
 920         collection2.update()
 921
 922         self.assertRegex(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 923         self.assertEqual(collection2.manifest_text(), "")
 924
 925         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 926                                                   collection1.manifest_locator(),
 927                                                   collection2.manifest_locator()))
 928
 929         collection1.update()
 930         collection2.update()
 931
 932         self.assertEqual(collection1.manifest_text(), "")
 933         self.assertRegex(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 934
 935         collection1.stop_threads()
 936         collection2.stop_threads()
 937
 938
 939 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 940     class Test(unittest.TestCase):
 941         def runTest(self):
 942             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 943             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 944                 f.write("Hello world!")
 945
 946             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 947             self.assertEqual(["testdir"], sorted(d1))
 948             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 949             self.assertEqual(["file1.txt"], sorted(d1))
 950
 951             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 952             self.assertEqual([], sorted(d1))
 953
 954     Test().runTest()
 955
 956
 957 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 958     class Test(unittest.TestCase):
 959         def runTest(self):
 960             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 961
 962             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 963             self.assertEqual([], sorted(d1))
 964
 965             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 966             self.assertEqual(["testdir2"], sorted(d1))
 967             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 968             self.assertEqual(["file1.txt"], sorted(d1))
 969
 970             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 971                 self.assertEqual(f.read(), "Hello world!")
 972
 973     Test().runTest()
 974
 975 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 976     def runTest(self):
 977         collection1 = arvados.collection.Collection(api_client=self.api)
 978         collection1.save_new()
 979
 980         collection2 = arvados.collection.Collection(api_client=self.api)
 981         collection2.save_new()
 982
 983         m = self.make_mount(fuse.MagicDirectory)
 984
 985         # See note in MountTestBase.setUp
 986         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 987                                                   collection1.manifest_locator(),
 988                                                   collection2.manifest_locator()))
 989
 990         collection1.update()
 991         collection2.update()
 992
 993         self.assertRegex(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 994         self.assertEqual(collection2.manifest_text(), "")
 995
 996         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 997                                                   collection1.manifest_locator(),
 998                                                   collection2.manifest_locator()))
 999
1000         collection1.update()
1001         collection2.update()
1002
1003         self.assertEqual(collection1.manifest_text(), "")
1004         self.assertRegex(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
1005
1006         collection1.stop_threads()
1007         collection2.stop_threads()
1008
1009 def fuseProjectMkdirTestHelper1(mounttmp):
1010     class Test(unittest.TestCase):
1011         def runTest(self):
1012             os.mkdir(os.path.join(mounttmp, "testcollection"))
1013             with self.assertRaises(OSError):
1014                 os.mkdir(os.path.join(mounttmp, "testcollection"))
1015     Test().runTest()
1016
1017 def fuseProjectMkdirTestHelper2(mounttmp):
1018     class Test(unittest.TestCase):
1019         def runTest(self):
1020             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1021                 f.write("Hello world!")
1022             with self.assertRaises(OSError):
1023                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1024             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1025             with self.assertRaises(OSError):
1026                 os.remove(os.path.join(mounttmp, "testcollection"))
1027             os.rmdir(os.path.join(mounttmp, "testcollection"))
1028     Test().runTest()
1029
1030 class FuseProjectMkdirRmdirTest(MountTestBase):
1031     def runTest(self):
1032         self.make_mount(fuse.ProjectDirectory,
1033                         project_object=self.api.users().current().execute())
1034
1035         d1 = llfuse.listdir(self.mounttmp)
1036         self.assertNotIn('testcollection', d1)
1037
1038         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1039
1040         d1 = llfuse.listdir(self.mounttmp)
1041         self.assertIn('testcollection', d1)
1042
1043         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1044
1045         d1 = llfuse.listdir(self.mounttmp)
1046         self.assertNotIn('testcollection', d1)
1047
1048
1049 def fuseProjectMvTestHelper1(mounttmp):
1050     class Test(unittest.TestCase):
1051         def runTest(self):
1052             d1 = llfuse.listdir(mounttmp)
1053             self.assertNotIn('testcollection', d1)
1054
1055             os.mkdir(os.path.join(mounttmp, "testcollection"))
1056
1057             d1 = llfuse.listdir(mounttmp)
1058             self.assertIn('testcollection', d1)
1059
1060             with self.assertRaises(OSError):
1061                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1062
1063             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1064
1065             d1 = llfuse.listdir(mounttmp)
1066             self.assertNotIn('testcollection', d1)
1067
1068             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1069             self.assertIn('testcollection', d1)
1070
1071     Test().runTest()
1072
1073 class FuseProjectMvTest(MountTestBase):
1074     def runTest(self):
1075         self.make_mount(fuse.ProjectDirectory,
1076                         project_object=self.api.users().current().execute())
1077
1078         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1079
1080
1081 def fuseFsyncTestHelper(mounttmp, k):
1082     class Test(unittest.TestCase):
1083         def runTest(self):
1084             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1085             os.fsync(fd)
1086             os.close(fd)
1087
1088     Test().runTest()
1089
1090 class FuseFsyncTest(FuseMagicTest):
1091     def runTest(self):
1092         self.make_mount(fuse.MagicDirectory)
1093         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1094
1095
1096 class MagicDirApiError(FuseMagicTest):
1097     def setUp(self):
1098         api = mock.MagicMock()
1099         api.keep.block_cache = mock.MagicMock(cache_max=1)
1100         super(MagicDirApiError, self).setUp(api=api)
1101         api.collections().get().execute.side_effect = iter([
1102             Exception('API fail'),
1103             {
1104                 "manifest_text": self.test_manifest,
1105                 "portable_data_hash": self.test_manifest_pdh,
1106             },
1107         ])
1108         api.keep.get.side_effect = Exception('Keep fail')
1109
1110     def runTest(self):
1111         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1112             self.make_mount(fuse.MagicDirectory)
1113
1114             self.operations.inodes.inode_cache.cap = 1
1115             self.operations.inodes.inode_cache.min_entries = 2
1116
1117             with self.assertRaises(OSError):
1118                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1119
1120             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1121
1122
1123 class SanitizeFilenameTest(MountTestBase):
1124     def test_sanitize_filename(self):
1125         pdir = fuse.ProjectDirectory(
1126             1, fuse.Inodes(None), self.api, 0, False, None,
1127             project_object=self.api.users().current().execute(),
1128         )
1129         acceptable = [
1130             "foo.txt",
1131             ".foo",
1132             "..foo",
1133             "...",
1134             "foo...",
1135             "foo..",
1136             "foo.",
1137             "-",
1138             "\x01\x02\x03",
1139             ]
1140         unacceptable = [
1141             "f\00",
1142             "\00\00",
1143             "/foo",
1144             "foo/",
1145             "//",
1146             ]
1147         for f in acceptable:
1148             self.assertEqual(f, pdir.sanitize_filename(f))
1149         for f in unacceptable:
1150             self.assertNotEqual(f, pdir.sanitize_filename(f))
1151             # The sanitized filename should be the same length, though.
1152             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1153         # Special cases
1154         self.assertEqual("_", pdir.sanitize_filename(""))
1155         self.assertEqual("_", pdir.sanitize_filename("."))
1156         self.assertEqual("__", pdir.sanitize_filename(".."))
1157
1158
1159 class FuseMagicTestPDHOnly(MountTestBase):
1160     def setUp(self, api=None):
1161         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1162
1163         cw = arvados.CollectionWriter()
1164
1165         cw.start_new_file('thing1.txt')
1166         cw.write("data 1")
1167
1168         self.testcollection = cw.finish()
1169         self.test_manifest = cw.manifest_text()
1170         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1171         self.testcollectionuuid = str(created['uuid'])
1172
1173     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1174         if skip_pdh_only is True:
1175             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1176         else:
1177             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1178
1179         mount_ls = llfuse.listdir(self.mounttmp)
1180         self.assertIn('README', mount_ls)
1181         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1182                              arvados.util.uuid_pattern.match(fn)
1183                              for fn in mount_ls),
1184                          "new FUSE MagicDirectory lists Collection")
1185
1186         # look up using pdh should succeed in all cases
1187         self.assertDirContents(self.testcollection, ['thing1.txt'])
1188         self.assertDirContents(os.path.join('by_id', self.testcollection),
1189                                ['thing1.txt'])
1190         mount_ls = llfuse.listdir(self.mounttmp)
1191         self.assertIn('README', mount_ls)
1192         self.assertIn(self.testcollection, mount_ls)
1193         self.assertIn(self.testcollection,
1194                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1195
1196         files = {}
1197         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1198
1199         for k, v in files.items():
1200             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1201                 self.assertEqual(v, f.read().decode())
1202
1203         # look up using uuid should fail when pdh_only is set
1204         if pdh_only is True:
1205             with self.assertRaises(OSError):
1206                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1207                                ['thing1.txt'])
1208         else:
1209             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1210                                ['thing1.txt'])
1211
1212     def test_with_pdh_only_true(self):
1213         self.verify_pdh_only(pdh_only=True)
1214
1215     def test_with_pdh_only_false(self):
1216         self.verify_pdh_only(pdh_only=False)
1217
1218     def test_with_default_by_id(self):
1219         self.verify_pdh_only(skip_pdh_only=True)
1220
1221
1222 class SlashSubstitutionTest(IntegrationTest):
1223     mnt_args = [
1224         '--read-write',
1225         '--mount-home', 'zzz',
1226         '--fsns', '[SLASH]'
1227     ]
1228
1229     def setUp(self):
1230         super(SlashSubstitutionTest, self).setUp()
1231
1232         self.api = arvados.safeapi.ThreadSafeApiCache(
1233             arvados.config.settings(),
1234             version='v1'
1235         )
1236         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1237         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1238         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1239
1240     @IntegrationTest.mount(argv=mnt_args)
1241     def test_slash_substitution_before_listing(self):
1242         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1243         self.checkContents()
1244     @staticmethod
1245     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1246         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1247             f.write('xxx')
1248         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1249             f.write('foo')
1250
1251     @IntegrationTest.mount(argv=mnt_args)
1252     @mock.patch('arvados.util.get_config_once')
1253     def test_slash_substitution_after_listing(self, get_config_once):
1254         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1255         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1256         self.checkContents()
1257     @staticmethod
1258     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1259         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1260             f.write('xxx')
1261         os.listdir(tmpdir)
1262         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1263             f.write('foo')
1264
1265     def checkContents(self):
1266         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1267         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1268
1269     @IntegrationTest.mount(argv=mnt_args)
1270     @mock.patch('arvados.util.get_config_once')
1271     def test_slash_substitution_conflict(self, get_config_once):
1272         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1273         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1274         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1275         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1276         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1277         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1278     @staticmethod
1279     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1280         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1281             f.write('foo')
1282
1283 class StorageClassesTest(IntegrationTest):
1284     mnt_args = [
1285         '--read-write',
1286         '--mount-home', 'homedir',
1287     ]
1288
1289     def setUp(self):
1290         super(StorageClassesTest, self).setUp()
1291         self.api = arvados.safeapi.ThreadSafeApiCache(
1292             arvados.config.settings(),
1293             version='v1',
1294         )
1295
1296     @IntegrationTest.mount(argv=mnt_args)
1297     def test_collection_default_storage_classes(self):
1298         coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
1299         self.api.collections().create(body={'name':'a_collection'}).execute()
1300         self.pool_test(coll_path)
1301     @staticmethod
1302     def _test_collection_default_storage_classes(self, coll):
1303         self.assertEqual(storage_classes_desired(coll), ['default'])
1304
1305     @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
1306     def test_collection_custom_storage_classes(self):
1307         coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
1308         os.mkdir(coll_path)
1309         self.pool_test(coll_path)
1310     @staticmethod
1311     def _test_collection_custom_storage_classes(self, coll):
1312         self.assertEqual(storage_classes_desired(coll), ['foo'])
1313
1314 def _readonlyCollectionTestHelper(mounttmp):
1315     f = open(os.path.join(mounttmp, 'thing1.txt'), 'rt')
1316     # Testing that close() doesn't raise an error.
1317     f.close()
1318
1319 class ReadonlyCollectionTest(MountTestBase):
1320     def setUp(self):
1321         super(ReadonlyCollectionTest, self).setUp()
1322         cw = arvados.collection.Collection()
1323         with cw.open('thing1.txt', 'wt') as f:
1324             f.write("data 1")
1325         cw.save_new(owner_uuid=run_test_server.fixture("groups")["aproject"]["uuid"])
1326         self.testcollection = cw.api_response()
1327
1328     def runTest(self):
1329         settings = arvados.config.settings().copy()
1330         settings["ARVADOS_API_TOKEN"] = run_test_server.fixture("api_client_authorizations")["project_viewer"]["api_token"]
1331         self.api = arvados.safeapi.ThreadSafeApiCache(settings, version='v1')
1332         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection, enable_write=False)
1333
1334         self.pool.apply(_readonlyCollectionTestHelper, (self.mounttmp,))
1335
1336
1337 @parameterized.parameterized_class([
1338     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1339         'project_object': run_test_server.fixture('users')['admin'],
1340     }},
1341     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1342         'project_object': run_test_server.fixture('groups')['public'],
1343     }},
1344 ])
1345 class UnsupportedCreateTest(MountTestBase):
1346     root_class = None
1347     root_kwargs = {}
1348
1349     def setUp(self):
1350         super().setUp()
1351         if 'prefs' in self.root_kwargs.get('project_object', ()):
1352             self.root_kwargs['project_object']['prefs'] = {}
1353         self.make_mount(self.root_class, **self.root_kwargs)
1354         # Make sure the directory knows about its top-level ents.
1355         os.listdir(self.mounttmp)
1356
1357     def test_create(self):
1358         test_path = Path(self.mounttmp, 'test_create')
1359         with self.assertRaises(OSError) as exc_check:
1360             with test_path.open('w'):
1361                 pass
1362         self.assertEqual(exc_check.exception.errno, errno.ENOTSUP)
1363
1364
1365 # FIXME: IMO, for consistency with the "create inside a project" case,
1366 # these operations should also return ENOTSUP instead of EPERM.
1367 # Right now they're returning EPERM because the clasess' writable() method
1368 # usually returns False, and the Operations class transforms that accordingly.
1369 # However, for cases where the mount will never be writable, I think ENOTSUP
1370 # is a clearer error: it lets the user know they can't fix the problem by
1371 # adding permissions in Arvados, etc.
1372 @parameterized.parameterized_class([
1373     {'root_class': fusedir.MagicDirectory,
1374      'preset_dir': 'by_id',
1375      'preset_file': 'README',
1376      },
1377
1378     {'root_class': fusedir.SharedDirectory,
1379      'root_kwargs': {
1380          'exclude': run_test_server.fixture('users')['admin']['uuid'],
1381      },
1382      'preset_dir': 'Active User',
1383      },
1384
1385     {'root_class': fusedir.TagDirectory,
1386      'root_kwargs': {
1387          'tag': run_test_server.fixture('links')['foo_collection_tag']['name'],
1388      },
1389      'preset_dir': run_test_server.fixture('collections')['foo_collection_in_aproject']['uuid'],
1390      },
1391
1392     {'root_class': fusedir.TagsDirectory,
1393      'preset_dir': run_test_server.fixture('links')['foo_collection_tag']['name'],
1394      },
1395 ])
1396 class UnsupportedOperationsTest(UnsupportedCreateTest):
1397     preset_dir = None
1398     preset_file = None
1399
1400     def test_create(self):
1401         test_path = Path(self.mounttmp, 'test_create')
1402         with self.assertRaises(OSError) as exc_check:
1403             with test_path.open('w'):
1404                 pass
1405         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1406
1407     def test_mkdir(self):
1408         test_path = Path(self.mounttmp, 'test_mkdir')
1409         with self.assertRaises(OSError) as exc_check:
1410             test_path.mkdir()
1411         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1412
1413     def test_rename(self):
1414         src_name = self.preset_dir or self.preset_file
1415         if src_name is None:
1416             return
1417         test_src = Path(self.mounttmp, src_name)
1418         test_dst = test_src.with_name('test_dst')
1419         with self.assertRaises(OSError) as exc_check:
1420             test_src.rename(test_dst)
1421         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1422
1423     def test_rmdir(self):
1424         if self.preset_dir is None:
1425             return
1426         test_path = Path(self.mounttmp, self.preset_dir)
1427         with self.assertRaises(OSError) as exc_check:
1428             test_path.rmdir()
1429         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1430
1431     def test_unlink(self):
1432         if self.preset_file is None:
1433             return
1434         test_path = Path(self.mounttmp, self.preset_file)
1435         with self.assertRaises(OSError) as exc_check:
1436             test_path.unlink()
1437         self.assertEqual(exc_check.exception.errno, errno.EPERM)