services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 from __future__ import absolute_import
   6 from future.utils import viewitems
   7 from builtins import str
   8 from builtins import object
   9 from pathlib import Path
  10 from six import assertRegex
  11 import errno
  12 import json
  13 import llfuse
  14 import logging
  15 import mock
  16 import os
  17 import subprocess
  18 import time
  19 import unittest
  20 import tempfile
  21 import parameterized
  22
  23 import arvados
  24 import arvados_fuse as fuse
  25 from arvados_fuse import fusedir
  26 from . import run_test_server
  27
  28 from .integration_test import IntegrationTest
  29 from .mount_test_base import MountTestBase
  30 from .test_tmp_collection import storage_classes_desired
  31
  32 logger = logging.getLogger('arvados.arv-mount')
  33
  34
  35 class AssertWithTimeout(object):
  36     """Allow some time for an assertion to pass."""
  37
  38     def __init__(self, timeout=0):
  39         self.timeout = timeout
  40
  41     def __iter__(self):
  42         self.deadline = time.time() + self.timeout
  43         self.done = False
  44         return self
  45
  46     def __next__(self):
  47         if self.done:
  48             raise StopIteration
  49         return self.attempt
  50
  51     def attempt(self, fn, *args, **kwargs):
  52         try:
  53             fn(*args, **kwargs)
  54         except AssertionError:
  55             if time.time() > self.deadline:
  56                 raise
  57             time.sleep(0.1)
  58         else:
  59             self.done = True
  60
  61 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
  62 class FuseMountTest(MountTestBase):
  63     def setUp(self):
  64         super(FuseMountTest, self).setUp()
  65
  66         cw = arvados.CollectionWriter()
  67
  68         cw.start_new_file('thing1.txt')
  69         cw.write("data 1")
  70         cw.start_new_file('thing2.txt')
  71         cw.write("data 2")
  72
  73         cw.start_new_stream('dir1')
  74         cw.start_new_file('thing3.txt')
  75         cw.write("data 3")
  76         cw.start_new_file('thing4.txt')
  77         cw.write("data 4")
  78
  79         cw.start_new_stream('dir2')
  80         cw.start_new_file('thing5.txt')
  81         cw.write("data 5")
  82         cw.start_new_file('thing6.txt')
  83         cw.write("data 6")
  84
  85         cw.start_new_stream('dir2/dir3')
  86         cw.start_new_file('thing7.txt')
  87         cw.write("data 7")
  88
  89         cw.start_new_file('thing8.txt')
  90         cw.write("data 8")
  91
  92         cw.start_new_stream('edgecases')
  93         for f in ":/.../-/*/ ".split("/"):
  94             cw.start_new_file(f)
  95             cw.write('x')
  96
  97         for f in ":/.../-/*/ ".split("/"):
  98             cw.start_new_stream('edgecases/dirs/' + f)
  99             cw.start_new_file('x/x')
 100             cw.write('x')
 101
 102         self.testcollection = cw.finish()
 103         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
 104
 105     def runTest(self):
 106         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
 107
 108         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 109                                       'edgecases', 'dir1', 'dir2'])
 110         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 111         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 112         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 113         self.assertDirContents('edgecases',
 114                                "dirs/:/.../-/*/ ".split("/"))
 115         self.assertDirContents('edgecases/dirs',
 116                                ":/.../-/*/ ".split("/"))
 117
 118         files = {'thing1.txt': 'data 1',
 119                  'thing2.txt': 'data 2',
 120                  'dir1/thing3.txt': 'data 3',
 121                  'dir1/thing4.txt': 'data 4',
 122                  'dir2/thing5.txt': 'data 5',
 123                  'dir2/thing6.txt': 'data 6',
 124                  'dir2/dir3/thing7.txt': 'data 7',
 125                  'dir2/dir3/thing8.txt': 'data 8'}
 126
 127         for k, v in viewitems(files):
 128             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 129                 self.assertEqual(v, f.read().decode())
 130
 131
 132 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 133 class FuseMagicTest(MountTestBase):
 134     def setUp(self, api=None):
 135         super(FuseMagicTest, self).setUp(api=api)
 136
 137         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 138         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
 139         self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
 140         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 141         self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
 142
 143         cw = arvados.CollectionWriter()
 144
 145         cw.start_new_file('thing1.txt')
 146         cw.write("data 1")
 147
 148         self.testcollection = cw.finish()
 149         self.test_manifest = cw.manifest_text()
 150         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 151         self.test_manifest_pdh = coll['portable_data_hash']
 152
 153     def runTest(self):
 154         self.make_mount(fuse.MagicDirectory)
 155
 156         mount_ls = llfuse.listdir(self.mounttmp)
 157         self.assertIn('README', mount_ls)
 158         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 159                              arvados.util.uuid_pattern.match(fn)
 160                              for fn in mount_ls),
 161                          "new FUSE MagicDirectory has no collections or projects")
 162         self.assertDirContents(self.testcollection, ['thing1.txt'])
 163         self.assertDirContents(os.path.join('by_id', self.testcollection),
 164                                ['thing1.txt'])
 165         self.assertIn(self.collection_in_test_project,
 166                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 167         self.assertIn(self.collection_in_test_project,
 168                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 169         self.assertIn(self.collection_in_filter_group,
 170                       llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
 171         self.assertIn(self.collection_in_filter_group,
 172                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
 173
 174
 175         mount_ls = llfuse.listdir(self.mounttmp)
 176         self.assertIn('README', mount_ls)
 177         self.assertIn(self.testcollection, mount_ls)
 178         self.assertIn(self.testcollection,
 179                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 180         self.assertIn(self.test_project, mount_ls)
 181         self.assertIn(self.test_project,
 182                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 183         self.assertIn(self.filter_group,
 184                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 185
 186         with self.assertRaises(OSError):
 187             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 188
 189         files = {}
 190         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 191
 192         for k, v in viewitems(files):
 193             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 194                 self.assertEqual(v, f.read().decode())
 195
 196
 197 class FuseTagsTest(MountTestBase):
 198     def runTest(self):
 199         self.make_mount(fuse.TagsDirectory)
 200
 201         d1 = llfuse.listdir(self.mounttmp)
 202         d1.sort()
 203         self.assertEqual(['foo_tag'], d1)
 204
 205         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 206         d2.sort()
 207         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 208
 209         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 210         d3.sort()
 211         self.assertEqual(['foo'], d3)
 212
 213
 214 class FuseTagsUpdateTest(MountTestBase):
 215     def tag_collection(self, coll_uuid, tag_name):
 216         return self.api.links().create(
 217             body={'link': {'head_uuid': coll_uuid,
 218                            'link_class': 'tag',
 219                            'name': tag_name,
 220         }}).execute()
 221
 222     def runTest(self):
 223         self.make_mount(fuse.TagsDirectory, poll_time=1)
 224
 225         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 226
 227         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 228         self.tag_collection(bar_uuid, 'fuse_test_tag')
 229         for attempt in AssertWithTimeout(10):
 230             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 231         self.assertDirContents('fuse_test_tag', [bar_uuid])
 232
 233         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 234         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 235         for attempt in AssertWithTimeout(10):
 236             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 237
 238         self.api.links().delete(uuid=l['uuid']).execute()
 239         for attempt in AssertWithTimeout(10):
 240             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 241
 242
 243 def fuseSharedTestHelper(mounttmp):
 244     class Test(unittest.TestCase):
 245         def runTest(self):
 246             # Double check that we can open and read objects in this folder as a file,
 247             # and that its contents are what we expect.
 248             baz_path = os.path.join(
 249                 mounttmp,
 250                 'FUSE User',
 251                 'FUSE Test Project',
 252                 'collection in FUSE project',
 253                 'baz')
 254             with open(baz_path) as f:
 255                 self.assertEqual("baz", f.read())
 256
 257             # check mtime on collection
 258             st = os.stat(baz_path)
 259             try:
 260                 mtime = st.st_mtime_ns // 1000000000
 261             except AttributeError:
 262                 mtime = st.st_mtime
 263             self.assertEqual(mtime, 1391448174)
 264
 265             # shared_dirs is a list of the directories exposed
 266             # by fuse.SharedDirectory (i.e. any object visible
 267             # to the current user)
 268             shared_dirs = llfuse.listdir(mounttmp)
 269             shared_dirs.sort()
 270             self.assertIn('FUSE User', shared_dirs)
 271
 272             # fuse_user_objs is a list of the objects owned by the FUSE
 273             # test user (which present as files in the 'FUSE User'
 274             # directory)
 275             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 276             fuse_user_objs.sort()
 277             self.assertEqual(['FUSE Test Project',                    # project owned by user
 278                               'collection #1 owned by FUSE',          # collection owned by user
 279                               'collection #2 owned by FUSE'          # collection owned by user
 280                           ], fuse_user_objs)
 281
 282             # test_proj_files is a list of the files in the FUSE Test Project.
 283             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 284             test_proj_files.sort()
 285             self.assertEqual(['collection in FUSE project'
 286                           ], test_proj_files)
 287
 288
 289     Test().runTest()
 290
 291 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 292 class FuseSharedTest(MountTestBase):
 293     def runTest(self):
 294         self.make_mount(fuse.SharedDirectory,
 295                         exclude=self.api.users().current().execute()['uuid'])
 296         keep = arvados.keep.KeepClient()
 297         keep.put("baz".encode())
 298
 299         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 300
 301
 302 class FuseHomeTest(MountTestBase):
 303     def runTest(self):
 304         self.make_mount(fuse.ProjectDirectory,
 305                         project_object=self.api.users().current().execute())
 306
 307         d1 = llfuse.listdir(self.mounttmp)
 308         self.assertIn('Unrestricted public data', d1)
 309
 310         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 311         public_project = run_test_server.fixture('groups')[
 312             'anonymously_accessible_project']
 313         found_in = 0
 314         found_not_in = 0
 315         for name, item in viewitems(run_test_server.fixture('collections')):
 316             if 'name' not in item:
 317                 pass
 318             elif item['owner_uuid'] == public_project['uuid']:
 319                 self.assertIn(item['name'], d2)
 320                 found_in += 1
 321             else:
 322                 # Artificial assumption here: there is no public
 323                 # collection fixture with the same name as a
 324                 # non-public collection.
 325                 self.assertNotIn(item['name'], d2)
 326                 found_not_in += 1
 327         self.assertNotEqual(0, found_in)
 328         self.assertNotEqual(0, found_not_in)
 329
 330         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 331         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 332
 333
 334 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 335     class Test(unittest.TestCase):
 336         def runTest(self):
 337             d1 = llfuse.listdir(mounttmp)
 338             self.assertEqual(["file1.txt"], d1)
 339             with open(os.path.join(mounttmp, "file1.txt")) as f:
 340                 self.assertEqual("blub", f.read())
 341     Test().runTest()
 342
 343 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 344     class Test(unittest.TestCase):
 345         def runTest(self):
 346             d1 = llfuse.listdir(mounttmp)
 347             self.assertEqual(["file1.txt"], d1)
 348             with open(os.path.join(mounttmp, "file1.txt")) as f:
 349                 self.assertEqual("plnp", f.read())
 350     Test().runTest()
 351
 352 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 353 class FuseModifyFileTest(MountTestBase):
 354     def runTest(self):
 355         collection = arvados.collection.Collection(api_client=self.api)
 356         with collection.open("file1.txt", "w") as f:
 357             f.write("blub")
 358
 359         collection.save_new()
 360
 361         m = self.make_mount(fuse.CollectionDirectory)
 362         with llfuse.lock:
 363             m.new_collection(collection.api_response(), collection)
 364
 365         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 366
 367         with collection.open("file1.txt", "w") as f:
 368             f.write("plnp")
 369
 370         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 371
 372
 373 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 374 class FuseAddFileToCollectionTest(MountTestBase):
 375     def runTest(self):
 376         collection = arvados.collection.Collection(api_client=self.api)
 377         with collection.open("file1.txt", "w") as f:
 378             f.write("blub")
 379
 380         collection.save_new()
 381
 382         m = self.make_mount(fuse.CollectionDirectory)
 383         with llfuse.lock:
 384             m.new_collection(collection.api_response(), collection)
 385
 386         d1 = llfuse.listdir(self.mounttmp)
 387         self.assertEqual(["file1.txt"], d1)
 388
 389         with collection.open("file2.txt", "w") as f:
 390             f.write("plnp")
 391
 392         d1 = llfuse.listdir(self.mounttmp)
 393         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 394
 395
 396 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 397 class FuseRemoveFileFromCollectionTest(MountTestBase):
 398     def runTest(self):
 399         collection = arvados.collection.Collection(api_client=self.api)
 400         with collection.open("file1.txt", "w") as f:
 401             f.write("blub")
 402
 403         with collection.open("file2.txt", "w") as f:
 404             f.write("plnp")
 405
 406         collection.save_new()
 407
 408         m = self.make_mount(fuse.CollectionDirectory)
 409         with llfuse.lock:
 410             m.new_collection(collection.api_response(), collection)
 411
 412         d1 = llfuse.listdir(self.mounttmp)
 413         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 414
 415         collection.remove("file2.txt")
 416
 417         d1 = llfuse.listdir(self.mounttmp)
 418         self.assertEqual(["file1.txt"], d1)
 419
 420
 421 def fuseCreateFileTestHelper(mounttmp):
 422     class Test(unittest.TestCase):
 423         def runTest(self):
 424             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 425                 pass
 426     Test().runTest()
 427
 428 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 429 class FuseCreateFileTest(MountTestBase):
 430     def runTest(self):
 431         collection = arvados.collection.Collection(api_client=self.api)
 432         collection.save_new()
 433
 434         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 435         self.assertEqual(collection2["manifest_text"], "")
 436
 437         collection.save_new()
 438
 439         m = self.make_mount(fuse.CollectionDirectory)
 440         with llfuse.lock:
 441             m.new_collection(collection.api_response(), collection)
 442         self.assertTrue(m.writable())
 443
 444         self.assertNotIn("file1.txt", collection)
 445
 446         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 447
 448         self.assertIn("file1.txt", collection)
 449
 450         d1 = llfuse.listdir(self.mounttmp)
 451         self.assertEqual(["file1.txt"], d1)
 452
 453         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 454         assertRegex(self, collection2["manifest_text"],
 455             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 456
 457
 458 def fuseWriteFileTestHelperWriteFile(mounttmp):
 459     class Test(unittest.TestCase):
 460         def runTest(self):
 461             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 462                 f.write("Hello world!")
 463     Test().runTest()
 464
 465 def fuseWriteFileTestHelperReadFile(mounttmp):
 466     class Test(unittest.TestCase):
 467         def runTest(self):
 468             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 469                 self.assertEqual(f.read(), "Hello world!")
 470     Test().runTest()
 471
 472 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 473 class FuseWriteFileTest(MountTestBase):
 474     def runTest(self):
 475         collection = arvados.collection.Collection(api_client=self.api)
 476         collection.save_new()
 477
 478         m = self.make_mount(fuse.CollectionDirectory)
 479         with llfuse.lock:
 480             m.new_collection(collection.api_response(), collection)
 481         self.assertTrue(m.writable())
 482
 483         self.assertNotIn("file1.txt", collection)
 484
 485         self.assertEqual(0, self.operations.write_counter.get())
 486         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 487         self.assertEqual(12, self.operations.write_counter.get())
 488
 489         with collection.open("file1.txt") as f:
 490             self.assertEqual(f.read(), "Hello world!")
 491
 492         self.assertEqual(0, self.operations.read_counter.get())
 493         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 494         self.assertEqual(12, self.operations.read_counter.get())
 495
 496         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 497         assertRegex(self, collection2["manifest_text"],
 498             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 499
 500
 501 def fuseUpdateFileTestHelper(mounttmp):
 502     class Test(unittest.TestCase):
 503         def runTest(self):
 504             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 505                 f.write("Hello world!")
 506
 507             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 508                 fr = f.read()
 509                 self.assertEqual(fr, "Hello world!")
 510                 f.seek(0)
 511                 f.write("Hola mundo!")
 512                 f.seek(0)
 513                 fr = f.read()
 514                 self.assertEqual(fr, "Hola mundo!!")
 515
 516             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 517                 self.assertEqual(f.read(), "Hola mundo!!")
 518
 519     Test().runTest()
 520
 521 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 522 class FuseUpdateFileTest(MountTestBase):
 523     def runTest(self):
 524         collection = arvados.collection.Collection(api_client=self.api)
 525         collection.save_new()
 526
 527         m = self.make_mount(fuse.CollectionDirectory)
 528         with llfuse.lock:
 529             m.new_collection(collection.api_response(), collection)
 530         self.assertTrue(m.writable())
 531
 532         # See note in MountTestBase.setUp
 533         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 534
 535         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 536         assertRegex(self, collection2["manifest_text"],
 537             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 538
 539
 540 def fuseMkdirTestHelper(mounttmp):
 541     class Test(unittest.TestCase):
 542         def runTest(self):
 543             with self.assertRaises(IOError):
 544                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 545                     f.write("Hello world!")
 546
 547             os.mkdir(os.path.join(mounttmp, "testdir"))
 548
 549             with self.assertRaises(OSError):
 550                 os.mkdir(os.path.join(mounttmp, "testdir"))
 551
 552             d1 = llfuse.listdir(mounttmp)
 553             self.assertEqual(["testdir"], d1)
 554
 555             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 556                 f.write("Hello world!")
 557
 558             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 559             self.assertEqual(["file1.txt"], d1)
 560
 561     Test().runTest()
 562
 563 class FuseMkdirTest(MountTestBase):
 564     def runTest(self):
 565         collection = arvados.collection.Collection(api_client=self.api)
 566         collection.save_new()
 567
 568         m = self.make_mount(fuse.CollectionDirectory)
 569         with llfuse.lock:
 570             m.new_collection(collection.api_response(), collection)
 571         self.assertTrue(m.writable())
 572
 573         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 574
 575         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 576         assertRegex(self, collection2["manifest_text"],
 577             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 578
 579
 580 def fuseRmTestHelperWriteFile(mounttmp):
 581     class Test(unittest.TestCase):
 582         def runTest(self):
 583             os.mkdir(os.path.join(mounttmp, "testdir"))
 584
 585             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 586                 f.write("Hello world!")
 587
 588     Test().runTest()
 589
 590 def fuseRmTestHelperDeleteFile(mounttmp):
 591     class Test(unittest.TestCase):
 592         def runTest(self):
 593             # Can't delete because it's not empty
 594             with self.assertRaises(OSError):
 595                 os.rmdir(os.path.join(mounttmp, "testdir"))
 596
 597             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 598             self.assertEqual(["file1.txt"], d1)
 599
 600             # Delete file
 601             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 602
 603             # Make sure it's empty
 604             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 605             self.assertEqual([], d1)
 606
 607             # Try to delete it again
 608             with self.assertRaises(OSError):
 609                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 610
 611     Test().runTest()
 612
 613 def fuseRmTestHelperRmdir(mounttmp):
 614     class Test(unittest.TestCase):
 615         def runTest(self):
 616             # Should be able to delete now that it is empty
 617             os.rmdir(os.path.join(mounttmp, "testdir"))
 618
 619             # Make sure it's empty
 620             d1 = llfuse.listdir(os.path.join(mounttmp))
 621             self.assertEqual([], d1)
 622
 623             # Try to delete it again
 624             with self.assertRaises(OSError):
 625                 os.rmdir(os.path.join(mounttmp, "testdir"))
 626
 627     Test().runTest()
 628
 629 class FuseRmTest(MountTestBase):
 630     def runTest(self):
 631         collection = arvados.collection.Collection(api_client=self.api)
 632         collection.save_new()
 633
 634         m = self.make_mount(fuse.CollectionDirectory)
 635         with llfuse.lock:
 636             m.new_collection(collection.api_response(), collection)
 637         self.assertTrue(m.writable())
 638
 639         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 640
 641         # Starting manifest
 642         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 643         assertRegex(self, collection2["manifest_text"],
 644             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 645         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 646
 647         # Empty directories are represented by an empty file named "."
 648         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 649         assertRegex(self, collection2["manifest_text"],
 650                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 651
 652         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 653
 654         # manifest should be empty now.
 655         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 656         self.assertEqual(collection2["manifest_text"], "")
 657
 658
 659 def fuseMvFileTestHelperWriteFile(mounttmp):
 660     class Test(unittest.TestCase):
 661         def runTest(self):
 662             os.mkdir(os.path.join(mounttmp, "testdir"))
 663
 664             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 665                 f.write("Hello world!")
 666
 667     Test().runTest()
 668
 669 def fuseMvFileTestHelperMoveFile(mounttmp):
 670     class Test(unittest.TestCase):
 671         def runTest(self):
 672             d1 = llfuse.listdir(os.path.join(mounttmp))
 673             self.assertEqual(["testdir"], d1)
 674             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 675             self.assertEqual(["file1.txt"], d1)
 676
 677             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 678
 679             d1 = llfuse.listdir(os.path.join(mounttmp))
 680             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 681             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 682             self.assertEqual([], d1)
 683
 684     Test().runTest()
 685
 686 class FuseMvFileTest(MountTestBase):
 687     def runTest(self):
 688         collection = arvados.collection.Collection(api_client=self.api)
 689         collection.save_new()
 690
 691         m = self.make_mount(fuse.CollectionDirectory)
 692         with llfuse.lock:
 693             m.new_collection(collection.api_response(), collection)
 694         self.assertTrue(m.writable())
 695
 696         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 697
 698         # Starting manifest
 699         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 700         assertRegex(self, collection2["manifest_text"],
 701             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 702
 703         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 704
 705         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 706         assertRegex(self, collection2["manifest_text"],
 707             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 708
 709
 710 def fuseRenameTestHelper(mounttmp):
 711     class Test(unittest.TestCase):
 712         def runTest(self):
 713             os.mkdir(os.path.join(mounttmp, "testdir"))
 714
 715             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 716                 f.write("Hello world!")
 717
 718     Test().runTest()
 719
 720 class FuseRenameTest(MountTestBase):
 721     def runTest(self):
 722         collection = arvados.collection.Collection(api_client=self.api)
 723         collection.save_new()
 724
 725         m = self.make_mount(fuse.CollectionDirectory)
 726         with llfuse.lock:
 727             m.new_collection(collection.api_response(), collection)
 728         self.assertTrue(m.writable())
 729
 730         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 731
 732         # Starting manifest
 733         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 734         assertRegex(self, collection2["manifest_text"],
 735             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 736
 737         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 738         self.assertEqual(["testdir"], d1)
 739         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 740         self.assertEqual(["file1.txt"], d1)
 741
 742         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 743
 744         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 745         self.assertEqual(["testdir2"], sorted(d1))
 746         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 747         self.assertEqual(["file1.txt"], d1)
 748
 749         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 750         assertRegex(self, collection2["manifest_text"],
 751             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 752
 753
 754 class FuseUpdateFromEventTest(MountTestBase):
 755     def runTest(self):
 756         collection = arvados.collection.Collection(api_client=self.api)
 757         collection.save_new()
 758
 759         m = self.make_mount(fuse.CollectionDirectory)
 760         with llfuse.lock:
 761             m.new_collection(collection.api_response(), collection)
 762
 763         self.operations.listen_for_events()
 764
 765         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 766         self.assertEqual([], sorted(d1))
 767
 768         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 769             with collection2.open("file1.txt", "w") as f:
 770                 f.write("foo")
 771
 772         for attempt in AssertWithTimeout(10):
 773             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 774
 775
 776 class FuseDeleteProjectEventTest(MountTestBase):
 777     def runTest(self):
 778
 779         aproject = self.api.groups().create(body={
 780             "name": "aproject",
 781             "group_class": "project"
 782         }).execute()
 783
 784         bproject = self.api.groups().create(body={
 785             "name": "bproject",
 786             "group_class": "project",
 787             "owner_uuid": aproject["uuid"]
 788         }).execute()
 789
 790         self.make_mount(fuse.ProjectDirectory,
 791                         project_object=self.api.users().current().execute())
 792
 793         self.operations.listen_for_events()
 794
 795         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 796         self.assertEqual(["bproject"], sorted(d1))
 797
 798         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 799
 800         for attempt in AssertWithTimeout(10):
 801             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 802
 803
 804 def fuseFileConflictTestHelper(mounttmp, uuid, keeptmp, settings):
 805     class Test(unittest.TestCase):
 806         def runTest(self):
 807             os.environ['KEEP_LOCAL_STORE'] = keeptmp
 808
 809             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 810                 with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
 811                     with collection2.open("file1.txt", "w") as f2:
 812                         f2.write("foo")
 813                 f.write("bar")
 814
 815             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 816             self.assertEqual(len(d1), 2)
 817
 818             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 819                 self.assertEqual(f.read(), "bar")
 820
 821             assertRegex(self, d1[1],
 822                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 823
 824             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 825                 self.assertEqual(f.read(), "foo")
 826
 827     Test().runTest()
 828
 829 class FuseFileConflictTest(MountTestBase):
 830     def runTest(self):
 831         collection = arvados.collection.Collection(api_client=self.api)
 832         collection.save_new()
 833
 834         m = self.make_mount(fuse.CollectionDirectory)
 835         with llfuse.lock:
 836             m.new_collection(collection.api_response(), collection)
 837
 838         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 839         self.assertEqual([], sorted(d1))
 840
 841         # See note in MountTestBase.setUp
 842         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
 843
 844
 845 def fuseUnlinkOpenFileTest(mounttmp):
 846     class Test(unittest.TestCase):
 847         def runTest(self):
 848             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 849                 f.write("foo")
 850
 851                 d1 = llfuse.listdir(os.path.join(mounttmp))
 852                 self.assertEqual(["file1.txt"], sorted(d1))
 853
 854                 os.remove(os.path.join(mounttmp, "file1.txt"))
 855
 856                 d1 = llfuse.listdir(os.path.join(mounttmp))
 857                 self.assertEqual([], sorted(d1))
 858
 859                 f.seek(0)
 860                 self.assertEqual(f.read(), "foo")
 861                 f.write("bar")
 862
 863                 f.seek(0)
 864                 self.assertEqual(f.read(), "foobar")
 865
 866     Test().runTest()
 867
 868 class FuseUnlinkOpenFileTest(MountTestBase):
 869     def runTest(self):
 870         collection = arvados.collection.Collection(api_client=self.api)
 871         collection.save_new()
 872
 873         m = self.make_mount(fuse.CollectionDirectory)
 874         with llfuse.lock:
 875             m.new_collection(collection.api_response(), collection)
 876
 877         # See note in MountTestBase.setUp
 878         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 879
 880         self.assertEqual(collection.manifest_text(), "")
 881
 882
 883 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 884     class Test(unittest.TestCase):
 885         def runTest(self):
 886             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 887                 f.write("Hello world!")
 888
 889             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 890             self.assertEqual(["file1.txt"], sorted(d1))
 891             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 892             self.assertEqual([], sorted(d1))
 893
 894     Test().runTest()
 895
 896 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 897     class Test(unittest.TestCase):
 898         def runTest(self):
 899             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 900
 901             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 902             self.assertEqual([], sorted(d1))
 903             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 904             self.assertEqual(["file2.txt"], sorted(d1))
 905
 906     Test().runTest()
 907
 908 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 909     def runTest(self):
 910         collection1 = arvados.collection.Collection(api_client=self.api)
 911         collection1.save_new()
 912
 913         collection2 = arvados.collection.Collection(api_client=self.api)
 914         collection2.save_new()
 915
 916         m = self.make_mount(fuse.MagicDirectory)
 917
 918         # See note in MountTestBase.setUp
 919         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 920                                                   collection1.manifest_locator(),
 921                                                   collection2.manifest_locator()))
 922
 923         collection1.update()
 924         collection2.update()
 925
 926         assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 927         self.assertEqual(collection2.manifest_text(), "")
 928
 929         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 930                                                   collection1.manifest_locator(),
 931                                                   collection2.manifest_locator()))
 932
 933         collection1.update()
 934         collection2.update()
 935
 936         self.assertEqual(collection1.manifest_text(), "")
 937         assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 938
 939         collection1.stop_threads()
 940         collection2.stop_threads()
 941
 942
 943 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 944     class Test(unittest.TestCase):
 945         def runTest(self):
 946             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 947             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 948                 f.write("Hello world!")
 949
 950             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 951             self.assertEqual(["testdir"], sorted(d1))
 952             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 953             self.assertEqual(["file1.txt"], sorted(d1))
 954
 955             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 956             self.assertEqual([], sorted(d1))
 957
 958     Test().runTest()
 959
 960
 961 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 962     class Test(unittest.TestCase):
 963         def runTest(self):
 964             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 965
 966             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 967             self.assertEqual([], sorted(d1))
 968
 969             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 970             self.assertEqual(["testdir2"], sorted(d1))
 971             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 972             self.assertEqual(["file1.txt"], sorted(d1))
 973
 974             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 975                 self.assertEqual(f.read(), "Hello world!")
 976
 977     Test().runTest()
 978
 979 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 980     def runTest(self):
 981         collection1 = arvados.collection.Collection(api_client=self.api)
 982         collection1.save_new()
 983
 984         collection2 = arvados.collection.Collection(api_client=self.api)
 985         collection2.save_new()
 986
 987         m = self.make_mount(fuse.MagicDirectory)
 988
 989         # See note in MountTestBase.setUp
 990         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 991                                                   collection1.manifest_locator(),
 992                                                   collection2.manifest_locator()))
 993
 994         collection1.update()
 995         collection2.update()
 996
 997         assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 998         self.assertEqual(collection2.manifest_text(), "")
 999
1000         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
1001                                                   collection1.manifest_locator(),
1002                                                   collection2.manifest_locator()))
1003
1004         collection1.update()
1005         collection2.update()
1006
1007         self.assertEqual(collection1.manifest_text(), "")
1008         assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
1009
1010         collection1.stop_threads()
1011         collection2.stop_threads()
1012
1013 def fuseProjectMkdirTestHelper1(mounttmp):
1014     class Test(unittest.TestCase):
1015         def runTest(self):
1016             os.mkdir(os.path.join(mounttmp, "testcollection"))
1017             with self.assertRaises(OSError):
1018                 os.mkdir(os.path.join(mounttmp, "testcollection"))
1019     Test().runTest()
1020
1021 def fuseProjectMkdirTestHelper2(mounttmp):
1022     class Test(unittest.TestCase):
1023         def runTest(self):
1024             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1025                 f.write("Hello world!")
1026             with self.assertRaises(OSError):
1027                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1028             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1029             with self.assertRaises(OSError):
1030                 os.remove(os.path.join(mounttmp, "testcollection"))
1031             os.rmdir(os.path.join(mounttmp, "testcollection"))
1032     Test().runTest()
1033
1034 class FuseProjectMkdirRmdirTest(MountTestBase):
1035     def runTest(self):
1036         self.make_mount(fuse.ProjectDirectory,
1037                         project_object=self.api.users().current().execute())
1038
1039         d1 = llfuse.listdir(self.mounttmp)
1040         self.assertNotIn('testcollection', d1)
1041
1042         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1043
1044         d1 = llfuse.listdir(self.mounttmp)
1045         self.assertIn('testcollection', d1)
1046
1047         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1048
1049         d1 = llfuse.listdir(self.mounttmp)
1050         self.assertNotIn('testcollection', d1)
1051
1052
1053 def fuseProjectMvTestHelper1(mounttmp):
1054     class Test(unittest.TestCase):
1055         def runTest(self):
1056             d1 = llfuse.listdir(mounttmp)
1057             self.assertNotIn('testcollection', d1)
1058
1059             os.mkdir(os.path.join(mounttmp, "testcollection"))
1060
1061             d1 = llfuse.listdir(mounttmp)
1062             self.assertIn('testcollection', d1)
1063
1064             with self.assertRaises(OSError):
1065                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1066
1067             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1068
1069             d1 = llfuse.listdir(mounttmp)
1070             self.assertNotIn('testcollection', d1)
1071
1072             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1073             self.assertIn('testcollection', d1)
1074
1075     Test().runTest()
1076
1077 class FuseProjectMvTest(MountTestBase):
1078     def runTest(self):
1079         self.make_mount(fuse.ProjectDirectory,
1080                         project_object=self.api.users().current().execute())
1081
1082         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1083
1084
1085 def fuseFsyncTestHelper(mounttmp, k):
1086     class Test(unittest.TestCase):
1087         def runTest(self):
1088             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1089             os.fsync(fd)
1090             os.close(fd)
1091
1092     Test().runTest()
1093
1094 class FuseFsyncTest(FuseMagicTest):
1095     def runTest(self):
1096         self.make_mount(fuse.MagicDirectory)
1097         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1098
1099
1100 class MagicDirApiError(FuseMagicTest):
1101     def setUp(self):
1102         api = mock.MagicMock()
1103         api.keep.block_cache = mock.MagicMock(cache_max=1)
1104         super(MagicDirApiError, self).setUp(api=api)
1105         api.collections().get().execute.side_effect = iter([
1106             Exception('API fail'),
1107             {
1108                 "manifest_text": self.test_manifest,
1109                 "portable_data_hash": self.test_manifest_pdh,
1110             },
1111         ])
1112         api.keep.get.side_effect = Exception('Keep fail')
1113
1114     def runTest(self):
1115         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1116             self.make_mount(fuse.MagicDirectory)
1117
1118             self.operations.inodes.inode_cache.cap = 1
1119             self.operations.inodes.inode_cache.min_entries = 2
1120
1121             with self.assertRaises(OSError):
1122                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1123
1124             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1125
1126
1127 class SanitizeFilenameTest(MountTestBase):
1128     def test_sanitize_filename(self):
1129         pdir = fuse.ProjectDirectory(
1130             1, {}, self.api, 0, False, None,
1131             project_object=self.api.users().current().execute(),
1132         )
1133         acceptable = [
1134             "foo.txt",
1135             ".foo",
1136             "..foo",
1137             "...",
1138             "foo...",
1139             "foo..",
1140             "foo.",
1141             "-",
1142             "\x01\x02\x03",
1143             ]
1144         unacceptable = [
1145             "f\00",
1146             "\00\00",
1147             "/foo",
1148             "foo/",
1149             "//",
1150             ]
1151         for f in acceptable:
1152             self.assertEqual(f, pdir.sanitize_filename(f))
1153         for f in unacceptable:
1154             self.assertNotEqual(f, pdir.sanitize_filename(f))
1155             # The sanitized filename should be the same length, though.
1156             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1157         # Special cases
1158         self.assertEqual("_", pdir.sanitize_filename(""))
1159         self.assertEqual("_", pdir.sanitize_filename("."))
1160         self.assertEqual("__", pdir.sanitize_filename(".."))
1161
1162
1163 class FuseMagicTestPDHOnly(MountTestBase):
1164     def setUp(self, api=None):
1165         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1166
1167         cw = arvados.CollectionWriter()
1168
1169         cw.start_new_file('thing1.txt')
1170         cw.write("data 1")
1171
1172         self.testcollection = cw.finish()
1173         self.test_manifest = cw.manifest_text()
1174         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1175         self.testcollectionuuid = str(created['uuid'])
1176
1177     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1178         if skip_pdh_only is True:
1179             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1180         else:
1181             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1182
1183         mount_ls = llfuse.listdir(self.mounttmp)
1184         self.assertIn('README', mount_ls)
1185         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1186                              arvados.util.uuid_pattern.match(fn)
1187                              for fn in mount_ls),
1188                          "new FUSE MagicDirectory lists Collection")
1189
1190         # look up using pdh should succeed in all cases
1191         self.assertDirContents(self.testcollection, ['thing1.txt'])
1192         self.assertDirContents(os.path.join('by_id', self.testcollection),
1193                                ['thing1.txt'])
1194         mount_ls = llfuse.listdir(self.mounttmp)
1195         self.assertIn('README', mount_ls)
1196         self.assertIn(self.testcollection, mount_ls)
1197         self.assertIn(self.testcollection,
1198                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1199
1200         files = {}
1201         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1202
1203         for k, v in viewitems(files):
1204             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1205                 self.assertEqual(v, f.read().decode())
1206
1207         # look up using uuid should fail when pdh_only is set
1208         if pdh_only is True:
1209             with self.assertRaises(OSError):
1210                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1211                                ['thing1.txt'])
1212         else:
1213             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1214                                ['thing1.txt'])
1215
1216     def test_with_pdh_only_true(self):
1217         self.verify_pdh_only(pdh_only=True)
1218
1219     def test_with_pdh_only_false(self):
1220         self.verify_pdh_only(pdh_only=False)
1221
1222     def test_with_default_by_id(self):
1223         self.verify_pdh_only(skip_pdh_only=True)
1224
1225
1226 class SlashSubstitutionTest(IntegrationTest):
1227     mnt_args = [
1228         '--read-write',
1229         '--mount-home', 'zzz',
1230     ]
1231
1232     def setUp(self):
1233         super(SlashSubstitutionTest, self).setUp()
1234         self.api = arvados.safeapi.ThreadSafeApiCache(
1235             arvados.config.settings(),
1236             version='v1',
1237         )
1238         self.api.config = lambda: {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1239         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1240         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1241         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1242
1243     @IntegrationTest.mount(argv=mnt_args)
1244     @mock.patch('arvados.util.get_config_once')
1245     def test_slash_substitution_before_listing(self, get_config_once):
1246         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1247         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1248         self.checkContents()
1249     @staticmethod
1250     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1251         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1252             f.write('xxx')
1253         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1254             f.write('foo')
1255
1256     @IntegrationTest.mount(argv=mnt_args)
1257     @mock.patch('arvados.util.get_config_once')
1258     def test_slash_substitution_after_listing(self, get_config_once):
1259         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1260         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1261         self.checkContents()
1262     @staticmethod
1263     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1264         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1265             f.write('xxx')
1266         os.listdir(tmpdir)
1267         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1268             f.write('foo')
1269
1270     def checkContents(self):
1271         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1272         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1273
1274     @IntegrationTest.mount(argv=mnt_args)
1275     @mock.patch('arvados.util.get_config_once')
1276     def test_slash_substitution_conflict(self, get_config_once):
1277         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1278         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1279         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1280         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1281         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1282         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1283     @staticmethod
1284     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1285         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1286             f.write('foo')
1287
1288 class StorageClassesTest(IntegrationTest):
1289     mnt_args = [
1290         '--read-write',
1291         '--mount-home', 'homedir',
1292     ]
1293
1294     def setUp(self):
1295         super(StorageClassesTest, self).setUp()
1296         self.api = arvados.safeapi.ThreadSafeApiCache(
1297             arvados.config.settings(),
1298             version='v1',
1299         )
1300
1301     @IntegrationTest.mount(argv=mnt_args)
1302     def test_collection_default_storage_classes(self):
1303         coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
1304         self.api.collections().create(body={'name':'a_collection'}).execute()
1305         self.pool_test(coll_path)
1306     @staticmethod
1307     def _test_collection_default_storage_classes(self, coll):
1308         self.assertEqual(storage_classes_desired(coll), ['default'])
1309
1310     @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
1311     def test_collection_custom_storage_classes(self):
1312         coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
1313         os.mkdir(coll_path)
1314         self.pool_test(coll_path)
1315     @staticmethod
1316     def _test_collection_custom_storage_classes(self, coll):
1317         self.assertEqual(storage_classes_desired(coll), ['foo'])
1318
1319 def _readonlyCollectionTestHelper(mounttmp):
1320     f = open(os.path.join(mounttmp, 'thing1.txt'), 'rt')
1321     # Testing that close() doesn't raise an error.
1322     f.close()
1323
1324 class ReadonlyCollectionTest(MountTestBase):
1325     def setUp(self):
1326         super(ReadonlyCollectionTest, self).setUp()
1327         cw = arvados.collection.Collection()
1328         with cw.open('thing1.txt', 'wt') as f:
1329             f.write("data 1")
1330         cw.save_new(owner_uuid=run_test_server.fixture("groups")["aproject"]["uuid"])
1331         self.testcollection = cw.api_response()
1332
1333     def runTest(self):
1334         settings = arvados.config.settings().copy()
1335         settings["ARVADOS_API_TOKEN"] = run_test_server.fixture("api_client_authorizations")["project_viewer"]["api_token"]
1336         self.api = arvados.safeapi.ThreadSafeApiCache(settings, version='v1')
1337         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection, enable_write=False)
1338
1339         self.pool.apply(_readonlyCollectionTestHelper, (self.mounttmp,))
1340
1341
1342 @parameterized.parameterized_class([
1343     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1344         'project_object': run_test_server.fixture('users')['admin'],
1345     }},
1346     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1347         'project_object': run_test_server.fixture('groups')['public'],
1348     }},
1349 ])
1350 class UnsupportedCreateTest(MountTestBase):
1351     root_class = None
1352     root_kwargs = {}
1353
1354     def setUp(self):
1355         super().setUp()
1356         if 'prefs' in self.root_kwargs.get('project_object', ()):
1357             self.root_kwargs['project_object']['prefs'] = {}
1358         self.make_mount(self.root_class, **self.root_kwargs)
1359         # Make sure the directory knows about its top-level ents.
1360         os.listdir(self.mounttmp)
1361
1362     def test_create(self):
1363         test_path = Path(self.mounttmp, 'test_create')
1364         with self.assertRaises(OSError) as exc_check:
1365             with test_path.open('w'):
1366                 pass
1367         self.assertEqual(exc_check.exception.errno, errno.ENOTSUP)
1368
1369
1370 # FIXME: IMO, for consistency with the "create inside a project" case,
1371 # these operations should also return ENOTSUP instead of EPERM.
1372 # Right now they're returning EPERM because the clasess' writable() method
1373 # usually returns False, and the Operations class transforms that accordingly.
1374 # However, for cases where the mount will never be writable, I think ENOTSUP
1375 # is a clearer error: it lets the user know they can't fix the problem by
1376 # adding permissions in Arvados, etc.
1377 @parameterized.parameterized_class([
1378     {'root_class': fusedir.MagicDirectory,
1379      'preset_dir': 'by_id',
1380      'preset_file': 'README',
1381      },
1382
1383     {'root_class': fusedir.SharedDirectory,
1384      'root_kwargs': {
1385          'exclude': run_test_server.fixture('users')['admin']['uuid'],
1386      },
1387      'preset_dir': 'Active User',
1388      },
1389
1390     {'root_class': fusedir.TagDirectory,
1391      'root_kwargs': {
1392          'tag': run_test_server.fixture('links')['foo_collection_tag']['name'],
1393      },
1394      'preset_dir': run_test_server.fixture('collections')['foo_collection_in_aproject']['uuid'],
1395      },
1396
1397     {'root_class': fusedir.TagsDirectory,
1398      'preset_dir': run_test_server.fixture('links')['foo_collection_tag']['name'],
1399      },
1400 ])
1401 class UnsupportedOperationsTest(UnsupportedCreateTest):
1402     preset_dir = None
1403     preset_file = None
1404
1405     def test_create(self):
1406         test_path = Path(self.mounttmp, 'test_create')
1407         with self.assertRaises(OSError) as exc_check:
1408             with test_path.open('w'):
1409                 pass
1410         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1411
1412     def test_mkdir(self):
1413         test_path = Path(self.mounttmp, 'test_mkdir')
1414         with self.assertRaises(OSError) as exc_check:
1415             test_path.mkdir()
1416         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1417
1418     def test_rename(self):
1419         src_name = self.preset_dir or self.preset_file
1420         if src_name is None:
1421             return
1422         test_src = Path(self.mounttmp, src_name)
1423         test_dst = test_src.with_name('test_dst')
1424         with self.assertRaises(OSError) as exc_check:
1425             test_src.rename(test_dst)
1426         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1427
1428     def test_rmdir(self):
1429         if self.preset_dir is None:
1430             return
1431         test_path = Path(self.mounttmp, self.preset_dir)
1432         with self.assertRaises(OSError) as exc_check:
1433             test_path.rmdir()
1434         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1435
1436     def test_unlink(self):
1437         if self.preset_file is None:
1438             return
1439         test_path = Path(self.mounttmp, self.preset_file)
1440         with self.assertRaises(OSError) as exc_check:
1441             test_path.unlink()
1442         self.assertEqual(exc_check.exception.errno, errno.EPERM)