services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 from __future__ import absolute_import
   6 from future.utils import viewitems
   7 from builtins import str
   8 from builtins import object
   9 from pathlib import Path
  10 from six import assertRegex
  11 import errno
  12 import json
  13 import llfuse
  14 import logging
  15 import os
  16 import subprocess
  17 import time
  18 import unittest
  19 import tempfile
  20 import parameterized
  21
  22 from unittest import mock
  23
  24 import arvados
  25 import arvados_fuse as fuse
  26 from arvados_fuse import fusedir
  27 from . import run_test_server
  28
  29 from .integration_test import IntegrationTest
  30 from .mount_test_base import MountTestBase
  31 from .test_tmp_collection import storage_classes_desired
  32
  33 logger = logging.getLogger('arvados.arv-mount')
  34
  35
  36 class AssertWithTimeout(object):
  37     """Allow some time for an assertion to pass."""
  38
  39     def __init__(self, timeout=0):
  40         self.timeout = timeout
  41
  42     def __iter__(self):
  43         self.deadline = time.time() + self.timeout
  44         self.done = False
  45         return self
  46
  47     def __next__(self):
  48         if self.done:
  49             raise StopIteration
  50         return self.attempt
  51
  52     def attempt(self, fn, *args, **kwargs):
  53         try:
  54             fn(*args, **kwargs)
  55         except AssertionError:
  56             if time.time() > self.deadline:
  57                 raise
  58             time.sleep(0.1)
  59         else:
  60             self.done = True
  61
  62 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
  63 class FuseMountTest(MountTestBase):
  64     def setUp(self):
  65         super(FuseMountTest, self).setUp()
  66
  67         cw = arvados.CollectionWriter()
  68
  69         cw.start_new_file('thing1.txt')
  70         cw.write("data 1")
  71         cw.start_new_file('thing2.txt')
  72         cw.write("data 2")
  73
  74         cw.start_new_stream('dir1')
  75         cw.start_new_file('thing3.txt')
  76         cw.write("data 3")
  77         cw.start_new_file('thing4.txt')
  78         cw.write("data 4")
  79
  80         cw.start_new_stream('dir2')
  81         cw.start_new_file('thing5.txt')
  82         cw.write("data 5")
  83         cw.start_new_file('thing6.txt')
  84         cw.write("data 6")
  85
  86         cw.start_new_stream('dir2/dir3')
  87         cw.start_new_file('thing7.txt')
  88         cw.write("data 7")
  89
  90         cw.start_new_file('thing8.txt')
  91         cw.write("data 8")
  92
  93         cw.start_new_stream('edgecases')
  94         for f in ":/.../-/*/ ".split("/"):
  95             cw.start_new_file(f)
  96             cw.write('x')
  97
  98         for f in ":/.../-/*/ ".split("/"):
  99             cw.start_new_stream('edgecases/dirs/' + f)
 100             cw.start_new_file('x/x')
 101             cw.write('x')
 102
 103         self.testcollection = cw.finish()
 104         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
 105
 106     def runTest(self):
 107         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
 108
 109         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 110                                       'edgecases', 'dir1', 'dir2'])
 111         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 112         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 113         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 114         self.assertDirContents('edgecases',
 115                                "dirs/:/.../-/*/ ".split("/"))
 116         self.assertDirContents('edgecases/dirs',
 117                                ":/.../-/*/ ".split("/"))
 118
 119         files = {'thing1.txt': 'data 1',
 120                  'thing2.txt': 'data 2',
 121                  'dir1/thing3.txt': 'data 3',
 122                  'dir1/thing4.txt': 'data 4',
 123                  'dir2/thing5.txt': 'data 5',
 124                  'dir2/thing6.txt': 'data 6',
 125                  'dir2/dir3/thing7.txt': 'data 7',
 126                  'dir2/dir3/thing8.txt': 'data 8'}
 127
 128         for k, v in viewitems(files):
 129             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 130                 self.assertEqual(v, f.read().decode())
 131
 132
 133 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 134 class FuseMagicTest(MountTestBase):
 135     def setUp(self, api=None):
 136         super(FuseMagicTest, self).setUp(api=api)
 137
 138         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 139         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
 140         self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
 141         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 142         self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
 143
 144         cw = arvados.CollectionWriter()
 145
 146         cw.start_new_file('thing1.txt')
 147         cw.write("data 1")
 148
 149         self.testcollection = cw.finish()
 150         self.test_manifest = cw.manifest_text()
 151         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 152         self.test_manifest_pdh = coll['portable_data_hash']
 153
 154     def runTest(self):
 155         self.make_mount(fuse.MagicDirectory)
 156
 157         mount_ls = llfuse.listdir(self.mounttmp)
 158         self.assertIn('README', mount_ls)
 159         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 160                              arvados.util.uuid_pattern.match(fn)
 161                              for fn in mount_ls),
 162                          "new FUSE MagicDirectory has no collections or projects")
 163         self.assertDirContents(self.testcollection, ['thing1.txt'])
 164         self.assertDirContents(os.path.join('by_id', self.testcollection),
 165                                ['thing1.txt'])
 166         self.assertIn(self.collection_in_test_project,
 167                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 168         self.assertIn(self.collection_in_test_project,
 169                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 170         self.assertIn(self.collection_in_filter_group,
 171                       llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
 172         self.assertIn(self.collection_in_filter_group,
 173                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
 174
 175
 176         mount_ls = llfuse.listdir(self.mounttmp)
 177         self.assertIn('README', mount_ls)
 178         self.assertIn(self.testcollection, mount_ls)
 179         self.assertIn(self.testcollection,
 180                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 181         self.assertIn(self.test_project, mount_ls)
 182         self.assertIn(self.test_project,
 183                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 184         self.assertIn(self.filter_group,
 185                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 186
 187         with self.assertRaises(OSError):
 188             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 189
 190         files = {}
 191         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 192
 193         for k, v in viewitems(files):
 194             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 195                 self.assertEqual(v, f.read().decode())
 196
 197
 198 class FuseTagsTest(MountTestBase):
 199     def runTest(self):
 200         self.make_mount(fuse.TagsDirectory)
 201
 202         d1 = llfuse.listdir(self.mounttmp)
 203         d1.sort()
 204         self.assertEqual(['foo_tag'], d1)
 205
 206         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 207         d2.sort()
 208         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 209
 210         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 211         d3.sort()
 212         self.assertEqual(['foo'], d3)
 213
 214
 215 class FuseTagsUpdateTest(MountTestBase):
 216     def tag_collection(self, coll_uuid, tag_name):
 217         return self.api.links().create(
 218             body={'link': {'head_uuid': coll_uuid,
 219                            'link_class': 'tag',
 220                            'name': tag_name,
 221         }}).execute()
 222
 223     def runTest(self):
 224         self.make_mount(fuse.TagsDirectory, poll_time=1)
 225
 226         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 227
 228         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 229         self.tag_collection(bar_uuid, 'fuse_test_tag')
 230         for attempt in AssertWithTimeout(10):
 231             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 232         self.assertDirContents('fuse_test_tag', [bar_uuid])
 233
 234         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 235         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 236         for attempt in AssertWithTimeout(10):
 237             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 238
 239         self.api.links().delete(uuid=l['uuid']).execute()
 240         for attempt in AssertWithTimeout(10):
 241             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 242
 243
 244 def fuseSharedTestHelper(mounttmp):
 245     class Test(unittest.TestCase):
 246         def runTest(self):
 247             # Double check that we can open and read objects in this folder as a file,
 248             # and that its contents are what we expect.
 249             baz_path = os.path.join(
 250                 mounttmp,
 251                 'FUSE User',
 252                 'FUSE Test Project',
 253                 'collection in FUSE project',
 254                 'baz')
 255             with open(baz_path) as f:
 256                 self.assertEqual("baz", f.read())
 257
 258             # check mtime on collection
 259             st = os.stat(baz_path)
 260             try:
 261                 mtime = st.st_mtime_ns // 1000000000
 262             except AttributeError:
 263                 mtime = st.st_mtime
 264             self.assertEqual(mtime, 1391448174)
 265
 266             # shared_dirs is a list of the directories exposed
 267             # by fuse.SharedDirectory (i.e. any object visible
 268             # to the current user)
 269             shared_dirs = llfuse.listdir(mounttmp)
 270             shared_dirs.sort()
 271             self.assertIn('FUSE User', shared_dirs)
 272
 273             # fuse_user_objs is a list of the objects owned by the FUSE
 274             # test user (which present as files in the 'FUSE User'
 275             # directory)
 276             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 277             fuse_user_objs.sort()
 278             self.assertEqual(['FUSE Test Project',                    # project owned by user
 279                               'collection #1 owned by FUSE',          # collection owned by user
 280                               'collection #2 owned by FUSE'          # collection owned by user
 281                           ], fuse_user_objs)
 282
 283             # test_proj_files is a list of the files in the FUSE Test Project.
 284             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 285             test_proj_files.sort()
 286             self.assertEqual(['collection in FUSE project'
 287                           ], test_proj_files)
 288
 289
 290     Test().runTest()
 291
 292 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 293 class FuseSharedTest(MountTestBase):
 294     def runTest(self):
 295         self.make_mount(fuse.SharedDirectory,
 296                         exclude=self.api.users().current().execute()['uuid'])
 297         keep = arvados.keep.KeepClient()
 298         keep.put("baz".encode())
 299
 300         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 301
 302
 303 class FuseHomeTest(MountTestBase):
 304     def runTest(self):
 305         self.make_mount(fuse.ProjectDirectory,
 306                         project_object=self.api.users().current().execute())
 307
 308         d1 = llfuse.listdir(self.mounttmp)
 309         self.assertIn('Unrestricted public data', d1)
 310
 311         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 312         public_project = run_test_server.fixture('groups')[
 313             'anonymously_accessible_project']
 314         found_in = 0
 315         found_not_in = 0
 316         for name, item in viewitems(run_test_server.fixture('collections')):
 317             if 'name' not in item:
 318                 pass
 319             elif item['owner_uuid'] == public_project['uuid']:
 320                 self.assertIn(item['name'], d2)
 321                 found_in += 1
 322             else:
 323                 # Artificial assumption here: there is no public
 324                 # collection fixture with the same name as a
 325                 # non-public collection.
 326                 self.assertNotIn(item['name'], d2)
 327                 found_not_in += 1
 328         self.assertNotEqual(0, found_in)
 329         self.assertNotEqual(0, found_not_in)
 330
 331         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 332         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 333
 334
 335 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 336     class Test(unittest.TestCase):
 337         def runTest(self):
 338             d1 = llfuse.listdir(mounttmp)
 339             self.assertEqual(["file1.txt"], d1)
 340             with open(os.path.join(mounttmp, "file1.txt")) as f:
 341                 self.assertEqual("blub", f.read())
 342     Test().runTest()
 343
 344 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 345     class Test(unittest.TestCase):
 346         def runTest(self):
 347             d1 = llfuse.listdir(mounttmp)
 348             self.assertEqual(["file1.txt"], d1)
 349             with open(os.path.join(mounttmp, "file1.txt")) as f:
 350                 self.assertEqual("plnp", f.read())
 351     Test().runTest()
 352
 353 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 354 class FuseModifyFileTest(MountTestBase):
 355     def runTest(self):
 356         collection = arvados.collection.Collection(api_client=self.api)
 357         with collection.open("file1.txt", "w") as f:
 358             f.write("blub")
 359
 360         collection.save_new()
 361
 362         m = self.make_mount(fuse.CollectionDirectory)
 363         with llfuse.lock:
 364             m.new_collection(collection.api_response(), collection)
 365
 366         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 367
 368         with collection.open("file1.txt", "w") as f:
 369             f.write("plnp")
 370
 371         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 372
 373
 374 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 375 class FuseAddFileToCollectionTest(MountTestBase):
 376     def runTest(self):
 377         collection = arvados.collection.Collection(api_client=self.api)
 378         with collection.open("file1.txt", "w") as f:
 379             f.write("blub")
 380
 381         collection.save_new()
 382
 383         m = self.make_mount(fuse.CollectionDirectory)
 384         with llfuse.lock:
 385             m.new_collection(collection.api_response(), collection)
 386
 387         d1 = llfuse.listdir(self.mounttmp)
 388         self.assertEqual(["file1.txt"], d1)
 389
 390         with collection.open("file2.txt", "w") as f:
 391             f.write("plnp")
 392
 393         d1 = llfuse.listdir(self.mounttmp)
 394         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 395
 396
 397 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 398 class FuseRemoveFileFromCollectionTest(MountTestBase):
 399     def runTest(self):
 400         collection = arvados.collection.Collection(api_client=self.api)
 401         with collection.open("file1.txt", "w") as f:
 402             f.write("blub")
 403
 404         with collection.open("file2.txt", "w") as f:
 405             f.write("plnp")
 406
 407         collection.save_new()
 408
 409         m = self.make_mount(fuse.CollectionDirectory)
 410         with llfuse.lock:
 411             m.new_collection(collection.api_response(), collection)
 412
 413         d1 = llfuse.listdir(self.mounttmp)
 414         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 415
 416         collection.remove("file2.txt")
 417
 418         d1 = llfuse.listdir(self.mounttmp)
 419         self.assertEqual(["file1.txt"], d1)
 420
 421
 422 def fuseCreateFileTestHelper(mounttmp):
 423     class Test(unittest.TestCase):
 424         def runTest(self):
 425             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 426                 pass
 427     Test().runTest()
 428
 429 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 430 class FuseCreateFileTest(MountTestBase):
 431     def runTest(self):
 432         collection = arvados.collection.Collection(api_client=self.api)
 433         collection.save_new()
 434
 435         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 436         self.assertEqual(collection2["manifest_text"], "")
 437
 438         collection.save_new()
 439
 440         m = self.make_mount(fuse.CollectionDirectory)
 441         with llfuse.lock:
 442             m.new_collection(collection.api_response(), collection)
 443         self.assertTrue(m.writable())
 444
 445         self.assertNotIn("file1.txt", collection)
 446
 447         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 448
 449         self.assertIn("file1.txt", collection)
 450
 451         d1 = llfuse.listdir(self.mounttmp)
 452         self.assertEqual(["file1.txt"], d1)
 453
 454         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 455         assertRegex(self, collection2["manifest_text"],
 456             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 457
 458
 459 def fuseWriteFileTestHelperWriteFile(mounttmp):
 460     class Test(unittest.TestCase):
 461         def runTest(self):
 462             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 463                 f.write("Hello world!")
 464     Test().runTest()
 465
 466 def fuseWriteFileTestHelperReadFile(mounttmp):
 467     class Test(unittest.TestCase):
 468         def runTest(self):
 469             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 470                 self.assertEqual(f.read(), "Hello world!")
 471     Test().runTest()
 472
 473 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 474 class FuseWriteFileTest(MountTestBase):
 475     def runTest(self):
 476         collection = arvados.collection.Collection(api_client=self.api)
 477         collection.save_new()
 478
 479         m = self.make_mount(fuse.CollectionDirectory)
 480         with llfuse.lock:
 481             m.new_collection(collection.api_response(), collection)
 482         self.assertTrue(m.writable())
 483
 484         self.assertNotIn("file1.txt", collection)
 485
 486         self.assertEqual(0, self.operations.write_counter.get())
 487         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 488         self.assertEqual(12, self.operations.write_counter.get())
 489
 490         with collection.open("file1.txt") as f:
 491             self.assertEqual(f.read(), "Hello world!")
 492
 493         self.assertEqual(0, self.operations.read_counter.get())
 494         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 495         self.assertEqual(12, self.operations.read_counter.get())
 496
 497         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 498         assertRegex(self, collection2["manifest_text"],
 499             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 500
 501
 502 def fuseUpdateFileTestHelper(mounttmp):
 503     class Test(unittest.TestCase):
 504         def runTest(self):
 505             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 506                 f.write("Hello world!")
 507
 508             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 509                 fr = f.read()
 510                 self.assertEqual(fr, "Hello world!")
 511                 f.seek(0)
 512                 f.write("Hola mundo!")
 513                 f.seek(0)
 514                 fr = f.read()
 515                 self.assertEqual(fr, "Hola mundo!!")
 516
 517             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 518                 self.assertEqual(f.read(), "Hola mundo!!")
 519
 520     Test().runTest()
 521
 522 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 523 class FuseUpdateFileTest(MountTestBase):
 524     def runTest(self):
 525         collection = arvados.collection.Collection(api_client=self.api)
 526         collection.save_new()
 527
 528         m = self.make_mount(fuse.CollectionDirectory)
 529         with llfuse.lock:
 530             m.new_collection(collection.api_response(), collection)
 531         self.assertTrue(m.writable())
 532
 533         # See note in MountTestBase.setUp
 534         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 535
 536         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 537         assertRegex(self, collection2["manifest_text"],
 538             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 539
 540
 541 def fuseMkdirTestHelper(mounttmp):
 542     class Test(unittest.TestCase):
 543         def runTest(self):
 544             with self.assertRaises(IOError):
 545                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 546                     f.write("Hello world!")
 547
 548             os.mkdir(os.path.join(mounttmp, "testdir"))
 549
 550             with self.assertRaises(OSError):
 551                 os.mkdir(os.path.join(mounttmp, "testdir"))
 552
 553             d1 = llfuse.listdir(mounttmp)
 554             self.assertEqual(["testdir"], d1)
 555
 556             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 557                 f.write("Hello world!")
 558
 559             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 560             self.assertEqual(["file1.txt"], d1)
 561
 562     Test().runTest()
 563
 564 class FuseMkdirTest(MountTestBase):
 565     def runTest(self):
 566         collection = arvados.collection.Collection(api_client=self.api)
 567         collection.save_new()
 568
 569         m = self.make_mount(fuse.CollectionDirectory)
 570         with llfuse.lock:
 571             m.new_collection(collection.api_response(), collection)
 572         self.assertTrue(m.writable())
 573
 574         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 575
 576         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 577         assertRegex(self, collection2["manifest_text"],
 578             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 579
 580
 581 def fuseRmTestHelperWriteFile(mounttmp):
 582     class Test(unittest.TestCase):
 583         def runTest(self):
 584             os.mkdir(os.path.join(mounttmp, "testdir"))
 585
 586             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 587                 f.write("Hello world!")
 588
 589     Test().runTest()
 590
 591 def fuseRmTestHelperDeleteFile(mounttmp):
 592     class Test(unittest.TestCase):
 593         def runTest(self):
 594             # Can't delete because it's not empty
 595             with self.assertRaises(OSError):
 596                 os.rmdir(os.path.join(mounttmp, "testdir"))
 597
 598             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 599             self.assertEqual(["file1.txt"], d1)
 600
 601             # Delete file
 602             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 603
 604             # Make sure it's empty
 605             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 606             self.assertEqual([], d1)
 607
 608             # Try to delete it again
 609             with self.assertRaises(OSError):
 610                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 611
 612     Test().runTest()
 613
 614 def fuseRmTestHelperRmdir(mounttmp):
 615     class Test(unittest.TestCase):
 616         def runTest(self):
 617             # Should be able to delete now that it is empty
 618             os.rmdir(os.path.join(mounttmp, "testdir"))
 619
 620             # Make sure it's empty
 621             d1 = llfuse.listdir(os.path.join(mounttmp))
 622             self.assertEqual([], d1)
 623
 624             # Try to delete it again
 625             with self.assertRaises(OSError):
 626                 os.rmdir(os.path.join(mounttmp, "testdir"))
 627
 628     Test().runTest()
 629
 630 class FuseRmTest(MountTestBase):
 631     def runTest(self):
 632         collection = arvados.collection.Collection(api_client=self.api)
 633         collection.save_new()
 634
 635         m = self.make_mount(fuse.CollectionDirectory)
 636         with llfuse.lock:
 637             m.new_collection(collection.api_response(), collection)
 638         self.assertTrue(m.writable())
 639
 640         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 641
 642         # Starting manifest
 643         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 644         assertRegex(self, collection2["manifest_text"],
 645             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 646         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 647
 648         # Empty directories are represented by an empty file named "."
 649         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 650         assertRegex(self, collection2["manifest_text"],
 651                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 652
 653         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 654
 655         # manifest should be empty now.
 656         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 657         self.assertEqual(collection2["manifest_text"], "")
 658
 659
 660 def fuseMvFileTestHelperWriteFile(mounttmp):
 661     class Test(unittest.TestCase):
 662         def runTest(self):
 663             os.mkdir(os.path.join(mounttmp, "testdir"))
 664
 665             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 666                 f.write("Hello world!")
 667
 668     Test().runTest()
 669
 670 def fuseMvFileTestHelperMoveFile(mounttmp):
 671     class Test(unittest.TestCase):
 672         def runTest(self):
 673             d1 = llfuse.listdir(os.path.join(mounttmp))
 674             self.assertEqual(["testdir"], d1)
 675             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 676             self.assertEqual(["file1.txt"], d1)
 677
 678             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 679
 680             d1 = llfuse.listdir(os.path.join(mounttmp))
 681             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 682             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 683             self.assertEqual([], d1)
 684
 685     Test().runTest()
 686
 687 class FuseMvFileTest(MountTestBase):
 688     def runTest(self):
 689         collection = arvados.collection.Collection(api_client=self.api)
 690         collection.save_new()
 691
 692         m = self.make_mount(fuse.CollectionDirectory)
 693         with llfuse.lock:
 694             m.new_collection(collection.api_response(), collection)
 695         self.assertTrue(m.writable())
 696
 697         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 698
 699         # Starting manifest
 700         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 701         assertRegex(self, collection2["manifest_text"],
 702             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 703
 704         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 705
 706         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 707         assertRegex(self, collection2["manifest_text"],
 708             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 709
 710
 711 def fuseRenameTestHelper(mounttmp):
 712     class Test(unittest.TestCase):
 713         def runTest(self):
 714             os.mkdir(os.path.join(mounttmp, "testdir"))
 715
 716             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 717                 f.write("Hello world!")
 718
 719     Test().runTest()
 720
 721 class FuseRenameTest(MountTestBase):
 722     def runTest(self):
 723         collection = arvados.collection.Collection(api_client=self.api)
 724         collection.save_new()
 725
 726         m = self.make_mount(fuse.CollectionDirectory)
 727         with llfuse.lock:
 728             m.new_collection(collection.api_response(), collection)
 729         self.assertTrue(m.writable())
 730
 731         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 732
 733         # Starting manifest
 734         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 735         assertRegex(self, collection2["manifest_text"],
 736             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 737
 738         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 739         self.assertEqual(["testdir"], d1)
 740         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 741         self.assertEqual(["file1.txt"], d1)
 742
 743         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 744
 745         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 746         self.assertEqual(["testdir2"], sorted(d1))
 747         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 748         self.assertEqual(["file1.txt"], d1)
 749
 750         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 751         assertRegex(self, collection2["manifest_text"],
 752             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 753
 754
 755 class FuseUpdateFromEventTest(MountTestBase):
 756     def runTest(self):
 757         collection = arvados.collection.Collection(api_client=self.api)
 758         collection.save_new()
 759
 760         m = self.make_mount(fuse.CollectionDirectory)
 761         with llfuse.lock:
 762             m.new_collection(collection.api_response(), collection)
 763
 764         self.operations.listen_for_events()
 765
 766         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 767         self.assertEqual([], sorted(d1))
 768
 769         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 770             with collection2.open("file1.txt", "w") as f:
 771                 f.write("foo")
 772
 773         for attempt in AssertWithTimeout(10):
 774             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 775
 776
 777 class FuseDeleteProjectEventTest(MountTestBase):
 778     def runTest(self):
 779
 780         aproject = self.api.groups().create(body={
 781             "name": "aproject",
 782             "group_class": "project"
 783         }).execute()
 784
 785         bproject = self.api.groups().create(body={
 786             "name": "bproject",
 787             "group_class": "project",
 788             "owner_uuid": aproject["uuid"]
 789         }).execute()
 790
 791         self.make_mount(fuse.ProjectDirectory,
 792                         project_object=self.api.users().current().execute())
 793
 794         self.operations.listen_for_events()
 795
 796         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 797         self.assertEqual(["bproject"], sorted(d1))
 798
 799         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 800
 801         for attempt in AssertWithTimeout(10):
 802             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 803
 804
 805 def fuseFileConflictTestHelper(mounttmp, uuid, keeptmp, settings):
 806     class Test(unittest.TestCase):
 807         def runTest(self):
 808             os.environ['KEEP_LOCAL_STORE'] = keeptmp
 809
 810             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 811                 with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
 812                     with collection2.open("file1.txt", "w") as f2:
 813                         f2.write("foo")
 814                 f.write("bar")
 815
 816             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 817             self.assertEqual(len(d1), 2)
 818
 819             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 820                 self.assertEqual(f.read(), "bar")
 821
 822             assertRegex(self, d1[1],
 823                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 824
 825             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 826                 self.assertEqual(f.read(), "foo")
 827
 828     Test().runTest()
 829
 830 class FuseFileConflictTest(MountTestBase):
 831     def runTest(self):
 832         collection = arvados.collection.Collection(api_client=self.api)
 833         collection.save_new()
 834
 835         m = self.make_mount(fuse.CollectionDirectory)
 836         with llfuse.lock:
 837             m.new_collection(collection.api_response(), collection)
 838
 839         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 840         self.assertEqual([], sorted(d1))
 841
 842         # See note in MountTestBase.setUp
 843         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
 844
 845
 846 def fuseUnlinkOpenFileTest(mounttmp):
 847     class Test(unittest.TestCase):
 848         def runTest(self):
 849             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 850                 f.write("foo")
 851
 852                 d1 = llfuse.listdir(os.path.join(mounttmp))
 853                 self.assertEqual(["file1.txt"], sorted(d1))
 854
 855                 os.remove(os.path.join(mounttmp, "file1.txt"))
 856
 857                 d1 = llfuse.listdir(os.path.join(mounttmp))
 858                 self.assertEqual([], sorted(d1))
 859
 860                 f.seek(0)
 861                 self.assertEqual(f.read(), "foo")
 862                 f.write("bar")
 863
 864                 f.seek(0)
 865                 self.assertEqual(f.read(), "foobar")
 866
 867     Test().runTest()
 868
 869 class FuseUnlinkOpenFileTest(MountTestBase):
 870     def runTest(self):
 871         collection = arvados.collection.Collection(api_client=self.api)
 872         collection.save_new()
 873
 874         m = self.make_mount(fuse.CollectionDirectory)
 875         with llfuse.lock:
 876             m.new_collection(collection.api_response(), collection)
 877
 878         # See note in MountTestBase.setUp
 879         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 880
 881         self.assertEqual(collection.manifest_text(), "")
 882
 883
 884 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 885     class Test(unittest.TestCase):
 886         def runTest(self):
 887             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 888                 f.write("Hello world!")
 889
 890             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 891             self.assertEqual(["file1.txt"], sorted(d1))
 892             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 893             self.assertEqual([], sorted(d1))
 894
 895     Test().runTest()
 896
 897 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 898     class Test(unittest.TestCase):
 899         def runTest(self):
 900             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 901
 902             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 903             self.assertEqual([], sorted(d1))
 904             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 905             self.assertEqual(["file2.txt"], sorted(d1))
 906
 907     Test().runTest()
 908
 909 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 910     def runTest(self):
 911         collection1 = arvados.collection.Collection(api_client=self.api)
 912         collection1.save_new()
 913
 914         collection2 = arvados.collection.Collection(api_client=self.api)
 915         collection2.save_new()
 916
 917         m = self.make_mount(fuse.MagicDirectory)
 918
 919         # See note in MountTestBase.setUp
 920         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 921                                                   collection1.manifest_locator(),
 922                                                   collection2.manifest_locator()))
 923
 924         collection1.update()
 925         collection2.update()
 926
 927         assertRegex(self, collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 928         self.assertEqual(collection2.manifest_text(), "")
 929
 930         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 931                                                   collection1.manifest_locator(),
 932                                                   collection2.manifest_locator()))
 933
 934         collection1.update()
 935         collection2.update()
 936
 937         self.assertEqual(collection1.manifest_text(), "")
 938         assertRegex(self, collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 939
 940         collection1.stop_threads()
 941         collection2.stop_threads()
 942
 943
 944 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 945     class Test(unittest.TestCase):
 946         def runTest(self):
 947             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 948             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 949                 f.write("Hello world!")
 950
 951             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 952             self.assertEqual(["testdir"], sorted(d1))
 953             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 954             self.assertEqual(["file1.txt"], sorted(d1))
 955
 956             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 957             self.assertEqual([], sorted(d1))
 958
 959     Test().runTest()
 960
 961
 962 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 963     class Test(unittest.TestCase):
 964         def runTest(self):
 965             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 966
 967             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 968             self.assertEqual([], sorted(d1))
 969
 970             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 971             self.assertEqual(["testdir2"], sorted(d1))
 972             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 973             self.assertEqual(["file1.txt"], sorted(d1))
 974
 975             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 976                 self.assertEqual(f.read(), "Hello world!")
 977
 978     Test().runTest()
 979
 980 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 981     def runTest(self):
 982         collection1 = arvados.collection.Collection(api_client=self.api)
 983         collection1.save_new()
 984
 985         collection2 = arvados.collection.Collection(api_client=self.api)
 986         collection2.save_new()
 987
 988         m = self.make_mount(fuse.MagicDirectory)
 989
 990         # See note in MountTestBase.setUp
 991         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 992                                                   collection1.manifest_locator(),
 993                                                   collection2.manifest_locator()))
 994
 995         collection1.update()
 996         collection2.update()
 997
 998         assertRegex(self, collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 999         self.assertEqual(collection2.manifest_text(), "")
1000
1001         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
1002                                                   collection1.manifest_locator(),
1003                                                   collection2.manifest_locator()))
1004
1005         collection1.update()
1006         collection2.update()
1007
1008         self.assertEqual(collection1.manifest_text(), "")
1009         assertRegex(self, collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
1010
1011         collection1.stop_threads()
1012         collection2.stop_threads()
1013
1014 def fuseProjectMkdirTestHelper1(mounttmp):
1015     class Test(unittest.TestCase):
1016         def runTest(self):
1017             os.mkdir(os.path.join(mounttmp, "testcollection"))
1018             with self.assertRaises(OSError):
1019                 os.mkdir(os.path.join(mounttmp, "testcollection"))
1020     Test().runTest()
1021
1022 def fuseProjectMkdirTestHelper2(mounttmp):
1023     class Test(unittest.TestCase):
1024         def runTest(self):
1025             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1026                 f.write("Hello world!")
1027             with self.assertRaises(OSError):
1028                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1029             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1030             with self.assertRaises(OSError):
1031                 os.remove(os.path.join(mounttmp, "testcollection"))
1032             os.rmdir(os.path.join(mounttmp, "testcollection"))
1033     Test().runTest()
1034
1035 class FuseProjectMkdirRmdirTest(MountTestBase):
1036     def runTest(self):
1037         self.make_mount(fuse.ProjectDirectory,
1038                         project_object=self.api.users().current().execute())
1039
1040         d1 = llfuse.listdir(self.mounttmp)
1041         self.assertNotIn('testcollection', d1)
1042
1043         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1044
1045         d1 = llfuse.listdir(self.mounttmp)
1046         self.assertIn('testcollection', d1)
1047
1048         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1049
1050         d1 = llfuse.listdir(self.mounttmp)
1051         self.assertNotIn('testcollection', d1)
1052
1053
1054 def fuseProjectMvTestHelper1(mounttmp):
1055     class Test(unittest.TestCase):
1056         def runTest(self):
1057             d1 = llfuse.listdir(mounttmp)
1058             self.assertNotIn('testcollection', d1)
1059
1060             os.mkdir(os.path.join(mounttmp, "testcollection"))
1061
1062             d1 = llfuse.listdir(mounttmp)
1063             self.assertIn('testcollection', d1)
1064
1065             with self.assertRaises(OSError):
1066                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1067
1068             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1069
1070             d1 = llfuse.listdir(mounttmp)
1071             self.assertNotIn('testcollection', d1)
1072
1073             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1074             self.assertIn('testcollection', d1)
1075
1076     Test().runTest()
1077
1078 class FuseProjectMvTest(MountTestBase):
1079     def runTest(self):
1080         self.make_mount(fuse.ProjectDirectory,
1081                         project_object=self.api.users().current().execute())
1082
1083         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1084
1085
1086 def fuseFsyncTestHelper(mounttmp, k):
1087     class Test(unittest.TestCase):
1088         def runTest(self):
1089             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1090             os.fsync(fd)
1091             os.close(fd)
1092
1093     Test().runTest()
1094
1095 class FuseFsyncTest(FuseMagicTest):
1096     def runTest(self):
1097         self.make_mount(fuse.MagicDirectory)
1098         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1099
1100
1101 class MagicDirApiError(FuseMagicTest):
1102     def setUp(self):
1103         api = mock.MagicMock()
1104         api.keep.block_cache = mock.MagicMock(cache_max=1)
1105         super(MagicDirApiError, self).setUp(api=api)
1106         api.collections().get().execute.side_effect = iter([
1107             Exception('API fail'),
1108             {
1109                 "manifest_text": self.test_manifest,
1110                 "portable_data_hash": self.test_manifest_pdh,
1111             },
1112         ])
1113         api.keep.get.side_effect = Exception('Keep fail')
1114
1115     def runTest(self):
1116         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1117             self.make_mount(fuse.MagicDirectory)
1118
1119             self.operations.inodes.inode_cache.cap = 1
1120             self.operations.inodes.inode_cache.min_entries = 2
1121
1122             with self.assertRaises(OSError):
1123                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1124
1125             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1126
1127
1128 class SanitizeFilenameTest(MountTestBase):
1129     def test_sanitize_filename(self):
1130         pdir = fuse.ProjectDirectory(
1131             1, fuse.Inodes(None), self.api, 0, False, None,
1132             project_object=self.api.users().current().execute(),
1133         )
1134         acceptable = [
1135             "foo.txt",
1136             ".foo",
1137             "..foo",
1138             "...",
1139             "foo...",
1140             "foo..",
1141             "foo.",
1142             "-",
1143             "\x01\x02\x03",
1144             ]
1145         unacceptable = [
1146             "f\00",
1147             "\00\00",
1148             "/foo",
1149             "foo/",
1150             "//",
1151             ]
1152         for f in acceptable:
1153             self.assertEqual(f, pdir.sanitize_filename(f))
1154         for f in unacceptable:
1155             self.assertNotEqual(f, pdir.sanitize_filename(f))
1156             # The sanitized filename should be the same length, though.
1157             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1158         # Special cases
1159         self.assertEqual("_", pdir.sanitize_filename(""))
1160         self.assertEqual("_", pdir.sanitize_filename("."))
1161         self.assertEqual("__", pdir.sanitize_filename(".."))
1162
1163
1164 class FuseMagicTestPDHOnly(MountTestBase):
1165     def setUp(self, api=None):
1166         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1167
1168         cw = arvados.CollectionWriter()
1169
1170         cw.start_new_file('thing1.txt')
1171         cw.write("data 1")
1172
1173         self.testcollection = cw.finish()
1174         self.test_manifest = cw.manifest_text()
1175         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1176         self.testcollectionuuid = str(created['uuid'])
1177
1178     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1179         if skip_pdh_only is True:
1180             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1181         else:
1182             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1183
1184         mount_ls = llfuse.listdir(self.mounttmp)
1185         self.assertIn('README', mount_ls)
1186         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1187                              arvados.util.uuid_pattern.match(fn)
1188                              for fn in mount_ls),
1189                          "new FUSE MagicDirectory lists Collection")
1190
1191         # look up using pdh should succeed in all cases
1192         self.assertDirContents(self.testcollection, ['thing1.txt'])
1193         self.assertDirContents(os.path.join('by_id', self.testcollection),
1194                                ['thing1.txt'])
1195         mount_ls = llfuse.listdir(self.mounttmp)
1196         self.assertIn('README', mount_ls)
1197         self.assertIn(self.testcollection, mount_ls)
1198         self.assertIn(self.testcollection,
1199                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1200
1201         files = {}
1202         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1203
1204         for k, v in viewitems(files):
1205             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1206                 self.assertEqual(v, f.read().decode())
1207
1208         # look up using uuid should fail when pdh_only is set
1209         if pdh_only is True:
1210             with self.assertRaises(OSError):
1211                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1212                                ['thing1.txt'])
1213         else:
1214             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1215                                ['thing1.txt'])
1216
1217     def test_with_pdh_only_true(self):
1218         self.verify_pdh_only(pdh_only=True)
1219
1220     def test_with_pdh_only_false(self):
1221         self.verify_pdh_only(pdh_only=False)
1222
1223     def test_with_default_by_id(self):
1224         self.verify_pdh_only(skip_pdh_only=True)
1225
1226
1227 class SlashSubstitutionTest(IntegrationTest):
1228     mnt_args = [
1229         '--read-write',
1230         '--mount-home', 'zzz',
1231         '--fsns', '[SLASH]'
1232     ]
1233
1234     def setUp(self):
1235         super(SlashSubstitutionTest, self).setUp()
1236
1237         self.api = arvados.safeapi.ThreadSafeApiCache(
1238             arvados.config.settings(),
1239             version='v1'
1240         )
1241         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1242         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1243         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1244
1245     @IntegrationTest.mount(argv=mnt_args)
1246     def test_slash_substitution_before_listing(self):
1247         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1248         self.checkContents()
1249     @staticmethod
1250     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1251         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1252             f.write('xxx')
1253         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1254             f.write('foo')
1255
1256     @IntegrationTest.mount(argv=mnt_args)
1257     @mock.patch('arvados.util.get_config_once')
1258     def test_slash_substitution_after_listing(self, get_config_once):
1259         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1260         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1261         self.checkContents()
1262     @staticmethod
1263     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1264         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1265             f.write('xxx')
1266         os.listdir(tmpdir)
1267         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1268             f.write('foo')
1269
1270     def checkContents(self):
1271         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1272         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
1273
1274     @IntegrationTest.mount(argv=mnt_args)
1275     @mock.patch('arvados.util.get_config_once')
1276     def test_slash_substitution_conflict(self, get_config_once):
1277         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1278         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1279         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1280         self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
1281         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1282         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1283     @staticmethod
1284     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1285         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1286             f.write('foo')
1287
1288 class StorageClassesTest(IntegrationTest):
1289     mnt_args = [
1290         '--read-write',
1291         '--mount-home', 'homedir',
1292     ]
1293
1294     def setUp(self):
1295         super(StorageClassesTest, self).setUp()
1296         self.api = arvados.safeapi.ThreadSafeApiCache(
1297             arvados.config.settings(),
1298             version='v1',
1299         )
1300
1301     @IntegrationTest.mount(argv=mnt_args)
1302     def test_collection_default_storage_classes(self):
1303         coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
1304         self.api.collections().create(body={'name':'a_collection'}).execute()
1305         self.pool_test(coll_path)
1306     @staticmethod
1307     def _test_collection_default_storage_classes(self, coll):
1308         self.assertEqual(storage_classes_desired(coll), ['default'])
1309
1310     @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
1311     def test_collection_custom_storage_classes(self):
1312         coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
1313         os.mkdir(coll_path)
1314         self.pool_test(coll_path)
1315     @staticmethod
1316     def _test_collection_custom_storage_classes(self, coll):
1317         self.assertEqual(storage_classes_desired(coll), ['foo'])
1318
1319 def _readonlyCollectionTestHelper(mounttmp):
1320     f = open(os.path.join(mounttmp, 'thing1.txt'), 'rt')
1321     # Testing that close() doesn't raise an error.
1322     f.close()
1323
1324 class ReadonlyCollectionTest(MountTestBase):
1325     def setUp(self):
1326         super(ReadonlyCollectionTest, self).setUp()
1327         cw = arvados.collection.Collection()
1328         with cw.open('thing1.txt', 'wt') as f:
1329             f.write("data 1")
1330         cw.save_new(owner_uuid=run_test_server.fixture("groups")["aproject"]["uuid"])
1331         self.testcollection = cw.api_response()
1332
1333     def runTest(self):
1334         settings = arvados.config.settings().copy()
1335         settings["ARVADOS_API_TOKEN"] = run_test_server.fixture("api_client_authorizations")["project_viewer"]["api_token"]
1336         self.api = arvados.safeapi.ThreadSafeApiCache(settings, version='v1')
1337         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection, enable_write=False)
1338
1339         self.pool.apply(_readonlyCollectionTestHelper, (self.mounttmp,))
1340
1341
1342 @parameterized.parameterized_class([
1343     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1344         'project_object': run_test_server.fixture('users')['admin'],
1345     }},
1346     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1347         'project_object': run_test_server.fixture('groups')['public'],
1348     }},
1349 ])
1350 class UnsupportedCreateTest(MountTestBase):
1351     root_class = None
1352     root_kwargs = {}
1353
1354     def setUp(self):
1355         super().setUp()
1356         if 'prefs' in self.root_kwargs.get('project_object', ()):
1357             self.root_kwargs['project_object']['prefs'] = {}
1358         self.make_mount(self.root_class, **self.root_kwargs)
1359         # Make sure the directory knows about its top-level ents.
1360         os.listdir(self.mounttmp)
1361
1362     def test_create(self):
1363         test_path = Path(self.mounttmp, 'test_create')
1364         with self.assertRaises(OSError) as exc_check:
1365             with test_path.open('w'):
1366                 pass
1367         self.assertEqual(exc_check.exception.errno, errno.ENOTSUP)
1368
1369
1370 # FIXME: IMO, for consistency with the "create inside a project" case,
1371 # these operations should also return ENOTSUP instead of EPERM.
1372 # Right now they're returning EPERM because the clasess' writable() method
1373 # usually returns False, and the Operations class transforms that accordingly.
1374 # However, for cases where the mount will never be writable, I think ENOTSUP
1375 # is a clearer error: it lets the user know they can't fix the problem by
1376 # adding permissions in Arvados, etc.
1377 @parameterized.parameterized_class([
1378     {'root_class': fusedir.MagicDirectory,
1379      'preset_dir': 'by_id',
1380      'preset_file': 'README',
1381      },
1382
1383     {'root_class': fusedir.SharedDirectory,
1384      'root_kwargs': {
1385          'exclude': run_test_server.fixture('users')['admin']['uuid'],
1386      },
1387      'preset_dir': 'Active User',
1388      },
1389
1390     {'root_class': fusedir.TagDirectory,
1391      'root_kwargs': {
1392          'tag': run_test_server.fixture('links')['foo_collection_tag']['name'],
1393      },
1394      'preset_dir': run_test_server.fixture('collections')['foo_collection_in_aproject']['uuid'],
1395      },
1396
1397     {'root_class': fusedir.TagsDirectory,
1398      'preset_dir': run_test_server.fixture('links')['foo_collection_tag']['name'],
1399      },
1400 ])
1401 class UnsupportedOperationsTest(UnsupportedCreateTest):
1402     preset_dir = None
1403     preset_file = None
1404
1405     def test_create(self):
1406         test_path = Path(self.mounttmp, 'test_create')
1407         with self.assertRaises(OSError) as exc_check:
1408             with test_path.open('w'):
1409                 pass
1410         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1411
1412     def test_mkdir(self):
1413         test_path = Path(self.mounttmp, 'test_mkdir')
1414         with self.assertRaises(OSError) as exc_check:
1415             test_path.mkdir()
1416         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1417
1418     def test_rename(self):
1419         src_name = self.preset_dir or self.preset_file
1420         if src_name is None:
1421             return
1422         test_src = Path(self.mounttmp, src_name)
1423         test_dst = test_src.with_name('test_dst')
1424         with self.assertRaises(OSError) as exc_check:
1425             test_src.rename(test_dst)
1426         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1427
1428     def test_rmdir(self):
1429         if self.preset_dir is None:
1430             return
1431         test_path = Path(self.mounttmp, self.preset_dir)
1432         with self.assertRaises(OSError) as exc_check:
1433             test_path.rmdir()
1434         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1435
1436     def test_unlink(self):
1437         if self.preset_file is None:
1438             return
1439         test_path = Path(self.mounttmp, self.preset_file)
1440         with self.assertRaises(OSError) as exc_check:
1441             test_path.unlink()
1442         self.assertEqual(exc_check.exception.errno, errno.EPERM)