services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 import errno
   6 import json
   7 import llfuse
   8 import logging
   9 import os
  10 import subprocess
  11 import time
  12 import unittest
  13 import tempfile
  14 import stat
  15
  16 from pathlib import Path
  17 from unittest import mock
  18
  19 import arvados
  20 import arvados_fuse as fuse
  21 import parameterized
  22
  23 from arvados_fuse import fusedir
  24
  25 from . import run_test_server
  26 from .integration_test import IntegrationTest
  27 from .mount_test_base import MountTestBase
  28 from .test_tmp_collection import storage_classes_desired
  29
  30 logger = logging.getLogger('arvados.arv-mount')
  31
  32 class AssertWithTimeout(object):
  33     """Allow some time for an assertion to pass."""
  34
  35     def __init__(self, timeout=0):
  36         self.timeout = timeout
  37
  38     def __iter__(self):
  39         self.deadline = time.time() + self.timeout
  40         self.done = False
  41         return self
  42
  43     def __next__(self):
  44         if self.done:
  45             raise StopIteration
  46         return self.attempt
  47
  48     def attempt(self, fn, *args, **kwargs):
  49         try:
  50             fn(*args, **kwargs)
  51         except AssertionError:
  52             if time.time() > self.deadline:
  53                 raise
  54             time.sleep(0.1)
  55         else:
  56             self.done = True
  57
  58 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
  59 class FuseMountTest(MountTestBase):
  60     def setUp(self):
  61         super(FuseMountTest, self).setUp()
  62
  63         cw = arvados.collection.Collection()
  64         with cw.open('thing1.txt', 'w') as f:
  65             f.write('data 1')
  66         with cw.open('thing2.txt', 'w') as f:
  67             f.write('data 2')
  68
  69         with cw.open('dir1/thing3.txt', 'w') as f:
  70             f.write('data 3')
  71         with cw.open('dir1/thing4.txt', 'w') as f:
  72             f.write('data 4')
  73
  74         with cw.open('dir2/thing5.txt', 'w') as f:
  75             f.write('data 5')
  76         with cw.open('dir2/thing6.txt', 'w') as f:
  77             f.write('data 6')
  78
  79         with cw.open('dir2/dir3/thing7.txt', 'w') as f:
  80             f.write('data 7')
  81         with cw.open('dir2/dir3/thing8.txt', 'w') as f:
  82             f.write('data 8')
  83
  84         for fnm in ":/.../-/*/ ".split("/"):
  85             with cw.open('edgecases/'+fnm, 'w') as f:
  86                 f.write('x')
  87
  88         for fnm in ":/.../-/*/ ".split("/"):
  89             with cw.open('edgecases/dirs/'+fnm+'/x/x', 'w') as f:
  90                 f.write('x')
  91
  92         self.testcollection = cw.portable_data_hash()
  93         self.test_manifest = cw.manifest_text()
  94         self.api.collections().create(body={"manifest_text": self.test_manifest}).execute()
  95
  96     def runTest(self):
  97         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  98
  99         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 100                                       'edgecases', 'dir1', 'dir2'])
 101         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 102         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 103         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 104         self.assertDirContents('edgecases',
 105                                "dirs/:/.../-/*/ ".split("/"))
 106         self.assertDirContents('edgecases/dirs',
 107                                ":/.../-/*/ ".split("/"))
 108
 109         files = {'thing1.txt': 'data 1',
 110                  'thing2.txt': 'data 2',
 111                  'dir1/thing3.txt': 'data 3',
 112                  'dir1/thing4.txt': 'data 4',
 113                  'dir2/thing5.txt': 'data 5',
 114                  'dir2/thing6.txt': 'data 6',
 115                  'dir2/dir3/thing7.txt': 'data 7',
 116                  'dir2/dir3/thing8.txt': 'data 8'}
 117
 118         for k, v in files.items():
 119             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 120                 self.assertEqual(v, f.read().decode())
 121
 122
 123 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 124 class FuseMagicTest(MountTestBase):
 125     def setUp(self, api=None):
 126         super(FuseMagicTest, self).setUp(api=api)
 127
 128         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
 129         self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
 130         self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
 131         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 132         self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
 133
 134         cw = arvados.collection.Collection()
 135         with cw.open('thing1.txt', 'w') as f:
 136             f.write('data 1')
 137
 138         self.testcollection = cw.portable_data_hash()
 139         self.test_manifest = cw.manifest_text()
 140         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 141         self.test_manifest_pdh = coll['portable_data_hash']
 142
 143     def runTest(self):
 144         self.make_mount(fuse.MagicDirectory)
 145
 146         mount_ls = llfuse.listdir(self.mounttmp)
 147         self.assertIn('README', mount_ls)
 148         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 149                              arvados.util.uuid_pattern.match(fn)
 150                              for fn in mount_ls),
 151                          "new FUSE MagicDirectory has no collections or projects")
 152         self.assertDirContents(self.testcollection, ['thing1.txt'])
 153         self.assertDirContents(os.path.join('by_id', self.testcollection),
 154                                ['thing1.txt'])
 155         self.assertIn(self.collection_in_test_project,
 156                       llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
 157         self.assertIn(self.collection_in_test_project,
 158                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
 159         self.assertIn(self.collection_in_filter_group,
 160                       llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
 161         self.assertIn(self.collection_in_filter_group,
 162                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
 163
 164
 165         mount_ls = llfuse.listdir(self.mounttmp)
 166         self.assertIn('README', mount_ls)
 167         self.assertIn(self.testcollection, mount_ls)
 168         self.assertIn(self.testcollection,
 169                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 170         self.assertIn(self.test_project, mount_ls)
 171         self.assertIn(self.test_project,
 172                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 173         self.assertIn(self.filter_group,
 174                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 175
 176         with self.assertRaises(OSError):
 177             llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
 178
 179         files = {}
 180         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 181
 182         for k, v in files.items():
 183             with open(os.path.join(self.mounttmp, k), 'rb') as f:
 184                 self.assertEqual(v, f.read().decode())
 185
 186
 187 class FuseTagsTest(MountTestBase):
 188     def runTest(self):
 189         self.make_mount(fuse.TagsDirectory)
 190
 191         d1 = llfuse.listdir(self.mounttmp)
 192         d1.sort()
 193         self.assertEqual(['foo_tag'], d1)
 194
 195         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 196         d2.sort()
 197         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 198
 199         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 200         d3.sort()
 201         self.assertEqual(['foo'], d3)
 202
 203
 204 class FuseTagsUpdateTest(MountTestBase):
 205     def tag_collection(self, coll_uuid, tag_name):
 206         return self.api.links().create(
 207             body={'link': {'head_uuid': coll_uuid,
 208                            'link_class': 'tag',
 209                            'name': tag_name,
 210         }}).execute()
 211
 212     def runTest(self):
 213         self.make_mount(fuse.TagsDirectory, poll_time=1)
 214
 215         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 216
 217         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 218         self.tag_collection(bar_uuid, 'fuse_test_tag')
 219         for attempt in AssertWithTimeout(10):
 220             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 221         self.assertDirContents('fuse_test_tag', [bar_uuid])
 222
 223         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 224         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 225         for attempt in AssertWithTimeout(10):
 226             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 227
 228         self.api.links().delete(uuid=l['uuid']).execute()
 229         for attempt in AssertWithTimeout(10):
 230             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 231
 232
 233 def fuseSharedTestHelper(mounttmp):
 234     class Test(unittest.TestCase):
 235         def runTest(self):
 236             # Double check that we can open and read objects in this folder as a file,
 237             # and that its contents are what we expect.
 238             baz_path = os.path.join(
 239                 mounttmp,
 240                 'FUSE User',
 241                 'FUSE Test Project',
 242                 'collection in FUSE project',
 243                 'baz')
 244             with open(baz_path) as f:
 245                 self.assertEqual("baz", f.read())
 246
 247             # check mtime on collection
 248             st = os.stat(baz_path)
 249             try:
 250                 mtime = st.st_mtime_ns // 1000000000
 251             except AttributeError:
 252                 mtime = st.st_mtime
 253             self.assertEqual(mtime, 1391448174)
 254
 255             # shared_dirs is a list of the directories exposed
 256             # by fuse.SharedDirectory (i.e. any object visible
 257             # to the current user)
 258             shared_dirs = llfuse.listdir(mounttmp)
 259             shared_dirs.sort()
 260             self.assertIn('FUSE User', shared_dirs)
 261
 262             # fuse_user_objs is a list of the objects owned by the FUSE
 263             # test user (which present as files in the 'FUSE User'
 264             # directory)
 265             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 266             fuse_user_objs.sort()
 267             self.assertEqual(['FUSE Test Project',                    # project owned by user
 268                               'collection #1 owned by FUSE',          # collection owned by user
 269                               'collection #2 owned by FUSE'          # collection owned by user
 270                           ], fuse_user_objs)
 271
 272             # test_proj_files is a list of the files in the FUSE Test Project.
 273             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 274             test_proj_files.sort()
 275             self.assertEqual(['collection in FUSE project'
 276                           ], test_proj_files)
 277
 278
 279     Test().runTest()
 280
 281 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 282 class FuseSharedTest(MountTestBase):
 283     def runTest(self):
 284         self.make_mount(fuse.SharedDirectory,
 285                         exclude=self.api.users().current().execute()['uuid'])
 286         keep = arvados.keep.KeepClient()
 287         keep.put("baz".encode())
 288
 289         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 290
 291
 292 class FuseHomeTest(MountTestBase):
 293     def runTest(self):
 294         self.make_mount(fuse.ProjectDirectory,
 295                         project_object=self.api.users().current().execute())
 296
 297         d1 = llfuse.listdir(self.mounttmp)
 298         self.assertIn('Unrestricted public data', d1)
 299
 300         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 301         public_project = run_test_server.fixture('groups')[
 302             'anonymously_accessible_project']
 303         found_in = 0
 304         found_not_in = 0
 305         for name, item in run_test_server.fixture('collections').items():
 306             if 'name' not in item:
 307                 pass
 308             elif item['owner_uuid'] == public_project['uuid']:
 309                 self.assertIn(item['name'], d2)
 310                 found_in += 1
 311             else:
 312                 # Artificial assumption here: there is no public
 313                 # collection fixture with the same name as a
 314                 # non-public collection.
 315                 self.assertNotIn(item['name'], d2)
 316                 found_not_in += 1
 317         self.assertNotEqual(0, found_in)
 318         self.assertNotEqual(0, found_not_in)
 319
 320         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 321         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 322
 323
 324 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 325     class Test(unittest.TestCase):
 326         def runTest(self):
 327             d1 = llfuse.listdir(mounttmp)
 328             self.assertEqual(["file1.txt"], d1)
 329             with open(os.path.join(mounttmp, "file1.txt")) as f:
 330                 self.assertEqual("blub", f.read())
 331     Test().runTest()
 332
 333 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 334     class Test(unittest.TestCase):
 335         def runTest(self):
 336             d1 = llfuse.listdir(mounttmp)
 337             self.assertEqual(["file1.txt"], d1)
 338             with open(os.path.join(mounttmp, "file1.txt")) as f:
 339                 self.assertEqual("plnp", f.read())
 340     Test().runTest()
 341
 342 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 343 class FuseModifyFileTest(MountTestBase):
 344     def runTest(self):
 345         collection = arvados.collection.Collection(api_client=self.api)
 346         with collection.open("file1.txt", "w") as f:
 347             f.write("blub")
 348
 349         collection.save_new()
 350
 351         m = self.make_mount(fuse.CollectionDirectory)
 352         with llfuse.lock:
 353             m.new_collection(collection.api_response(), collection)
 354
 355         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 356
 357         with collection.open("file1.txt", "w") as f:
 358             f.write("plnp")
 359
 360         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 361
 362
 363 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 364 class FuseAddFileToCollectionTest(MountTestBase):
 365     def runTest(self):
 366         collection = arvados.collection.Collection(api_client=self.api)
 367         with collection.open("file1.txt", "w") as f:
 368             f.write("blub")
 369
 370         collection.save_new()
 371
 372         m = self.make_mount(fuse.CollectionDirectory)
 373         with llfuse.lock:
 374             m.new_collection(collection.api_response(), collection)
 375
 376         d1 = llfuse.listdir(self.mounttmp)
 377         self.assertEqual(["file1.txt"], d1)
 378
 379         with collection.open("file2.txt", "w") as f:
 380             f.write("plnp")
 381
 382         d1 = llfuse.listdir(self.mounttmp)
 383         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 384
 385
 386 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 387 class FuseRemoveFileFromCollectionTest(MountTestBase):
 388     def runTest(self):
 389         collection = arvados.collection.Collection(api_client=self.api)
 390         with collection.open("file1.txt", "w") as f:
 391             f.write("blub")
 392
 393         with collection.open("file2.txt", "w") as f:
 394             f.write("plnp")
 395
 396         collection.save_new()
 397
 398         m = self.make_mount(fuse.CollectionDirectory)
 399         with llfuse.lock:
 400             m.new_collection(collection.api_response(), collection)
 401
 402         d1 = llfuse.listdir(self.mounttmp)
 403         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 404
 405         collection.remove("file2.txt")
 406
 407         d1 = llfuse.listdir(self.mounttmp)
 408         self.assertEqual(["file1.txt"], d1)
 409
 410
 411 def fuseCreateFileTestHelper(mounttmp):
 412     class Test(unittest.TestCase):
 413         def runTest(self):
 414             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 415                 pass
 416     Test().runTest()
 417
 418 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 419 class FuseCreateFileTest(MountTestBase):
 420     def runTest(self):
 421         collection = arvados.collection.Collection(api_client=self.api)
 422         collection.save_new()
 423
 424         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 425         self.assertEqual(collection2["manifest_text"], "")
 426
 427         collection.save_new()
 428
 429         m = self.make_mount(fuse.CollectionDirectory)
 430         with llfuse.lock:
 431             m.new_collection(collection.api_response(), collection)
 432         self.assertTrue(m.writable())
 433
 434         self.assertNotIn("file1.txt", collection)
 435
 436         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 437
 438         self.assertIn("file1.txt", collection)
 439
 440         d1 = llfuse.listdir(self.mounttmp)
 441         self.assertEqual(["file1.txt"], d1)
 442
 443         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 444         self.assertRegex(collection2["manifest_text"],
 445             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 446
 447
 448 def fuseWriteFileTestHelperWriteFile(mounttmp):
 449     class Test(unittest.TestCase):
 450         def runTest(self):
 451             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 452                 f.write("Hello world!")
 453     Test().runTest()
 454
 455 def fuseWriteFileTestHelperReadFile(mounttmp):
 456     class Test(unittest.TestCase):
 457         def runTest(self):
 458             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 459                 self.assertEqual(f.read(), "Hello world!")
 460     Test().runTest()
 461
 462 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 463 class FuseWriteFileTest(MountTestBase):
 464     def runTest(self):
 465         collection = arvados.collection.Collection(api_client=self.api)
 466         collection.save_new()
 467
 468         m = self.make_mount(fuse.CollectionDirectory)
 469         with llfuse.lock:
 470             m.new_collection(collection.api_response(), collection)
 471         self.assertTrue(m.writable())
 472
 473         self.assertNotIn("file1.txt", collection)
 474
 475         self.assertEqual(0, self.operations.write_counter.get())
 476         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 477         self.assertEqual(12, self.operations.write_counter.get())
 478
 479         with collection.open("file1.txt") as f:
 480             self.assertEqual(f.read(), "Hello world!")
 481
 482         self.assertEqual(0, self.operations.read_counter.get())
 483         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 484         self.assertEqual(12, self.operations.read_counter.get())
 485
 486         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 487         self.assertRegex(collection2["manifest_text"],
 488             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 489
 490 def fuseMknodTestHelperReadFile(mounttmp):
 491     class Test(unittest.TestCase):
 492         def runTest(self):
 493             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 494                 self.assertEqual(f.read(), "")
 495     Test().runTest()
 496
 497 class FuseMknodTest(MountTestBase):
 498     def runTest(self):
 499         # Check that os.mknod() can be used to create normal files.
 500         collection = arvados.collection.Collection(api_client=self.api)
 501         collection.save_new()
 502
 503         m = self.make_mount(fuse.CollectionDirectory)
 504         with llfuse.lock:
 505             m.new_collection(collection.api_response(), collection)
 506         self.assertTrue(m.writable())
 507
 508         self.assertNotIn("file1.txt", collection)
 509
 510         self.assertEqual(0, self.operations.write_counter.get())
 511         self.pool.apply(os.mknod, (os.path.join(self.mounttmp, "file1.txt"),))
 512
 513         with collection.open("file1.txt") as f:
 514             self.assertEqual(f.read(), "")
 515
 516         self.pool.apply(fuseMknodTestHelperReadFile, (self.mounttmp,))
 517
 518         # Fail trying to create a FIFO
 519         with self.assertRaises(OSError) as exc_check:
 520             self.pool.apply(os.mknod, (os.path.join(self.mounttmp, "file2.txt"), stat.S_IFIFO))
 521
 522 class FuseMknodReadOnlyTest(MountTestBase):
 523     def runTest(self):
 524         collection = arvados.collection.Collection(api_client=self.api)
 525         collection.save_new()
 526
 527         m = self.make_mount(fuse.CollectionDirectory, enable_write=False)
 528         with llfuse.lock:
 529             m.new_collection(collection.api_response(), collection)
 530         self.assertTrue(m.writable() is False)
 531         with self.assertRaises(OSError) as exc_check:
 532             self.pool.apply(os.mknod, (os.path.join(self.mounttmp, "file1.txt"),))
 533
 534 class FuseMknodProjectTest(MountTestBase):
 535     def runTest(self):
 536         self.make_mount(fuse.ProjectDirectory,
 537                         project_object=self.api.users().current().execute())
 538         with self.assertRaises(OSError) as exc_check:
 539             self.pool.apply(os.mknod, (os.path.join(self.mounttmp, "file1.txt"),))
 540
 541
 542 def fuseUpdateFileTestHelper(mounttmp):
 543     class Test(unittest.TestCase):
 544         def runTest(self):
 545             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 546                 f.write("Hello world!")
 547
 548             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 549                 fr = f.read()
 550                 self.assertEqual(fr, "Hello world!")
 551                 f.seek(0)
 552                 f.write("Hola mundo!")
 553                 f.seek(0)
 554                 fr = f.read()
 555                 self.assertEqual(fr, "Hola mundo!!")
 556
 557             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 558                 self.assertEqual(f.read(), "Hola mundo!!")
 559
 560     Test().runTest()
 561
 562 @parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 563 class FuseUpdateFileTest(MountTestBase):
 564     def runTest(self):
 565         collection = arvados.collection.Collection(api_client=self.api)
 566         collection.save_new()
 567
 568         m = self.make_mount(fuse.CollectionDirectory)
 569         with llfuse.lock:
 570             m.new_collection(collection.api_response(), collection)
 571         self.assertTrue(m.writable())
 572
 573         # See note in MountTestBase.setUp
 574         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 575
 576         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 577         self.assertRegex(collection2["manifest_text"],
 578             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 579
 580
 581 def fuseMkdirTestHelper(mounttmp):
 582     class Test(unittest.TestCase):
 583         def runTest(self):
 584             with self.assertRaises(IOError):
 585                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 586                     f.write("Hello world!")
 587
 588             os.mkdir(os.path.join(mounttmp, "testdir"))
 589
 590             with self.assertRaises(OSError):
 591                 os.mkdir(os.path.join(mounttmp, "testdir"))
 592
 593             d1 = llfuse.listdir(mounttmp)
 594             self.assertEqual(["testdir"], d1)
 595
 596             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 597                 f.write("Hello world!")
 598
 599             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 600             self.assertEqual(["file1.txt"], d1)
 601
 602     Test().runTest()
 603
 604 class FuseMkdirTest(MountTestBase):
 605     def runTest(self):
 606         collection = arvados.collection.Collection(api_client=self.api)
 607         collection.save_new()
 608
 609         m = self.make_mount(fuse.CollectionDirectory)
 610         with llfuse.lock:
 611             m.new_collection(collection.api_response(), collection)
 612         self.assertTrue(m.writable())
 613
 614         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 615
 616         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 617         self.assertRegex(collection2["manifest_text"],
 618             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 619
 620
 621 def fuseRmTestHelperWriteFile(mounttmp):
 622     class Test(unittest.TestCase):
 623         def runTest(self):
 624             os.mkdir(os.path.join(mounttmp, "testdir"))
 625
 626             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 627                 f.write("Hello world!")
 628
 629     Test().runTest()
 630
 631 def fuseRmTestHelperDeleteFile(mounttmp):
 632     class Test(unittest.TestCase):
 633         def runTest(self):
 634             # Can't delete because it's not empty
 635             with self.assertRaises(OSError):
 636                 os.rmdir(os.path.join(mounttmp, "testdir"))
 637
 638             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 639             self.assertEqual(["file1.txt"], d1)
 640
 641             # Delete file
 642             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 643
 644             # Make sure it's empty
 645             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 646             self.assertEqual([], d1)
 647
 648             # Try to delete it again
 649             with self.assertRaises(OSError):
 650                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 651
 652     Test().runTest()
 653
 654 def fuseRmTestHelperRmdir(mounttmp):
 655     class Test(unittest.TestCase):
 656         def runTest(self):
 657             # Should be able to delete now that it is empty
 658             os.rmdir(os.path.join(mounttmp, "testdir"))
 659
 660             # Make sure it's empty
 661             d1 = llfuse.listdir(os.path.join(mounttmp))
 662             self.assertEqual([], d1)
 663
 664             # Try to delete it again
 665             with self.assertRaises(OSError):
 666                 os.rmdir(os.path.join(mounttmp, "testdir"))
 667
 668     Test().runTest()
 669
 670 class FuseRmTest(MountTestBase):
 671     def runTest(self):
 672         collection = arvados.collection.Collection(api_client=self.api)
 673         collection.save_new()
 674
 675         m = self.make_mount(fuse.CollectionDirectory)
 676         with llfuse.lock:
 677             m.new_collection(collection.api_response(), collection)
 678         self.assertTrue(m.writable())
 679
 680         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 681
 682         # Starting manifest
 683         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 684         self.assertRegex(collection2["manifest_text"],
 685             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 686         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 687
 688         # Empty directories are represented by an empty file named "."
 689         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 690         self.assertRegex(collection2["manifest_text"],
 691                                  r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 692
 693         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 694
 695         # manifest should be empty now.
 696         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 697         self.assertEqual(collection2["manifest_text"], "")
 698
 699
 700 def fuseMvFileTestHelperWriteFile(mounttmp):
 701     class Test(unittest.TestCase):
 702         def runTest(self):
 703             os.mkdir(os.path.join(mounttmp, "testdir"))
 704
 705             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 706                 f.write("Hello world!")
 707
 708     Test().runTest()
 709
 710 def fuseMvFileTestHelperMoveFile(mounttmp):
 711     class Test(unittest.TestCase):
 712         def runTest(self):
 713             d1 = llfuse.listdir(os.path.join(mounttmp))
 714             self.assertEqual(["testdir"], d1)
 715             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 716             self.assertEqual(["file1.txt"], d1)
 717
 718             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 719
 720             d1 = llfuse.listdir(os.path.join(mounttmp))
 721             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 722             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 723             self.assertEqual([], d1)
 724
 725     Test().runTest()
 726
 727 class FuseMvFileTest(MountTestBase):
 728     def runTest(self):
 729         collection = arvados.collection.Collection(api_client=self.api)
 730         collection.save_new()
 731
 732         m = self.make_mount(fuse.CollectionDirectory)
 733         with llfuse.lock:
 734             m.new_collection(collection.api_response(), collection)
 735         self.assertTrue(m.writable())
 736
 737         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 738
 739         # Starting manifest
 740         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 741         self.assertRegex(collection2["manifest_text"],
 742             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 743
 744         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 745
 746         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 747         self.assertRegex(collection2["manifest_text"],
 748             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt\n\./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
 749
 750
 751 def fuseRenameTestHelper(mounttmp):
 752     class Test(unittest.TestCase):
 753         def runTest(self):
 754             os.mkdir(os.path.join(mounttmp, "testdir"))
 755
 756             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 757                 f.write("Hello world!")
 758
 759     Test().runTest()
 760
 761 class FuseRenameTest(MountTestBase):
 762     def runTest(self):
 763         collection = arvados.collection.Collection(api_client=self.api)
 764         collection.save_new()
 765
 766         m = self.make_mount(fuse.CollectionDirectory)
 767         with llfuse.lock:
 768             m.new_collection(collection.api_response(), collection)
 769         self.assertTrue(m.writable())
 770
 771         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 772
 773         # Starting manifest
 774         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 775         self.assertRegex(collection2["manifest_text"],
 776             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 777
 778         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 779         self.assertEqual(["testdir"], d1)
 780         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 781         self.assertEqual(["file1.txt"], d1)
 782
 783         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 784
 785         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 786         self.assertEqual(["testdir2"], sorted(d1))
 787         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 788         self.assertEqual(["file1.txt"], d1)
 789
 790         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 791         self.assertRegex(collection2["manifest_text"],
 792             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 793
 794
 795 class FuseUpdateFromEventTest(MountTestBase):
 796     def runTest(self):
 797         collection = arvados.collection.Collection(api_client=self.api)
 798         collection.save_new()
 799
 800         m = self.make_mount(fuse.CollectionDirectory)
 801         with llfuse.lock:
 802             m.new_collection(collection.api_response(), collection)
 803
 804         self.operations.listen_for_events()
 805
 806         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 807         self.assertEqual([], sorted(d1))
 808
 809         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 810             with collection2.open("file1.txt", "w") as f:
 811                 f.write("foo")
 812
 813         for attempt in AssertWithTimeout(10):
 814             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 815
 816
 817 class FuseDeleteProjectEventTest(MountTestBase):
 818     def runTest(self):
 819
 820         aproject = self.api.groups().create(body={
 821             "name": "aproject",
 822             "group_class": "project"
 823         }).execute()
 824
 825         bproject = self.api.groups().create(body={
 826             "name": "bproject",
 827             "group_class": "project",
 828             "owner_uuid": aproject["uuid"]
 829         }).execute()
 830
 831         self.make_mount(fuse.ProjectDirectory,
 832                         project_object=self.api.users().current().execute())
 833
 834         self.operations.listen_for_events()
 835
 836         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 837         self.assertEqual(["bproject"], sorted(d1))
 838
 839         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 840
 841         for attempt in AssertWithTimeout(10):
 842             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 843
 844
 845 def fuseFileConflictTestHelper(mounttmp, uuid, keeptmp, settings):
 846     class Test(unittest.TestCase):
 847         def runTest(self):
 848             os.environ['KEEP_LOCAL_STORE'] = keeptmp
 849
 850             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 851                 with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
 852                     with collection2.open("file1.txt", "w") as f2:
 853                         f2.write("foo")
 854                 f.write("bar")
 855
 856             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 857             self.assertEqual(len(d1), 2)
 858
 859             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 860                 self.assertEqual(f.read(), "bar")
 861
 862             self.assertRegex(d1[1],
 863                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 864
 865             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 866                 self.assertEqual(f.read(), "foo")
 867
 868     Test().runTest()
 869
 870 class FuseFileConflictTest(MountTestBase):
 871     def runTest(self):
 872         collection = arvados.collection.Collection(api_client=self.api)
 873         collection.save_new()
 874
 875         m = self.make_mount(fuse.CollectionDirectory)
 876         with llfuse.lock:
 877             m.new_collection(collection.api_response(), collection)
 878
 879         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 880         self.assertEqual([], sorted(d1))
 881
 882         # See note in MountTestBase.setUp
 883         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
 884
 885
 886 def fuseUnlinkOpenFileTest(mounttmp):
 887     class Test(unittest.TestCase):
 888         def runTest(self):
 889             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 890                 f.write("foo")
 891
 892                 d1 = llfuse.listdir(os.path.join(mounttmp))
 893                 self.assertEqual(["file1.txt"], sorted(d1))
 894
 895                 os.remove(os.path.join(mounttmp, "file1.txt"))
 896
 897                 d1 = llfuse.listdir(os.path.join(mounttmp))
 898                 self.assertEqual([], sorted(d1))
 899
 900                 f.seek(0)
 901                 self.assertEqual(f.read(), "foo")
 902                 f.write("bar")
 903
 904                 f.seek(0)
 905                 self.assertEqual(f.read(), "foobar")
 906
 907     Test().runTest()
 908
 909 class FuseUnlinkOpenFileTest(MountTestBase):
 910     def runTest(self):
 911         collection = arvados.collection.Collection(api_client=self.api)
 912         collection.save_new()
 913
 914         m = self.make_mount(fuse.CollectionDirectory)
 915         with llfuse.lock:
 916             m.new_collection(collection.api_response(), collection)
 917
 918         # See note in MountTestBase.setUp
 919         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 920
 921         self.assertEqual(collection.manifest_text(), "")
 922
 923
 924 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 925     class Test(unittest.TestCase):
 926         def runTest(self):
 927             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 928                 f.write("Hello world!")
 929
 930             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 931             self.assertEqual(["file1.txt"], sorted(d1))
 932             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 933             self.assertEqual([], sorted(d1))
 934
 935     Test().runTest()
 936
 937 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 938     class Test(unittest.TestCase):
 939         def runTest(self):
 940             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 941
 942             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 943             self.assertEqual([], sorted(d1))
 944             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 945             self.assertEqual(["file2.txt"], sorted(d1))
 946
 947     Test().runTest()
 948
 949 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 950     def runTest(self):
 951         collection1 = arvados.collection.Collection(api_client=self.api)
 952         collection1.save_new()
 953
 954         collection2 = arvados.collection.Collection(api_client=self.api)
 955         collection2.save_new()
 956
 957         m = self.make_mount(fuse.MagicDirectory)
 958
 959         # See note in MountTestBase.setUp
 960         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 961                                                   collection1.manifest_locator(),
 962                                                   collection2.manifest_locator()))
 963
 964         collection1.update()
 965         collection2.update()
 966
 967         self.assertRegex(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 968         self.assertEqual(collection2.manifest_text(), "")
 969
 970         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 971                                                   collection1.manifest_locator(),
 972                                                   collection2.manifest_locator()))
 973
 974         collection1.update()
 975         collection2.update()
 976
 977         self.assertEqual(collection1.manifest_text(), "")
 978         self.assertRegex(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 979
 980         collection1.stop_threads()
 981         collection2.stop_threads()
 982
 983
 984 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 985     class Test(unittest.TestCase):
 986         def runTest(self):
 987             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 988             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 989                 f.write("Hello world!")
 990
 991             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 992             self.assertEqual(["testdir"], sorted(d1))
 993             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 994             self.assertEqual(["file1.txt"], sorted(d1))
 995
 996             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 997             self.assertEqual([], sorted(d1))
 998
 999     Test().runTest()
1000
1001
1002 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
1003     class Test(unittest.TestCase):
1004         def runTest(self):
1005             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
1006
1007             d1 = os.listdir(os.path.join(mounttmp, uuid1))
1008             self.assertEqual([], sorted(d1))
1009
1010             d1 = os.listdir(os.path.join(mounttmp, uuid2))
1011             self.assertEqual(["testdir2"], sorted(d1))
1012             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
1013             self.assertEqual(["file1.txt"], sorted(d1))
1014
1015             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
1016                 self.assertEqual(f.read(), "Hello world!")
1017
1018     Test().runTest()
1019
1020 class FuseMvDirBetweenCollectionsTest(MountTestBase):
1021     def runTest(self):
1022         collection1 = arvados.collection.Collection(api_client=self.api)
1023         collection1.save_new()
1024
1025         collection2 = arvados.collection.Collection(api_client=self.api)
1026         collection2.save_new()
1027
1028         m = self.make_mount(fuse.MagicDirectory)
1029
1030         # See note in MountTestBase.setUp
1031         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
1032                                                   collection1.manifest_locator(),
1033                                                   collection2.manifest_locator()))
1034
1035         collection1.update()
1036         collection2.update()
1037
1038         self.assertRegex(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
1039         self.assertEqual(collection2.manifest_text(), "")
1040
1041         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
1042                                                   collection1.manifest_locator(),
1043                                                   collection2.manifest_locator()))
1044
1045         collection1.update()
1046         collection2.update()
1047
1048         self.assertEqual(collection1.manifest_text(), "")
1049         self.assertRegex(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
1050
1051         collection1.stop_threads()
1052         collection2.stop_threads()
1053
1054 def fuseProjectMkdirTestHelper1(mounttmp):
1055     class Test(unittest.TestCase):
1056         def runTest(self):
1057             os.mkdir(os.path.join(mounttmp, "testcollection"))
1058             with self.assertRaises(OSError):
1059                 os.mkdir(os.path.join(mounttmp, "testcollection"))
1060     Test().runTest()
1061
1062 def fuseProjectMkdirTestHelper2(mounttmp):
1063     class Test(unittest.TestCase):
1064         def runTest(self):
1065             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
1066                 f.write("Hello world!")
1067             with self.assertRaises(OSError):
1068                 os.rmdir(os.path.join(mounttmp, "testcollection"))
1069             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
1070             with self.assertRaises(OSError):
1071                 os.remove(os.path.join(mounttmp, "testcollection"))
1072             os.rmdir(os.path.join(mounttmp, "testcollection"))
1073     Test().runTest()
1074
1075 class FuseProjectMkdirRmdirTest(MountTestBase):
1076     def runTest(self):
1077         self.make_mount(fuse.ProjectDirectory,
1078                         project_object=self.api.users().current().execute())
1079
1080         d1 = llfuse.listdir(self.mounttmp)
1081         self.assertNotIn('testcollection', d1)
1082
1083         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1084
1085         d1 = llfuse.listdir(self.mounttmp)
1086         self.assertIn('testcollection', d1)
1087
1088         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1089
1090         d1 = llfuse.listdir(self.mounttmp)
1091         self.assertNotIn('testcollection', d1)
1092
1093
1094 def fuseProjectMvTestHelper1(mounttmp):
1095     class Test(unittest.TestCase):
1096         def runTest(self):
1097             d1 = llfuse.listdir(mounttmp)
1098             self.assertNotIn('testcollection', d1)
1099
1100             os.mkdir(os.path.join(mounttmp, "testcollection"))
1101
1102             d1 = llfuse.listdir(mounttmp)
1103             self.assertIn('testcollection', d1)
1104
1105             with self.assertRaises(OSError):
1106                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1107
1108             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1109
1110             d1 = llfuse.listdir(mounttmp)
1111             self.assertNotIn('testcollection', d1)
1112
1113             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1114             self.assertIn('testcollection', d1)
1115
1116     Test().runTest()
1117
1118 class FuseProjectMvTest(MountTestBase):
1119     def runTest(self):
1120         self.make_mount(fuse.ProjectDirectory,
1121                         project_object=self.api.users().current().execute())
1122
1123         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1124
1125
1126 def fuseFsyncTestHelper(mounttmp, k):
1127     class Test(unittest.TestCase):
1128         def runTest(self):
1129             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1130             os.fsync(fd)
1131             os.close(fd)
1132
1133     Test().runTest()
1134
1135 class FuseFsyncTest(FuseMagicTest):
1136     def runTest(self):
1137         self.make_mount(fuse.MagicDirectory)
1138         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1139
1140
1141 class MagicDirApiError(FuseMagicTest):
1142     def setUp(self):
1143         api = mock.MagicMock()
1144         api.keep.block_cache = mock.MagicMock(cache_max=1)
1145         super(MagicDirApiError, self).setUp(api=api)
1146         api.collections().get().execute.side_effect = iter([
1147             Exception('API fail'),
1148             {
1149                 "manifest_text": self.test_manifest,
1150                 "portable_data_hash": self.test_manifest_pdh,
1151             },
1152         ])
1153         api.keep.get.side_effect = Exception('Keep fail')
1154
1155     def runTest(self):
1156         with mock.patch('arvados_fuse.fresh.FreshBase._poll_time', new_callable=mock.PropertyMock, return_value=60) as mock_poll_time:
1157             self.make_mount(fuse.MagicDirectory)
1158
1159             self.operations.inodes.inode_cache.cap = 1
1160             self.operations.inodes.inode_cache.min_entries = 2
1161
1162             with self.assertRaises(OSError):
1163                 llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1164
1165             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1166
1167
1168 class SanitizeFilenameTest(MountTestBase):
1169     def test_sanitize_filename(self):
1170         pdir = fuse.ProjectDirectory(
1171             1, fuse.Inodes(None), self.api, 0, False, None,
1172             project_object=self.api.users().current().execute(),
1173         )
1174         acceptable = [
1175             "foo.txt",
1176             ".foo",
1177             "..foo",
1178             "...",
1179             "foo...",
1180             "foo..",
1181             "foo.",
1182             "-",
1183             "\x01\x02\x03",
1184             ]
1185         unacceptable = [
1186             "f\00",
1187             "\00\00",
1188             "/foo",
1189             "foo/",
1190             "//",
1191             ]
1192         for f in acceptable:
1193             self.assertEqual(f, pdir.sanitize_filename(f))
1194         for f in unacceptable:
1195             self.assertNotEqual(f, pdir.sanitize_filename(f))
1196             # The sanitized filename should be the same length, though.
1197             self.assertEqual(len(f), len(pdir.sanitize_filename(f)))
1198         # Special cases
1199         self.assertEqual("_", pdir.sanitize_filename(""))
1200         self.assertEqual("_", pdir.sanitize_filename("."))
1201         self.assertEqual("__", pdir.sanitize_filename(".."))
1202
1203
1204 class FuseMagicTestPDHOnly(MountTestBase):
1205     def setUp(self, api=None):
1206         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1207
1208         cw = arvados.collection.Collection()
1209         with cw.open('thing1.txt', 'w') as f:
1210             f.write('data 1')
1211
1212         self.testcollection = cw.portable_data_hash()
1213         self.test_manifest = cw.manifest_text()
1214         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1215         self.testcollectionuuid = str(created['uuid'])
1216
1217     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1218         if skip_pdh_only is True:
1219             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1220         else:
1221             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1222
1223         mount_ls = llfuse.listdir(self.mounttmp)
1224         self.assertIn('README', mount_ls)
1225         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1226                              arvados.util.uuid_pattern.match(fn)
1227                              for fn in mount_ls),
1228                          "new FUSE MagicDirectory lists Collection")
1229
1230         # look up using pdh should succeed in all cases
1231         self.assertDirContents(self.testcollection, ['thing1.txt'])
1232         self.assertDirContents(os.path.join('by_id', self.testcollection),
1233                                ['thing1.txt'])
1234         mount_ls = llfuse.listdir(self.mounttmp)
1235         self.assertIn('README', mount_ls)
1236         self.assertIn(self.testcollection, mount_ls)
1237         self.assertIn(self.testcollection,
1238                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1239
1240         files = {}
1241         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1242
1243         for k, v in files.items():
1244             with open(os.path.join(self.mounttmp, k), 'rb') as f:
1245                 self.assertEqual(v, f.read().decode())
1246
1247         # look up using uuid should fail when pdh_only is set
1248         if pdh_only is True:
1249             with self.assertRaises(OSError):
1250                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1251                                ['thing1.txt'])
1252         else:
1253             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1254                                ['thing1.txt'])
1255
1256     def test_with_pdh_only_true(self):
1257         self.verify_pdh_only(pdh_only=True)
1258
1259     def test_with_pdh_only_false(self):
1260         self.verify_pdh_only(pdh_only=False)
1261
1262     def test_with_default_by_id(self):
1263         self.verify_pdh_only(skip_pdh_only=True)
1264
1265
1266 class SlashSubstitutionTest(IntegrationTest):
1267     mnt_args = [
1268         '--read-write',
1269         '--mount-home', 'zzz',
1270         '--fsns', '[SLASH]'
1271     ]
1272
1273     def setUp(self):
1274         super(SlashSubstitutionTest, self).setUp()
1275
1276         self.api = arvados.safeapi.ThreadSafeApiCache(
1277             arvados.config.settings(),
1278             version='v1'
1279         )
1280         self.testcoll = self.api.collections().create(body={"name": "foo/bar/baz"}).execute()
1281         self.testcolleasy = self.api.collections().create(body={"name": "foo-bar-baz"}).execute()
1282         self.fusename = 'foo[SLASH]bar[SLASH]baz'
1283
1284     @IntegrationTest.mount(argv=mnt_args)
1285     def test_slash_substitution_before_listing(self):
1286         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1287         self.checkContents()
1288     @staticmethod
1289     def _test_slash_substitution_before_listing(self, tmpdir, fusename):
1290         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1291             f.write('xxx')
1292         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1293             f.write('foo')
1294
1295     @IntegrationTest.mount(argv=mnt_args)
1296     @mock.patch('arvados.util.get_config_once')
1297     def test_slash_substitution_after_listing(self, get_config_once):
1298         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1299         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1300         self.checkContents()
1301     @staticmethod
1302     def _test_slash_substitution_after_listing(self, tmpdir, fusename):
1303         with open(os.path.join(tmpdir, 'foo-bar-baz', 'waz'), 'w') as f:
1304             f.write('xxx')
1305         os.listdir(tmpdir)
1306         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1307             f.write('foo')
1308
1309     def checkContents(self):
1310         self.assertRegex(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], r' acbd18db') # md5(foo)
1311         self.assertRegex(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], r' f561aaf6') # md5(xxx)
1312
1313     @IntegrationTest.mount(argv=mnt_args)
1314     @mock.patch('arvados.util.get_config_once')
1315     def test_slash_substitution_conflict(self, get_config_once):
1316         self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
1317         get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
1318         self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
1319         self.assertRegex(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], r' acbd18db') # md5(foo)
1320         # foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
1321         self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
1322     @staticmethod
1323     def _test_slash_substitution_conflict(self, tmpdir, fusename):
1324         with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
1325             f.write('foo')
1326
1327 class StorageClassesTest(IntegrationTest):
1328     mnt_args = [
1329         '--read-write',
1330         '--mount-home', 'homedir',
1331     ]
1332
1333     def setUp(self):
1334         super(StorageClassesTest, self).setUp()
1335         self.api = arvados.safeapi.ThreadSafeApiCache(
1336             arvados.config.settings(),
1337             version='v1',
1338         )
1339
1340     @IntegrationTest.mount(argv=mnt_args)
1341     def test_collection_default_storage_classes(self):
1342         coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
1343         self.api.collections().create(body={'name':'a_collection'}).execute()
1344         self.pool_test(coll_path)
1345     @staticmethod
1346     def _test_collection_default_storage_classes(self, coll):
1347         self.assertEqual(storage_classes_desired(coll), ['default'])
1348
1349     @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
1350     def test_collection_custom_storage_classes(self):
1351         coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
1352         os.mkdir(coll_path)
1353         self.pool_test(coll_path)
1354     @staticmethod
1355     def _test_collection_custom_storage_classes(self, coll):
1356         self.assertEqual(storage_classes_desired(coll), ['foo'])
1357
1358 def _readonlyCollectionTestHelper(mounttmp):
1359     f = open(os.path.join(mounttmp, 'thing1.txt'), 'rt')
1360     # Testing that close() doesn't raise an error.
1361     f.close()
1362
1363 class ReadonlyCollectionTest(MountTestBase):
1364     def setUp(self):
1365         super(ReadonlyCollectionTest, self).setUp()
1366         cw = arvados.collection.Collection()
1367         with cw.open('thing1.txt', 'wt') as f:
1368             f.write("data 1")
1369         cw.save_new(owner_uuid=run_test_server.fixture("groups")["aproject"]["uuid"])
1370         self.testcollection = cw.api_response()
1371
1372     def runTest(self):
1373         settings = arvados.config.settings().copy()
1374         settings["ARVADOS_API_TOKEN"] = run_test_server.fixture("api_client_authorizations")["project_viewer"]["api_token"]
1375         self.api = arvados.safeapi.ThreadSafeApiCache(settings, version='v1')
1376         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection, enable_write=False)
1377
1378         self.pool.apply(_readonlyCollectionTestHelper, (self.mounttmp,))
1379
1380
1381 @parameterized.parameterized_class([
1382     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1383         'project_object': run_test_server.fixture('users')['admin'],
1384     }},
1385     {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
1386         'project_object': run_test_server.fixture('groups')['public'],
1387     }},
1388 ])
1389 class UnsupportedCreateTest(MountTestBase):
1390     root_class = None
1391     root_kwargs = {}
1392
1393     def setUp(self):
1394         super().setUp()
1395         if 'prefs' in self.root_kwargs.get('project_object', ()):
1396             self.root_kwargs['project_object']['prefs'] = {}
1397         self.make_mount(self.root_class, **self.root_kwargs)
1398         # Make sure the directory knows about its top-level ents.
1399         os.listdir(self.mounttmp)
1400
1401     def test_create(self):
1402         test_path = Path(self.mounttmp, 'test_create')
1403         with self.assertRaises(OSError) as exc_check:
1404             with test_path.open('w'):
1405                 pass
1406         self.assertEqual(exc_check.exception.errno, errno.ENOTSUP)
1407
1408
1409 # FIXME: IMO, for consistency with the "create inside a project" case,
1410 # these operations should also return ENOTSUP instead of EPERM.
1411 # Right now they're returning EPERM because the clasess' writable() method
1412 # usually returns False, and the Operations class transforms that accordingly.
1413 # However, for cases where the mount will never be writable, I think ENOTSUP
1414 # is a clearer error: it lets the user know they can't fix the problem by
1415 # adding permissions in Arvados, etc.
1416 @parameterized.parameterized_class([
1417     {'root_class': fusedir.MagicDirectory,
1418      'preset_dir': 'by_id',
1419      'preset_file': 'README',
1420      },
1421
1422     {'root_class': fusedir.SharedDirectory,
1423      'root_kwargs': {
1424          'exclude': run_test_server.fixture('users')['admin']['uuid'],
1425      },
1426      'preset_dir': 'Active User',
1427      },
1428
1429     {'root_class': fusedir.TagDirectory,
1430      'root_kwargs': {
1431          'tag': run_test_server.fixture('links')['foo_collection_tag']['name'],
1432      },
1433      'preset_dir': run_test_server.fixture('collections')['foo_collection_in_aproject']['uuid'],
1434      },
1435
1436     {'root_class': fusedir.TagsDirectory,
1437      'preset_dir': run_test_server.fixture('links')['foo_collection_tag']['name'],
1438      },
1439 ])
1440 class UnsupportedOperationsTest(UnsupportedCreateTest):
1441     preset_dir = None
1442     preset_file = None
1443
1444     def test_create(self):
1445         test_path = Path(self.mounttmp, 'test_create')
1446         with self.assertRaises(OSError) as exc_check:
1447             with test_path.open('w'):
1448                 pass
1449         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1450
1451     def test_mkdir(self):
1452         test_path = Path(self.mounttmp, 'test_mkdir')
1453         with self.assertRaises(OSError) as exc_check:
1454             test_path.mkdir()
1455         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1456
1457     def test_rename(self):
1458         src_name = self.preset_dir or self.preset_file
1459         if src_name is None:
1460             return
1461         test_src = Path(self.mounttmp, src_name)
1462         test_dst = test_src.with_name('test_dst')
1463         with self.assertRaises(OSError) as exc_check:
1464             test_src.rename(test_dst)
1465         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1466
1467     def test_rmdir(self):
1468         if self.preset_dir is None:
1469             return
1470         test_path = Path(self.mounttmp, self.preset_dir)
1471         with self.assertRaises(OSError) as exc_check:
1472             test_path.rmdir()
1473         self.assertEqual(exc_check.exception.errno, errno.EPERM)
1474
1475     def test_unlink(self):
1476         if self.preset_file is None:
1477             return
1478         test_path = Path(self.mounttmp, self.preset_file)
1479         with self.assertRaises(OSError) as exc_check:
1480             test_path.unlink()
1481         self.assertEqual(exc_check.exception.errno, errno.EPERM)