services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 import json
   6 import llfuse
   7 import logging
   8 import mock
   9 import os
  10 import subprocess
  11 import time
  12 import unittest
  13
  14 import arvados
  15 import arvados_fuse as fuse
  16 import run_test_server
  17
  18 from mount_test_base import MountTestBase
  19
  20 logger = logging.getLogger('arvados.arv-mount')
  21
  22
  23 class AssertWithTimeout(object):
  24     """Allow some time for an assertion to pass."""
  25
  26     def __init__(self, timeout=0):
  27         self.timeout = timeout
  28
  29     def __iter__(self):
  30         self.deadline = time.time() + self.timeout
  31         self.done = False
  32         return self
  33
  34     def next(self):
  35         if self.done:
  36             raise StopIteration
  37         return self.attempt
  38
  39     def attempt(self, fn, *args, **kwargs):
  40         try:
  41             fn(*args, **kwargs)
  42         except AssertionError:
  43             if time.time() > self.deadline:
  44                 raise
  45             time.sleep(0.1)
  46         else:
  47             self.done = True
  48
  49
  50 class FuseMountTest(MountTestBase):
  51     def setUp(self):
  52         super(FuseMountTest, self).setUp()
  53
  54         cw = arvados.CollectionWriter()
  55
  56         cw.start_new_file('thing1.txt')
  57         cw.write("data 1")
  58         cw.start_new_file('thing2.txt')
  59         cw.write("data 2")
  60
  61         cw.start_new_stream('dir1')
  62         cw.start_new_file('thing3.txt')
  63         cw.write("data 3")
  64         cw.start_new_file('thing4.txt')
  65         cw.write("data 4")
  66
  67         cw.start_new_stream('dir2')
  68         cw.start_new_file('thing5.txt')
  69         cw.write("data 5")
  70         cw.start_new_file('thing6.txt')
  71         cw.write("data 6")
  72
  73         cw.start_new_stream('dir2/dir3')
  74         cw.start_new_file('thing7.txt')
  75         cw.write("data 7")
  76
  77         cw.start_new_file('thing8.txt')
  78         cw.write("data 8")
  79
  80         cw.start_new_stream('edgecases')
  81         for f in ":/.../-/*/\x01\\/ ".split("/"):
  82             cw.start_new_file(f)
  83             cw.write('x')
  84
  85         for f in ":/.../-/*/\x01\\/ ".split("/"):
  86             cw.start_new_stream('edgecases/dirs/' + f)
  87             cw.start_new_file('x/x')
  88             cw.write('x')
  89
  90         self.testcollection = cw.finish()
  91         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  92
  93     def runTest(self):
  94         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  95
  96         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
  97                                       'edgecases', 'dir1', 'dir2'])
  98         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
  99         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 100         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 101         self.assertDirContents('edgecases',
 102                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
 103         self.assertDirContents('edgecases/dirs',
 104                                ":/.../-/*/\x01\\/ ".split("/"))
 105
 106         files = {'thing1.txt': 'data 1',
 107                  'thing2.txt': 'data 2',
 108                  'dir1/thing3.txt': 'data 3',
 109                  'dir1/thing4.txt': 'data 4',
 110                  'dir2/thing5.txt': 'data 5',
 111                  'dir2/thing6.txt': 'data 6',
 112                  'dir2/dir3/thing7.txt': 'data 7',
 113                  'dir2/dir3/thing8.txt': 'data 8'}
 114
 115         for k, v in files.items():
 116             with open(os.path.join(self.mounttmp, k)) as f:
 117                 self.assertEqual(v, f.read())
 118
 119
 120 class FuseMagicTest(MountTestBase):
 121     def setUp(self, api=None):
 122         super(FuseMagicTest, self).setUp(api=api)
 123
 124         cw = arvados.CollectionWriter()
 125
 126         cw.start_new_file('thing1.txt')
 127         cw.write("data 1")
 128
 129         self.testcollection = cw.finish()
 130         self.test_manifest = cw.manifest_text()
 131         coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 132         self.test_manifest_pdh = coll['portable_data_hash']
 133
 134     def runTest(self):
 135         self.make_mount(fuse.MagicDirectory)
 136
 137         mount_ls = llfuse.listdir(self.mounttmp)
 138         self.assertIn('README', mount_ls)
 139         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 140                              arvados.util.uuid_pattern.match(fn)
 141                              for fn in mount_ls),
 142                          "new FUSE MagicDirectory lists Collection")
 143         self.assertDirContents(self.testcollection, ['thing1.txt'])
 144         self.assertDirContents(os.path.join('by_id', self.testcollection),
 145                                ['thing1.txt'])
 146         mount_ls = llfuse.listdir(self.mounttmp)
 147         self.assertIn('README', mount_ls)
 148         self.assertIn(self.testcollection, mount_ls)
 149         self.assertIn(self.testcollection,
 150                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 151
 152         files = {}
 153         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 154
 155         for k, v in files.items():
 156             with open(os.path.join(self.mounttmp, k)) as f:
 157                 self.assertEqual(v, f.read())
 158
 159
 160 class FuseTagsTest(MountTestBase):
 161     def runTest(self):
 162         self.make_mount(fuse.TagsDirectory)
 163
 164         d1 = llfuse.listdir(self.mounttmp)
 165         d1.sort()
 166         self.assertEqual(['foo_tag'], d1)
 167
 168         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 169         d2.sort()
 170         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 171
 172         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 173         d3.sort()
 174         self.assertEqual(['foo'], d3)
 175
 176
 177 class FuseTagsUpdateTest(MountTestBase):
 178     def tag_collection(self, coll_uuid, tag_name):
 179         return self.api.links().create(
 180             body={'link': {'head_uuid': coll_uuid,
 181                            'link_class': 'tag',
 182                            'name': tag_name,
 183         }}).execute()
 184
 185     def runTest(self):
 186         self.make_mount(fuse.TagsDirectory, poll_time=1)
 187
 188         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 189
 190         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 191         self.tag_collection(bar_uuid, 'fuse_test_tag')
 192         for attempt in AssertWithTimeout(10):
 193             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 194         self.assertDirContents('fuse_test_tag', [bar_uuid])
 195
 196         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 197         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 198         for attempt in AssertWithTimeout(10):
 199             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 200
 201         self.api.links().delete(uuid=l['uuid']).execute()
 202         for attempt in AssertWithTimeout(10):
 203             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 204
 205
 206 def fuseSharedTestHelper(mounttmp):
 207     class Test(unittest.TestCase):
 208         def runTest(self):
 209             # Double check that we can open and read objects in this folder as a file,
 210             # and that its contents are what we expect.
 211             baz_path = os.path.join(
 212                 mounttmp,
 213                 'FUSE User',
 214                 'FUSE Test Project',
 215                 'collection in FUSE project',
 216                 'baz')
 217             with open(baz_path) as f:
 218                 self.assertEqual("baz", f.read())
 219
 220             # check mtime on collection
 221             st = os.stat(baz_path)
 222             try:
 223                 mtime = st.st_mtime_ns / 1000000000
 224             except AttributeError:
 225                 mtime = st.st_mtime
 226             self.assertEqual(mtime, 1391448174)
 227
 228             # shared_dirs is a list of the directories exposed
 229             # by fuse.SharedDirectory (i.e. any object visible
 230             # to the current user)
 231             shared_dirs = llfuse.listdir(mounttmp)
 232             shared_dirs.sort()
 233             self.assertIn('FUSE User', shared_dirs)
 234
 235             # fuse_user_objs is a list of the objects owned by the FUSE
 236             # test user (which present as files in the 'FUSE User'
 237             # directory)
 238             fuse_user_objs = llfuse.listdir(os.path.join(mounttmp, 'FUSE User'))
 239             fuse_user_objs.sort()
 240             self.assertEqual(['FUSE Test Project',                    # project owned by user
 241                               'collection #1 owned by FUSE',          # collection owned by user
 242                               'collection #2 owned by FUSE'          # collection owned by user
 243                           ], fuse_user_objs)
 244
 245             # test_proj_files is a list of the files in the FUSE Test Project.
 246             test_proj_files = llfuse.listdir(os.path.join(mounttmp, 'FUSE User', 'FUSE Test Project'))
 247             test_proj_files.sort()
 248             self.assertEqual(['collection in FUSE project'
 249                           ], test_proj_files)
 250
 251
 252     Test().runTest()
 253
 254 class FuseSharedTest(MountTestBase):
 255     def runTest(self):
 256         self.make_mount(fuse.SharedDirectory,
 257                         exclude=self.api.users().current().execute()['uuid'])
 258         keep = arvados.keep.KeepClient()
 259         keep.put("baz")
 260
 261         self.pool.apply(fuseSharedTestHelper, (self.mounttmp,))
 262
 263
 264 class FuseHomeTest(MountTestBase):
 265     def runTest(self):
 266         self.make_mount(fuse.ProjectDirectory,
 267                         project_object=self.api.users().current().execute())
 268
 269         d1 = llfuse.listdir(self.mounttmp)
 270         self.assertIn('Unrestricted public data', d1)
 271
 272         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 273         public_project = run_test_server.fixture('groups')[
 274             'anonymously_accessible_project']
 275         found_in = 0
 276         found_not_in = 0
 277         for name, item in run_test_server.fixture('collections').iteritems():
 278             if 'name' not in item:
 279                 pass
 280             elif item['owner_uuid'] == public_project['uuid']:
 281                 self.assertIn(item['name'], d2)
 282                 found_in += 1
 283             else:
 284                 # Artificial assumption here: there is no public
 285                 # collection fixture with the same name as a
 286                 # non-public collection.
 287                 self.assertNotIn(item['name'], d2)
 288                 found_not_in += 1
 289         self.assertNotEqual(0, found_in)
 290         self.assertNotEqual(0, found_not_in)
 291
 292         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 293         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 294
 295
 296 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 297     class Test(unittest.TestCase):
 298         def runTest(self):
 299             d1 = llfuse.listdir(mounttmp)
 300             self.assertEqual(["file1.txt"], d1)
 301             with open(os.path.join(mounttmp, "file1.txt")) as f:
 302                 self.assertEqual("blub", f.read())
 303     Test().runTest()
 304
 305 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 306     class Test(unittest.TestCase):
 307         def runTest(self):
 308             d1 = llfuse.listdir(mounttmp)
 309             self.assertEqual(["file1.txt"], d1)
 310             with open(os.path.join(mounttmp, "file1.txt")) as f:
 311                 self.assertEqual("plnp", f.read())
 312     Test().runTest()
 313
 314 class FuseModifyFileTest(MountTestBase):
 315     def runTest(self):
 316         collection = arvados.collection.Collection(api_client=self.api)
 317         with collection.open("file1.txt", "w") as f:
 318             f.write("blub")
 319
 320         collection.save_new()
 321
 322         m = self.make_mount(fuse.CollectionDirectory)
 323         with llfuse.lock:
 324             m.new_collection(collection.api_response(), collection)
 325
 326         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 327
 328         with collection.open("file1.txt", "w") as f:
 329             f.write("plnp")
 330
 331         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 332
 333
 334 class FuseAddFileToCollectionTest(MountTestBase):
 335     def runTest(self):
 336         collection = arvados.collection.Collection(api_client=self.api)
 337         with collection.open("file1.txt", "w") as f:
 338             f.write("blub")
 339
 340         collection.save_new()
 341
 342         m = self.make_mount(fuse.CollectionDirectory)
 343         with llfuse.lock:
 344             m.new_collection(collection.api_response(), collection)
 345
 346         d1 = llfuse.listdir(self.mounttmp)
 347         self.assertEqual(["file1.txt"], d1)
 348
 349         with collection.open("file2.txt", "w") as f:
 350             f.write("plnp")
 351
 352         d1 = llfuse.listdir(self.mounttmp)
 353         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 354
 355
 356 class FuseRemoveFileFromCollectionTest(MountTestBase):
 357     def runTest(self):
 358         collection = arvados.collection.Collection(api_client=self.api)
 359         with collection.open("file1.txt", "w") as f:
 360             f.write("blub")
 361
 362         with collection.open("file2.txt", "w") as f:
 363             f.write("plnp")
 364
 365         collection.save_new()
 366
 367         m = self.make_mount(fuse.CollectionDirectory)
 368         with llfuse.lock:
 369             m.new_collection(collection.api_response(), collection)
 370
 371         d1 = llfuse.listdir(self.mounttmp)
 372         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 373
 374         collection.remove("file2.txt")
 375
 376         d1 = llfuse.listdir(self.mounttmp)
 377         self.assertEqual(["file1.txt"], d1)
 378
 379
 380 def fuseCreateFileTestHelper(mounttmp):
 381     class Test(unittest.TestCase):
 382         def runTest(self):
 383             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 384                 pass
 385     Test().runTest()
 386
 387 class FuseCreateFileTest(MountTestBase):
 388     def runTest(self):
 389         collection = arvados.collection.Collection(api_client=self.api)
 390         collection.save_new()
 391
 392         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 393         self.assertEqual(collection2["manifest_text"], "")
 394
 395         collection.save_new()
 396
 397         m = self.make_mount(fuse.CollectionDirectory)
 398         with llfuse.lock:
 399             m.new_collection(collection.api_response(), collection)
 400         self.assertTrue(m.writable())
 401
 402         self.assertNotIn("file1.txt", collection)
 403
 404         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 405
 406         self.assertIn("file1.txt", collection)
 407
 408         d1 = llfuse.listdir(self.mounttmp)
 409         self.assertEqual(["file1.txt"], d1)
 410
 411         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 412         self.assertRegexpMatches(collection2["manifest_text"],
 413             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 414
 415
 416 def fuseWriteFileTestHelperWriteFile(mounttmp):
 417     class Test(unittest.TestCase):
 418         def runTest(self):
 419             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 420                 f.write("Hello world!")
 421     Test().runTest()
 422
 423 def fuseWriteFileTestHelperReadFile(mounttmp):
 424     class Test(unittest.TestCase):
 425         def runTest(self):
 426             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 427                 self.assertEqual(f.read(), "Hello world!")
 428     Test().runTest()
 429
 430 class FuseWriteFileTest(MountTestBase):
 431     def runTest(self):
 432         collection = arvados.collection.Collection(api_client=self.api)
 433         collection.save_new()
 434
 435         m = self.make_mount(fuse.CollectionDirectory)
 436         with llfuse.lock:
 437             m.new_collection(collection.api_response(), collection)
 438         self.assertTrue(m.writable())
 439
 440         self.assertNotIn("file1.txt", collection)
 441
 442         self.assertEqual(0, self.operations.write_counter.get())
 443         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 444         self.assertEqual(12, self.operations.write_counter.get())
 445
 446         with collection.open("file1.txt") as f:
 447             self.assertEqual(f.read(), "Hello world!")
 448
 449         self.assertEqual(0, self.operations.read_counter.get())
 450         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 451         self.assertEqual(12, self.operations.read_counter.get())
 452
 453         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 454         self.assertRegexpMatches(collection2["manifest_text"],
 455             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 456
 457
 458 def fuseUpdateFileTestHelper(mounttmp):
 459     class Test(unittest.TestCase):
 460         def runTest(self):
 461             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 462                 f.write("Hello world!")
 463
 464             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 465                 fr = f.read()
 466                 self.assertEqual(fr, "Hello world!")
 467                 f.seek(0)
 468                 f.write("Hola mundo!")
 469                 f.seek(0)
 470                 fr = f.read()
 471                 self.assertEqual(fr, "Hola mundo!!")
 472
 473             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 474                 self.assertEqual(f.read(), "Hola mundo!!")
 475
 476     Test().runTest()
 477
 478 class FuseUpdateFileTest(MountTestBase):
 479     def runTest(self):
 480         collection = arvados.collection.Collection(api_client=self.api)
 481         collection.save_new()
 482
 483         m = self.make_mount(fuse.CollectionDirectory)
 484         with llfuse.lock:
 485             m.new_collection(collection.api_response(), collection)
 486         self.assertTrue(m.writable())
 487
 488         # See note in MountTestBase.setUp
 489         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 490
 491         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 492         self.assertRegexpMatches(collection2["manifest_text"],
 493             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 494
 495
 496 def fuseMkdirTestHelper(mounttmp):
 497     class Test(unittest.TestCase):
 498         def runTest(self):
 499             with self.assertRaises(IOError):
 500                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 501                     f.write("Hello world!")
 502
 503             os.mkdir(os.path.join(mounttmp, "testdir"))
 504
 505             with self.assertRaises(OSError):
 506                 os.mkdir(os.path.join(mounttmp, "testdir"))
 507
 508             d1 = llfuse.listdir(mounttmp)
 509             self.assertEqual(["testdir"], d1)
 510
 511             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 512                 f.write("Hello world!")
 513
 514             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 515             self.assertEqual(["file1.txt"], d1)
 516
 517     Test().runTest()
 518
 519 class FuseMkdirTest(MountTestBase):
 520     def runTest(self):
 521         collection = arvados.collection.Collection(api_client=self.api)
 522         collection.save_new()
 523
 524         m = self.make_mount(fuse.CollectionDirectory)
 525         with llfuse.lock:
 526             m.new_collection(collection.api_response(), collection)
 527         self.assertTrue(m.writable())
 528
 529         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 530
 531         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 532         self.assertRegexpMatches(collection2["manifest_text"],
 533             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 534
 535
 536 def fuseRmTestHelperWriteFile(mounttmp):
 537     class Test(unittest.TestCase):
 538         def runTest(self):
 539             os.mkdir(os.path.join(mounttmp, "testdir"))
 540
 541             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 542                 f.write("Hello world!")
 543
 544     Test().runTest()
 545
 546 def fuseRmTestHelperDeleteFile(mounttmp):
 547     class Test(unittest.TestCase):
 548         def runTest(self):
 549             # Can't delete because it's not empty
 550             with self.assertRaises(OSError):
 551                 os.rmdir(os.path.join(mounttmp, "testdir"))
 552
 553             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 554             self.assertEqual(["file1.txt"], d1)
 555
 556             # Delete file
 557             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 558
 559             # Make sure it's empty
 560             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 561             self.assertEqual([], d1)
 562
 563             # Try to delete it again
 564             with self.assertRaises(OSError):
 565                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 566
 567     Test().runTest()
 568
 569 def fuseRmTestHelperRmdir(mounttmp):
 570     class Test(unittest.TestCase):
 571         def runTest(self):
 572             # Should be able to delete now that it is empty
 573             os.rmdir(os.path.join(mounttmp, "testdir"))
 574
 575             # Make sure it's empty
 576             d1 = llfuse.listdir(os.path.join(mounttmp))
 577             self.assertEqual([], d1)
 578
 579             # Try to delete it again
 580             with self.assertRaises(OSError):
 581                 os.rmdir(os.path.join(mounttmp, "testdir"))
 582
 583     Test().runTest()
 584
 585 class FuseRmTest(MountTestBase):
 586     def runTest(self):
 587         collection = arvados.collection.Collection(api_client=self.api)
 588         collection.save_new()
 589
 590         m = self.make_mount(fuse.CollectionDirectory)
 591         with llfuse.lock:
 592             m.new_collection(collection.api_response(), collection)
 593         self.assertTrue(m.writable())
 594
 595         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 596
 597         # Starting manifest
 598         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 599         self.assertRegexpMatches(collection2["manifest_text"],
 600             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 601         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 602
 603         # Can't have empty directories :-( so manifest will be empty.
 604         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 605         self.assertEqual(collection2["manifest_text"], "")
 606
 607         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 608
 609         # manifest should be empty now.
 610         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 611         self.assertEqual(collection2["manifest_text"], "")
 612
 613
 614 def fuseMvFileTestHelperWriteFile(mounttmp):
 615     class Test(unittest.TestCase):
 616         def runTest(self):
 617             os.mkdir(os.path.join(mounttmp, "testdir"))
 618
 619             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 620                 f.write("Hello world!")
 621
 622     Test().runTest()
 623
 624 def fuseMvFileTestHelperMoveFile(mounttmp):
 625     class Test(unittest.TestCase):
 626         def runTest(self):
 627             d1 = llfuse.listdir(os.path.join(mounttmp))
 628             self.assertEqual(["testdir"], d1)
 629             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 630             self.assertEqual(["file1.txt"], d1)
 631
 632             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 633
 634             d1 = llfuse.listdir(os.path.join(mounttmp))
 635             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 636             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 637             self.assertEqual([], d1)
 638
 639     Test().runTest()
 640
 641 class FuseMvFileTest(MountTestBase):
 642     def runTest(self):
 643         collection = arvados.collection.Collection(api_client=self.api)
 644         collection.save_new()
 645
 646         m = self.make_mount(fuse.CollectionDirectory)
 647         with llfuse.lock:
 648             m.new_collection(collection.api_response(), collection)
 649         self.assertTrue(m.writable())
 650
 651         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 652
 653         # Starting manifest
 654         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 655         self.assertRegexpMatches(collection2["manifest_text"],
 656             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 657
 658         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 659
 660         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 661         self.assertRegexpMatches(collection2["manifest_text"],
 662             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 663
 664
 665 def fuseRenameTestHelper(mounttmp):
 666     class Test(unittest.TestCase):
 667         def runTest(self):
 668             os.mkdir(os.path.join(mounttmp, "testdir"))
 669
 670             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 671                 f.write("Hello world!")
 672
 673     Test().runTest()
 674
 675 class FuseRenameTest(MountTestBase):
 676     def runTest(self):
 677         collection = arvados.collection.Collection(api_client=self.api)
 678         collection.save_new()
 679
 680         m = self.make_mount(fuse.CollectionDirectory)
 681         with llfuse.lock:
 682             m.new_collection(collection.api_response(), collection)
 683         self.assertTrue(m.writable())
 684
 685         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 686
 687         # Starting manifest
 688         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 689         self.assertRegexpMatches(collection2["manifest_text"],
 690             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 691
 692         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 693         self.assertEqual(["testdir"], d1)
 694         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 695         self.assertEqual(["file1.txt"], d1)
 696
 697         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 698
 699         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 700         self.assertEqual(["testdir2"], sorted(d1))
 701         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 702         self.assertEqual(["file1.txt"], d1)
 703
 704         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 705         self.assertRegexpMatches(collection2["manifest_text"],
 706             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 707
 708
 709 class FuseUpdateFromEventTest(MountTestBase):
 710     def runTest(self):
 711         collection = arvados.collection.Collection(api_client=self.api)
 712         collection.save_new()
 713
 714         m = self.make_mount(fuse.CollectionDirectory)
 715         with llfuse.lock:
 716             m.new_collection(collection.api_response(), collection)
 717
 718         self.operations.listen_for_events()
 719
 720         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 721         self.assertEqual([], sorted(d1))
 722
 723         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 724             with collection2.open("file1.txt", "w") as f:
 725                 f.write("foo")
 726
 727         for attempt in AssertWithTimeout(10):
 728             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 729
 730
 731 class FuseDeleteProjectEventTest(MountTestBase):
 732     def runTest(self):
 733
 734         aproject = self.api.groups().create(body={
 735             "name": "aproject",
 736             "group_class": "project"
 737         }).execute()
 738
 739         bproject = self.api.groups().create(body={
 740             "name": "bproject",
 741             "group_class": "project",
 742             "owner_uuid": aproject["uuid"]
 743         }).execute()
 744
 745         self.make_mount(fuse.ProjectDirectory,
 746                         project_object=self.api.users().current().execute())
 747
 748         self.operations.listen_for_events()
 749
 750         d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
 751         self.assertEqual(["bproject"], sorted(d1))
 752
 753         self.api.groups().delete(uuid=bproject["uuid"]).execute()
 754
 755         for attempt in AssertWithTimeout(10):
 756             attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
 757
 758
 759 def fuseFileConflictTestHelper(mounttmp):
 760     class Test(unittest.TestCase):
 761         def runTest(self):
 762             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 763                 f.write("bar")
 764
 765             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 766             self.assertEqual(len(d1), 2)
 767
 768             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 769                 self.assertEqual(f.read(), "bar")
 770
 771             self.assertRegexpMatches(d1[1],
 772                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 773
 774             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 775                 self.assertEqual(f.read(), "foo")
 776
 777     Test().runTest()
 778
 779 class FuseFileConflictTest(MountTestBase):
 780     def runTest(self):
 781         collection = arvados.collection.Collection(api_client=self.api)
 782         collection.save_new()
 783
 784         m = self.make_mount(fuse.CollectionDirectory)
 785         with llfuse.lock:
 786             m.new_collection(collection.api_response(), collection)
 787
 788         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 789         self.assertEqual([], sorted(d1))
 790
 791         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 792             with collection2.open("file1.txt", "w") as f:
 793                 f.write("foo")
 794
 795         # See note in MountTestBase.setUp
 796         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 797
 798
 799 def fuseUnlinkOpenFileTest(mounttmp):
 800     class Test(unittest.TestCase):
 801         def runTest(self):
 802             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 803                 f.write("foo")
 804
 805                 d1 = llfuse.listdir(os.path.join(mounttmp))
 806                 self.assertEqual(["file1.txt"], sorted(d1))
 807
 808                 os.remove(os.path.join(mounttmp, "file1.txt"))
 809
 810                 d1 = llfuse.listdir(os.path.join(mounttmp))
 811                 self.assertEqual([], sorted(d1))
 812
 813                 f.seek(0)
 814                 self.assertEqual(f.read(), "foo")
 815                 f.write("bar")
 816
 817                 f.seek(0)
 818                 self.assertEqual(f.read(), "foobar")
 819
 820     Test().runTest()
 821
 822 class FuseUnlinkOpenFileTest(MountTestBase):
 823     def runTest(self):
 824         collection = arvados.collection.Collection(api_client=self.api)
 825         collection.save_new()
 826
 827         m = self.make_mount(fuse.CollectionDirectory)
 828         with llfuse.lock:
 829             m.new_collection(collection.api_response(), collection)
 830
 831         # See note in MountTestBase.setUp
 832         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 833
 834         self.assertEqual(collection.manifest_text(), "")
 835
 836
 837 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 838     class Test(unittest.TestCase):
 839         def runTest(self):
 840             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 841                 f.write("Hello world!")
 842
 843             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 844             self.assertEqual(["file1.txt"], sorted(d1))
 845             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 846             self.assertEqual([], sorted(d1))
 847
 848     Test().runTest()
 849
 850 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 851     class Test(unittest.TestCase):
 852         def runTest(self):
 853             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 854
 855             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 856             self.assertEqual([], sorted(d1))
 857             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 858             self.assertEqual(["file2.txt"], sorted(d1))
 859
 860     Test().runTest()
 861
 862 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 863     def runTest(self):
 864         collection1 = arvados.collection.Collection(api_client=self.api)
 865         collection1.save_new()
 866
 867         collection2 = arvados.collection.Collection(api_client=self.api)
 868         collection2.save_new()
 869
 870         m = self.make_mount(fuse.MagicDirectory)
 871
 872         # See note in MountTestBase.setUp
 873         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 874                                                   collection1.manifest_locator(),
 875                                                   collection2.manifest_locator()))
 876
 877         collection1.update()
 878         collection2.update()
 879
 880         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 881         self.assertEqual(collection2.manifest_text(), "")
 882
 883         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 884                                                   collection1.manifest_locator(),
 885                                                   collection2.manifest_locator()))
 886
 887         collection1.update()
 888         collection2.update()
 889
 890         self.assertEqual(collection1.manifest_text(), "")
 891         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 892
 893         collection1.stop_threads()
 894         collection2.stop_threads()
 895
 896
 897 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 898     class Test(unittest.TestCase):
 899         def runTest(self):
 900             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 901             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 902                 f.write("Hello world!")
 903
 904             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 905             self.assertEqual(["testdir"], sorted(d1))
 906             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 907             self.assertEqual(["file1.txt"], sorted(d1))
 908
 909             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 910             self.assertEqual([], sorted(d1))
 911
 912     Test().runTest()
 913
 914
 915 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 916     class Test(unittest.TestCase):
 917         def runTest(self):
 918             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 919
 920             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 921             self.assertEqual([], sorted(d1))
 922
 923             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 924             self.assertEqual(["testdir2"], sorted(d1))
 925             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 926             self.assertEqual(["file1.txt"], sorted(d1))
 927
 928             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 929                 self.assertEqual(f.read(), "Hello world!")
 930
 931     Test().runTest()
 932
 933 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 934     def runTest(self):
 935         collection1 = arvados.collection.Collection(api_client=self.api)
 936         collection1.save_new()
 937
 938         collection2 = arvados.collection.Collection(api_client=self.api)
 939         collection2.save_new()
 940
 941         m = self.make_mount(fuse.MagicDirectory)
 942
 943         # See note in MountTestBase.setUp
 944         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 945                                                   collection1.manifest_locator(),
 946                                                   collection2.manifest_locator()))
 947
 948         collection1.update()
 949         collection2.update()
 950
 951         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 952         self.assertEqual(collection2.manifest_text(), "")
 953
 954         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 955                                                   collection1.manifest_locator(),
 956                                                   collection2.manifest_locator()))
 957
 958         collection1.update()
 959         collection2.update()
 960
 961         self.assertEqual(collection1.manifest_text(), "")
 962         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 963
 964         collection1.stop_threads()
 965         collection2.stop_threads()
 966
 967 def fuseProjectMkdirTestHelper1(mounttmp):
 968     class Test(unittest.TestCase):
 969         def runTest(self):
 970             os.mkdir(os.path.join(mounttmp, "testcollection"))
 971             with self.assertRaises(OSError):
 972                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 973     Test().runTest()
 974
 975 def fuseProjectMkdirTestHelper2(mounttmp):
 976     class Test(unittest.TestCase):
 977         def runTest(self):
 978             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 979                 f.write("Hello world!")
 980             with self.assertRaises(OSError):
 981                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 982             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 983             with self.assertRaises(OSError):
 984                 os.remove(os.path.join(mounttmp, "testcollection"))
 985             os.rmdir(os.path.join(mounttmp, "testcollection"))
 986     Test().runTest()
 987
 988 class FuseProjectMkdirRmdirTest(MountTestBase):
 989     def runTest(self):
 990         self.make_mount(fuse.ProjectDirectory,
 991                         project_object=self.api.users().current().execute())
 992
 993         d1 = llfuse.listdir(self.mounttmp)
 994         self.assertNotIn('testcollection', d1)
 995
 996         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
 997
 998         d1 = llfuse.listdir(self.mounttmp)
 999         self.assertIn('testcollection', d1)
1000
1001         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1002
1003         d1 = llfuse.listdir(self.mounttmp)
1004         self.assertNotIn('testcollection', d1)
1005
1006
1007 def fuseProjectMvTestHelper1(mounttmp):
1008     class Test(unittest.TestCase):
1009         def runTest(self):
1010             d1 = llfuse.listdir(mounttmp)
1011             self.assertNotIn('testcollection', d1)
1012
1013             os.mkdir(os.path.join(mounttmp, "testcollection"))
1014
1015             d1 = llfuse.listdir(mounttmp)
1016             self.assertIn('testcollection', d1)
1017
1018             with self.assertRaises(OSError):
1019                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1020
1021             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1022
1023             d1 = llfuse.listdir(mounttmp)
1024             self.assertNotIn('testcollection', d1)
1025
1026             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1027             self.assertIn('testcollection', d1)
1028
1029     Test().runTest()
1030
1031 class FuseProjectMvTest(MountTestBase):
1032     def runTest(self):
1033         self.make_mount(fuse.ProjectDirectory,
1034                         project_object=self.api.users().current().execute())
1035
1036         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1037
1038
1039 def fuseFsyncTestHelper(mounttmp, k):
1040     class Test(unittest.TestCase):
1041         def runTest(self):
1042             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1043             os.fsync(fd)
1044             os.close(fd)
1045
1046     Test().runTest()
1047
1048 class FuseFsyncTest(FuseMagicTest):
1049     def runTest(self):
1050         self.make_mount(fuse.MagicDirectory)
1051         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1052
1053
1054 class MagicDirApiError(FuseMagicTest):
1055     def setUp(self):
1056         api = mock.MagicMock()
1057         super(MagicDirApiError, self).setUp(api=api)
1058         api.collections().get().execute.side_effect = iter([
1059             Exception('API fail'),
1060             {
1061                 "manifest_text": self.test_manifest,
1062                 "portable_data_hash": self.test_manifest_pdh,
1063             },
1064         ])
1065         api.keep.get.side_effect = Exception('Keep fail')
1066
1067     def runTest(self):
1068         self.make_mount(fuse.MagicDirectory)
1069
1070         self.operations.inodes.inode_cache.cap = 1
1071         self.operations.inodes.inode_cache.min_entries = 2
1072
1073         with self.assertRaises(OSError):
1074             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1075
1076         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1077
1078
1079 class FuseUnitTest(unittest.TestCase):
1080     def test_sanitize_filename(self):
1081         acceptable = [
1082             "foo.txt",
1083             ".foo",
1084             "..foo",
1085             "...",
1086             "foo...",
1087             "foo..",
1088             "foo.",
1089             "-",
1090             "\x01\x02\x03",
1091             ]
1092         unacceptable = [
1093             "f\00",
1094             "\00\00",
1095             "/foo",
1096             "foo/",
1097             "//",
1098             ]
1099         for f in acceptable:
1100             self.assertEqual(f, fuse.sanitize_filename(f))
1101         for f in unacceptable:
1102             self.assertNotEqual(f, fuse.sanitize_filename(f))
1103             # The sanitized filename should be the same length, though.
1104             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1105         # Special cases
1106         self.assertEqual("_", fuse.sanitize_filename(""))
1107         self.assertEqual("_", fuse.sanitize_filename("."))
1108         self.assertEqual("__", fuse.sanitize_filename(".."))
1109
1110
1111 class FuseMagicTestPDHOnly(MountTestBase):
1112     def setUp(self, api=None):
1113         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1114
1115         cw = arvados.CollectionWriter()
1116
1117         cw.start_new_file('thing1.txt')
1118         cw.write("data 1")
1119
1120         self.testcollection = cw.finish()
1121         self.test_manifest = cw.manifest_text()
1122         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1123         self.testcollectionuuid = str(created['uuid'])
1124
1125     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1126         if skip_pdh_only is True:
1127             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1128         else:
1129             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1130
1131         mount_ls = llfuse.listdir(self.mounttmp)
1132         self.assertIn('README', mount_ls)
1133         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1134                              arvados.util.uuid_pattern.match(fn)
1135                              for fn in mount_ls),
1136                          "new FUSE MagicDirectory lists Collection")
1137
1138         # look up using pdh should succeed in all cases
1139         self.assertDirContents(self.testcollection, ['thing1.txt'])
1140         self.assertDirContents(os.path.join('by_id', self.testcollection),
1141                                ['thing1.txt'])
1142         mount_ls = llfuse.listdir(self.mounttmp)
1143         self.assertIn('README', mount_ls)
1144         self.assertIn(self.testcollection, mount_ls)
1145         self.assertIn(self.testcollection,
1146                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1147
1148         files = {}
1149         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1150
1151         for k, v in files.items():
1152             with open(os.path.join(self.mounttmp, k)) as f:
1153                 self.assertEqual(v, f.read())
1154
1155         # look up using uuid should fail when pdh_only is set
1156         if pdh_only is True:
1157             with self.assertRaises(OSError):
1158                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1159                                ['thing1.txt'])
1160         else:
1161             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1162                                ['thing1.txt'])
1163
1164     def test_with_pdh_only_true(self):
1165         self.verify_pdh_only(pdh_only=True)
1166
1167     def test_with_pdh_only_false(self):
1168         self.verify_pdh_only(pdh_only=False)
1169
1170     def test_with_default_by_id(self):
1171         self.verify_pdh_only(skip_pdh_only=True)