services/fuse/tests/test_mount.py

   1 import arvados
   2 import arvados.safeapi
   3 import arvados_fuse as fuse
   4 import glob
   5 import json
   6 import llfuse
   7 import os
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tempfile
  12 import threading
  13 import time
  14 import unittest
  15 import logging
  16 import multiprocessing
  17 import run_test_server
  18 import mock
  19 import re
  20
  21 from mount_test_base import MountTestBase
  22
  23 logger = logging.getLogger('arvados.arv-mount')
  24
  25
  26 class AssertWithTimeout(object):
  27     """Allow some time for an assertion to pass."""
  28
  29     def __init__(self, timeout=0):
  30         self.timeout = timeout
  31
  32     def __iter__(self):
  33         self.deadline = time.time() + self.timeout
  34         self.done = False
  35         return self
  36
  37     def next(self):
  38         if self.done:
  39             raise StopIteration
  40         return self.attempt
  41
  42     def attempt(self, fn, *args, **kwargs):
  43         try:
  44             fn(*args, **kwargs)
  45         except AssertionError:
  46             if time.time() > self.deadline:
  47                 raise
  48             time.sleep(0.1)
  49         else:
  50             self.done = True
  51
  52
  53 class FuseMountTest(MountTestBase):
  54     def setUp(self):
  55         super(FuseMountTest, self).setUp()
  56
  57         cw = arvados.CollectionWriter()
  58
  59         cw.start_new_file('thing1.txt')
  60         cw.write("data 1")
  61         cw.start_new_file('thing2.txt')
  62         cw.write("data 2")
  63
  64         cw.start_new_stream('dir1')
  65         cw.start_new_file('thing3.txt')
  66         cw.write("data 3")
  67         cw.start_new_file('thing4.txt')
  68         cw.write("data 4")
  69
  70         cw.start_new_stream('dir2')
  71         cw.start_new_file('thing5.txt')
  72         cw.write("data 5")
  73         cw.start_new_file('thing6.txt')
  74         cw.write("data 6")
  75
  76         cw.start_new_stream('dir2/dir3')
  77         cw.start_new_file('thing7.txt')
  78         cw.write("data 7")
  79
  80         cw.start_new_file('thing8.txt')
  81         cw.write("data 8")
  82
  83         cw.start_new_stream('edgecases')
  84         for f in ":/.../-/*/\x01\\/ ".split("/"):
  85             cw.start_new_file(f)
  86             cw.write('x')
  87
  88         for f in ":/.../-/*/\x01\\/ ".split("/"):
  89             cw.start_new_stream('edgecases/dirs/' + f)
  90             cw.start_new_file('x/x')
  91             cw.write('x')
  92
  93         self.testcollection = cw.finish()
  94         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  95
  96     def runTest(self):
  97         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  98
  99         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 100                                       'edgecases', 'dir1', 'dir2'])
 101         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 102         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 103         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 104         self.assertDirContents('edgecases',
 105                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
 106         self.assertDirContents('edgecases/dirs',
 107                                ":/.../-/*/\x01\\/ ".split("/"))
 108
 109         files = {'thing1.txt': 'data 1',
 110                  'thing2.txt': 'data 2',
 111                  'dir1/thing3.txt': 'data 3',
 112                  'dir1/thing4.txt': 'data 4',
 113                  'dir2/thing5.txt': 'data 5',
 114                  'dir2/thing6.txt': 'data 6',
 115                  'dir2/dir3/thing7.txt': 'data 7',
 116                  'dir2/dir3/thing8.txt': 'data 8'}
 117
 118         for k, v in files.items():
 119             with open(os.path.join(self.mounttmp, k)) as f:
 120                 self.assertEqual(v, f.read())
 121
 122
 123 class FuseNoAPITest(MountTestBase):
 124     def setUp(self):
 125         super(FuseNoAPITest, self).setUp()
 126         keep = arvados.keep.KeepClient(local_store=self.keeptmp)
 127         self.file_data = "API-free text\n"
 128         self.file_loc = keep.put(self.file_data)
 129         self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
 130                 self.file_loc, len(self.file_data)))
 131
 132     def runTest(self):
 133         self.make_mount(fuse.MagicDirectory)
 134         self.assertDirContents(self.coll_loc, ['api-free.txt'])
 135         with open(os.path.join(
 136                 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
 137             actual = keep_file.read(-1)
 138         self.assertEqual(self.file_data, actual)
 139
 140
 141 class FuseMagicTest(MountTestBase):
 142     def setUp(self, api=None):
 143         super(FuseMagicTest, self).setUp(api=api)
 144
 145         cw = arvados.CollectionWriter()
 146
 147         cw.start_new_file('thing1.txt')
 148         cw.write("data 1")
 149
 150         self.testcollection = cw.finish()
 151         self.test_manifest = cw.manifest_text()
 152         self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 153
 154     def runTest(self):
 155         self.make_mount(fuse.MagicDirectory)
 156
 157         mount_ls = llfuse.listdir(self.mounttmp)
 158         self.assertIn('README', mount_ls)
 159         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 160                              arvados.util.uuid_pattern.match(fn)
 161                              for fn in mount_ls),
 162                          "new FUSE MagicDirectory lists Collection")
 163         self.assertDirContents(self.testcollection, ['thing1.txt'])
 164         self.assertDirContents(os.path.join('by_id', self.testcollection),
 165                                ['thing1.txt'])
 166         mount_ls = llfuse.listdir(self.mounttmp)
 167         self.assertIn('README', mount_ls)
 168         self.assertIn(self.testcollection, mount_ls)
 169         self.assertIn(self.testcollection,
 170                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 171
 172         files = {}
 173         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 174
 175         for k, v in files.items():
 176             with open(os.path.join(self.mounttmp, k)) as f:
 177                 self.assertEqual(v, f.read())
 178
 179
 180 class FuseTagsTest(MountTestBase):
 181     def runTest(self):
 182         self.make_mount(fuse.TagsDirectory)
 183
 184         d1 = llfuse.listdir(self.mounttmp)
 185         d1.sort()
 186         self.assertEqual(['foo_tag'], d1)
 187
 188         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 189         d2.sort()
 190         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 191
 192         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 193         d3.sort()
 194         self.assertEqual(['foo'], d3)
 195
 196
 197 class FuseTagsUpdateTest(MountTestBase):
 198     def tag_collection(self, coll_uuid, tag_name):
 199         return self.api.links().create(
 200             body={'link': {'head_uuid': coll_uuid,
 201                            'link_class': 'tag',
 202                            'name': tag_name,
 203         }}).execute()
 204
 205     def runTest(self):
 206         self.make_mount(fuse.TagsDirectory, poll_time=1)
 207
 208         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 209
 210         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 211         self.tag_collection(bar_uuid, 'fuse_test_tag')
 212         for attempt in AssertWithTimeout(10):
 213             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 214         self.assertDirContents('fuse_test_tag', [bar_uuid])
 215
 216         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 217         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 218         for attempt in AssertWithTimeout(10):
 219             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 220
 221         self.api.links().delete(uuid=l['uuid']).execute()
 222         for attempt in AssertWithTimeout(10):
 223             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 224
 225
 226 class FuseSharedTest(MountTestBase):
 227     def runTest(self):
 228         self.make_mount(fuse.SharedDirectory,
 229                         exclude=self.api.users().current().execute()['uuid'])
 230
 231         # shared_dirs is a list of the directories exposed
 232         # by fuse.SharedDirectory (i.e. any object visible
 233         # to the current user)
 234         shared_dirs = llfuse.listdir(self.mounttmp)
 235         shared_dirs.sort()
 236         self.assertIn('FUSE User', shared_dirs)
 237
 238         # fuse_user_objs is a list of the objects owned by the FUSE
 239         # test user (which present as files in the 'FUSE User'
 240         # directory)
 241         fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
 242         fuse_user_objs.sort()
 243         self.assertEqual(['FUSE Test Project',                    # project owned by user
 244                           'collection #1 owned by FUSE',          # collection owned by user
 245                           'collection #2 owned by FUSE',          # collection owned by user
 246                           'pipeline instance owned by FUSE.pipelineInstance',  # pipeline instance owned by user
 247                       ], fuse_user_objs)
 248
 249         # test_proj_files is a list of the files in the FUSE Test Project.
 250         test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
 251         test_proj_files.sort()
 252         self.assertEqual(['collection in FUSE project',
 253                           'pipeline instance in FUSE project.pipelineInstance',
 254                           'pipeline template in FUSE project.pipelineTemplate'
 255                       ], test_proj_files)
 256
 257         # Double check that we can open and read objects in this folder as a file,
 258         # and that its contents are what we expect.
 259         pipeline_template_path = os.path.join(
 260                 self.mounttmp,
 261                 'FUSE User',
 262                 'FUSE Test Project',
 263                 'pipeline template in FUSE project.pipelineTemplate')
 264         with open(pipeline_template_path) as f:
 265             j = json.load(f)
 266             self.assertEqual("pipeline template in FUSE project", j['name'])
 267
 268         # check mtime on template
 269         st = os.stat(pipeline_template_path)
 270         self.assertEqual(st.st_mtime, 1397493304)
 271
 272         # check mtime on collection
 273         st = os.stat(os.path.join(
 274                 self.mounttmp,
 275                 'FUSE User',
 276                 'collection #1 owned by FUSE'))
 277         self.assertEqual(st.st_mtime, 1391448174)
 278
 279
 280 class FuseHomeTest(MountTestBase):
 281     def runTest(self):
 282         self.make_mount(fuse.ProjectDirectory,
 283                         project_object=self.api.users().current().execute())
 284
 285         d1 = llfuse.listdir(self.mounttmp)
 286         self.assertIn('Unrestricted public data', d1)
 287
 288         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 289         public_project = run_test_server.fixture('groups')[
 290             'anonymously_accessible_project']
 291         found_in = 0
 292         found_not_in = 0
 293         for name, item in run_test_server.fixture('collections').iteritems():
 294             if 'name' not in item:
 295                 pass
 296             elif item['owner_uuid'] == public_project['uuid']:
 297                 self.assertIn(item['name'], d2)
 298                 found_in += 1
 299             else:
 300                 # Artificial assumption here: there is no public
 301                 # collection fixture with the same name as a
 302                 # non-public collection.
 303                 self.assertNotIn(item['name'], d2)
 304                 found_not_in += 1
 305         self.assertNotEqual(0, found_in)
 306         self.assertNotEqual(0, found_not_in)
 307
 308         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 309         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 310
 311
 312 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 313     class Test(unittest.TestCase):
 314         def runTest(self):
 315             d1 = llfuse.listdir(mounttmp)
 316             self.assertEqual(["file1.txt"], d1)
 317             with open(os.path.join(mounttmp, "file1.txt")) as f:
 318                 self.assertEqual("blub", f.read())
 319     Test().runTest()
 320
 321 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 322     class Test(unittest.TestCase):
 323         def runTest(self):
 324             d1 = llfuse.listdir(mounttmp)
 325             self.assertEqual(["file1.txt"], d1)
 326             with open(os.path.join(mounttmp, "file1.txt")) as f:
 327                 self.assertEqual("plnp", f.read())
 328     Test().runTest()
 329
 330 class FuseModifyFileTest(MountTestBase):
 331     def runTest(self):
 332         collection = arvados.collection.Collection(api_client=self.api)
 333         with collection.open("file1.txt", "w") as f:
 334             f.write("blub")
 335
 336         collection.save_new()
 337
 338         m = self.make_mount(fuse.CollectionDirectory)
 339         with llfuse.lock:
 340             m.new_collection(collection.api_response(), collection)
 341
 342         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 343
 344         with collection.open("file1.txt", "w") as f:
 345             f.write("plnp")
 346
 347         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 348
 349
 350 class FuseAddFileToCollectionTest(MountTestBase):
 351     def runTest(self):
 352         collection = arvados.collection.Collection(api_client=self.api)
 353         with collection.open("file1.txt", "w") as f:
 354             f.write("blub")
 355
 356         collection.save_new()
 357
 358         m = self.make_mount(fuse.CollectionDirectory)
 359         with llfuse.lock:
 360             m.new_collection(collection.api_response(), collection)
 361
 362         d1 = llfuse.listdir(self.mounttmp)
 363         self.assertEqual(["file1.txt"], d1)
 364
 365         with collection.open("file2.txt", "w") as f:
 366             f.write("plnp")
 367
 368         d1 = llfuse.listdir(self.mounttmp)
 369         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 370
 371
 372 class FuseRemoveFileFromCollectionTest(MountTestBase):
 373     def runTest(self):
 374         collection = arvados.collection.Collection(api_client=self.api)
 375         with collection.open("file1.txt", "w") as f:
 376             f.write("blub")
 377
 378         with collection.open("file2.txt", "w") as f:
 379             f.write("plnp")
 380
 381         collection.save_new()
 382
 383         m = self.make_mount(fuse.CollectionDirectory)
 384         with llfuse.lock:
 385             m.new_collection(collection.api_response(), collection)
 386
 387         d1 = llfuse.listdir(self.mounttmp)
 388         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 389
 390         collection.remove("file2.txt")
 391
 392         d1 = llfuse.listdir(self.mounttmp)
 393         self.assertEqual(["file1.txt"], d1)
 394
 395
 396 def fuseCreateFileTestHelper(mounttmp):
 397     class Test(unittest.TestCase):
 398         def runTest(self):
 399             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 400                 pass
 401     Test().runTest()
 402
 403 class FuseCreateFileTest(MountTestBase):
 404     def runTest(self):
 405         collection = arvados.collection.Collection(api_client=self.api)
 406         collection.save_new()
 407
 408         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 409         self.assertEqual(collection2["manifest_text"], "")
 410
 411         collection.save_new()
 412
 413         m = self.make_mount(fuse.CollectionDirectory)
 414         with llfuse.lock:
 415             m.new_collection(collection.api_response(), collection)
 416         self.assertTrue(m.writable())
 417
 418         self.assertNotIn("file1.txt", collection)
 419
 420         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 421
 422         self.assertIn("file1.txt", collection)
 423
 424         d1 = llfuse.listdir(self.mounttmp)
 425         self.assertEqual(["file1.txt"], d1)
 426
 427         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 428         self.assertRegexpMatches(collection2["manifest_text"],
 429             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 430
 431
 432 def fuseWriteFileTestHelperWriteFile(mounttmp):
 433     class Test(unittest.TestCase):
 434         def runTest(self):
 435             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 436                 f.write("Hello world!")
 437     Test().runTest()
 438
 439 def fuseWriteFileTestHelperReadFile(mounttmp):
 440     class Test(unittest.TestCase):
 441         def runTest(self):
 442             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 443                 self.assertEqual(f.read(), "Hello world!")
 444     Test().runTest()
 445
 446 class FuseWriteFileTest(MountTestBase):
 447     def runTest(self):
 448         collection = arvados.collection.Collection(api_client=self.api)
 449         collection.save_new()
 450
 451         m = self.make_mount(fuse.CollectionDirectory)
 452         with llfuse.lock:
 453             m.new_collection(collection.api_response(), collection)
 454         self.assertTrue(m.writable())
 455
 456         self.assertNotIn("file1.txt", collection)
 457
 458         self.assertEqual(0, self.operations.write_counter.get())
 459         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 460         self.assertEqual(12, self.operations.write_counter.get())
 461
 462         with collection.open("file1.txt") as f:
 463             self.assertEqual(f.read(), "Hello world!")
 464
 465         self.assertEqual(0, self.operations.read_counter.get())
 466         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 467         self.assertEqual(12, self.operations.read_counter.get())
 468
 469         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 470         self.assertRegexpMatches(collection2["manifest_text"],
 471             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 472
 473
 474 def fuseUpdateFileTestHelper(mounttmp):
 475     class Test(unittest.TestCase):
 476         def runTest(self):
 477             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 478                 f.write("Hello world!")
 479
 480             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 481                 fr = f.read()
 482                 self.assertEqual(fr, "Hello world!")
 483                 f.seek(0)
 484                 f.write("Hola mundo!")
 485                 f.seek(0)
 486                 fr = f.read()
 487                 self.assertEqual(fr, "Hola mundo!!")
 488
 489             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 490                 self.assertEqual(f.read(), "Hola mundo!!")
 491
 492     Test().runTest()
 493
 494 class FuseUpdateFileTest(MountTestBase):
 495     def runTest(self):
 496         collection = arvados.collection.Collection(api_client=self.api)
 497         collection.save_new()
 498
 499         m = self.make_mount(fuse.CollectionDirectory)
 500         with llfuse.lock:
 501             m.new_collection(collection.api_response(), collection)
 502         self.assertTrue(m.writable())
 503
 504         # See note in MountTestBase.setUp
 505         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 506
 507         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 508         self.assertRegexpMatches(collection2["manifest_text"],
 509             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 510
 511
 512 def fuseMkdirTestHelper(mounttmp):
 513     class Test(unittest.TestCase):
 514         def runTest(self):
 515             with self.assertRaises(IOError):
 516                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 517                     f.write("Hello world!")
 518
 519             os.mkdir(os.path.join(mounttmp, "testdir"))
 520
 521             with self.assertRaises(OSError):
 522                 os.mkdir(os.path.join(mounttmp, "testdir"))
 523
 524             d1 = llfuse.listdir(mounttmp)
 525             self.assertEqual(["testdir"], d1)
 526
 527             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 528                 f.write("Hello world!")
 529
 530             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 531             self.assertEqual(["file1.txt"], d1)
 532
 533     Test().runTest()
 534
 535 class FuseMkdirTest(MountTestBase):
 536     def runTest(self):
 537         collection = arvados.collection.Collection(api_client=self.api)
 538         collection.save_new()
 539
 540         m = self.make_mount(fuse.CollectionDirectory)
 541         with llfuse.lock:
 542             m.new_collection(collection.api_response(), collection)
 543         self.assertTrue(m.writable())
 544
 545         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 546
 547         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 548         self.assertRegexpMatches(collection2["manifest_text"],
 549             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 550
 551
 552 def fuseRmTestHelperWriteFile(mounttmp):
 553     class Test(unittest.TestCase):
 554         def runTest(self):
 555             os.mkdir(os.path.join(mounttmp, "testdir"))
 556
 557             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 558                 f.write("Hello world!")
 559
 560     Test().runTest()
 561
 562 def fuseRmTestHelperDeleteFile(mounttmp):
 563     class Test(unittest.TestCase):
 564         def runTest(self):
 565             # Can't delete because it's not empty
 566             with self.assertRaises(OSError):
 567                 os.rmdir(os.path.join(mounttmp, "testdir"))
 568
 569             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 570             self.assertEqual(["file1.txt"], d1)
 571
 572             # Delete file
 573             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 574
 575             # Make sure it's empty
 576             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 577             self.assertEqual([], d1)
 578
 579             # Try to delete it again
 580             with self.assertRaises(OSError):
 581                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 582
 583     Test().runTest()
 584
 585 def fuseRmTestHelperRmdir(mounttmp):
 586     class Test(unittest.TestCase):
 587         def runTest(self):
 588             # Should be able to delete now that it is empty
 589             os.rmdir(os.path.join(mounttmp, "testdir"))
 590
 591             # Make sure it's empty
 592             d1 = llfuse.listdir(os.path.join(mounttmp))
 593             self.assertEqual([], d1)
 594
 595             # Try to delete it again
 596             with self.assertRaises(OSError):
 597                 os.rmdir(os.path.join(mounttmp, "testdir"))
 598
 599     Test().runTest()
 600
 601 class FuseRmTest(MountTestBase):
 602     def runTest(self):
 603         collection = arvados.collection.Collection(api_client=self.api)
 604         collection.save_new()
 605
 606         m = self.make_mount(fuse.CollectionDirectory)
 607         with llfuse.lock:
 608             m.new_collection(collection.api_response(), collection)
 609         self.assertTrue(m.writable())
 610
 611         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 612
 613         # Starting manifest
 614         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 615         self.assertRegexpMatches(collection2["manifest_text"],
 616             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 617         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 618
 619         # Can't have empty directories :-( so manifest will be empty.
 620         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 621         self.assertEqual(collection2["manifest_text"], "")
 622
 623         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 624
 625         # manifest should be empty now.
 626         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 627         self.assertEqual(collection2["manifest_text"], "")
 628
 629
 630 def fuseMvFileTestHelperWriteFile(mounttmp):
 631     class Test(unittest.TestCase):
 632         def runTest(self):
 633             os.mkdir(os.path.join(mounttmp, "testdir"))
 634
 635             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 636                 f.write("Hello world!")
 637
 638     Test().runTest()
 639
 640 def fuseMvFileTestHelperMoveFile(mounttmp):
 641     class Test(unittest.TestCase):
 642         def runTest(self):
 643             d1 = llfuse.listdir(os.path.join(mounttmp))
 644             self.assertEqual(["testdir"], d1)
 645             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 646             self.assertEqual(["file1.txt"], d1)
 647
 648             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 649
 650             d1 = llfuse.listdir(os.path.join(mounttmp))
 651             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 652             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 653             self.assertEqual([], d1)
 654
 655     Test().runTest()
 656
 657 class FuseMvFileTest(MountTestBase):
 658     def runTest(self):
 659         collection = arvados.collection.Collection(api_client=self.api)
 660         collection.save_new()
 661
 662         m = self.make_mount(fuse.CollectionDirectory)
 663         with llfuse.lock:
 664             m.new_collection(collection.api_response(), collection)
 665         self.assertTrue(m.writable())
 666
 667         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 668
 669         # Starting manifest
 670         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 671         self.assertRegexpMatches(collection2["manifest_text"],
 672             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 673
 674         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 675
 676         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 677         self.assertRegexpMatches(collection2["manifest_text"],
 678             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 679
 680
 681 def fuseRenameTestHelper(mounttmp):
 682     class Test(unittest.TestCase):
 683         def runTest(self):
 684             os.mkdir(os.path.join(mounttmp, "testdir"))
 685
 686             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 687                 f.write("Hello world!")
 688
 689     Test().runTest()
 690
 691 class FuseRenameTest(MountTestBase):
 692     def runTest(self):
 693         collection = arvados.collection.Collection(api_client=self.api)
 694         collection.save_new()
 695
 696         m = self.make_mount(fuse.CollectionDirectory)
 697         with llfuse.lock:
 698             m.new_collection(collection.api_response(), collection)
 699         self.assertTrue(m.writable())
 700
 701         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 702
 703         # Starting manifest
 704         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 705         self.assertRegexpMatches(collection2["manifest_text"],
 706             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 707
 708         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 709         self.assertEqual(["testdir"], d1)
 710         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 711         self.assertEqual(["file1.txt"], d1)
 712
 713         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 714
 715         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 716         self.assertEqual(["testdir2"], sorted(d1))
 717         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 718         self.assertEqual(["file1.txt"], d1)
 719
 720         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 721         self.assertRegexpMatches(collection2["manifest_text"],
 722             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 723
 724
 725 class FuseUpdateFromEventTest(MountTestBase):
 726     def runTest(self):
 727         collection = arvados.collection.Collection(api_client=self.api)
 728         collection.save_new()
 729
 730         m = self.make_mount(fuse.CollectionDirectory)
 731         with llfuse.lock:
 732             m.new_collection(collection.api_response(), collection)
 733
 734         self.operations.listen_for_events()
 735
 736         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 737         self.assertEqual([], sorted(d1))
 738
 739         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 740             with collection2.open("file1.txt", "w") as f:
 741                 f.write("foo")
 742
 743         for attempt in AssertWithTimeout(10):
 744             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 745
 746
 747 def fuseFileConflictTestHelper(mounttmp):
 748     class Test(unittest.TestCase):
 749         def runTest(self):
 750             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 751                 f.write("bar")
 752
 753             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 754             self.assertEqual(len(d1), 2)
 755
 756             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 757                 self.assertEqual(f.read(), "bar")
 758
 759             self.assertRegexpMatches(d1[1],
 760                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 761
 762             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 763                 self.assertEqual(f.read(), "foo")
 764
 765     Test().runTest()
 766
 767 class FuseFileConflictTest(MountTestBase):
 768     def runTest(self):
 769         collection = arvados.collection.Collection(api_client=self.api)
 770         collection.save_new()
 771
 772         m = self.make_mount(fuse.CollectionDirectory)
 773         with llfuse.lock:
 774             m.new_collection(collection.api_response(), collection)
 775
 776         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 777         self.assertEqual([], sorted(d1))
 778
 779         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 780             with collection2.open("file1.txt", "w") as f:
 781                 f.write("foo")
 782
 783         # See note in MountTestBase.setUp
 784         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 785
 786
 787 def fuseUnlinkOpenFileTest(mounttmp):
 788     class Test(unittest.TestCase):
 789         def runTest(self):
 790             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 791                 f.write("foo")
 792
 793                 d1 = llfuse.listdir(os.path.join(mounttmp))
 794                 self.assertEqual(["file1.txt"], sorted(d1))
 795
 796                 os.remove(os.path.join(mounttmp, "file1.txt"))
 797
 798                 d1 = llfuse.listdir(os.path.join(mounttmp))
 799                 self.assertEqual([], sorted(d1))
 800
 801                 f.seek(0)
 802                 self.assertEqual(f.read(), "foo")
 803                 f.write("bar")
 804
 805                 f.seek(0)
 806                 self.assertEqual(f.read(), "foobar")
 807
 808     Test().runTest()
 809
 810 class FuseUnlinkOpenFileTest(MountTestBase):
 811     def runTest(self):
 812         collection = arvados.collection.Collection(api_client=self.api)
 813         collection.save_new()
 814
 815         m = self.make_mount(fuse.CollectionDirectory)
 816         with llfuse.lock:
 817             m.new_collection(collection.api_response(), collection)
 818
 819         # See note in MountTestBase.setUp
 820         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 821
 822         self.assertEqual(collection.manifest_text(), "")
 823
 824
 825 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 826     class Test(unittest.TestCase):
 827         def runTest(self):
 828             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 829                 f.write("Hello world!")
 830
 831             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 832             self.assertEqual(["file1.txt"], sorted(d1))
 833             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 834             self.assertEqual([], sorted(d1))
 835
 836     Test().runTest()
 837
 838 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 839     class Test(unittest.TestCase):
 840         def runTest(self):
 841             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 842
 843             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 844             self.assertEqual([], sorted(d1))
 845             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 846             self.assertEqual(["file2.txt"], sorted(d1))
 847
 848     Test().runTest()
 849
 850 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 851     def runTest(self):
 852         collection1 = arvados.collection.Collection(api_client=self.api)
 853         collection1.save_new()
 854
 855         collection2 = arvados.collection.Collection(api_client=self.api)
 856         collection2.save_new()
 857
 858         m = self.make_mount(fuse.MagicDirectory)
 859
 860         # See note in MountTestBase.setUp
 861         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 862                                                   collection1.manifest_locator(),
 863                                                   collection2.manifest_locator()))
 864
 865         collection1.update()
 866         collection2.update()
 867
 868         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 869         self.assertEqual(collection2.manifest_text(), "")
 870
 871         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 872                                                   collection1.manifest_locator(),
 873                                                   collection2.manifest_locator()))
 874
 875         collection1.update()
 876         collection2.update()
 877
 878         self.assertEqual(collection1.manifest_text(), "")
 879         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 880
 881         collection1.stop_threads()
 882         collection2.stop_threads()
 883
 884
 885 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 886     class Test(unittest.TestCase):
 887         def runTest(self):
 888             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 889             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 890                 f.write("Hello world!")
 891
 892             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 893             self.assertEqual(["testdir"], sorted(d1))
 894             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 895             self.assertEqual(["file1.txt"], sorted(d1))
 896
 897             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 898             self.assertEqual([], sorted(d1))
 899
 900     Test().runTest()
 901
 902
 903 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 904     class Test(unittest.TestCase):
 905         def runTest(self):
 906             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 907
 908             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 909             self.assertEqual([], sorted(d1))
 910
 911             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 912             self.assertEqual(["testdir2"], sorted(d1))
 913             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 914             self.assertEqual(["file1.txt"], sorted(d1))
 915
 916             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 917                 self.assertEqual(f.read(), "Hello world!")
 918
 919     Test().runTest()
 920
 921 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 922     def runTest(self):
 923         collection1 = arvados.collection.Collection(api_client=self.api)
 924         collection1.save_new()
 925
 926         collection2 = arvados.collection.Collection(api_client=self.api)
 927         collection2.save_new()
 928
 929         m = self.make_mount(fuse.MagicDirectory)
 930
 931         # See note in MountTestBase.setUp
 932         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 933                                                   collection1.manifest_locator(),
 934                                                   collection2.manifest_locator()))
 935
 936         collection1.update()
 937         collection2.update()
 938
 939         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 940         self.assertEqual(collection2.manifest_text(), "")
 941
 942         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 943                                                   collection1.manifest_locator(),
 944                                                   collection2.manifest_locator()))
 945
 946         collection1.update()
 947         collection2.update()
 948
 949         self.assertEqual(collection1.manifest_text(), "")
 950         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 951
 952         collection1.stop_threads()
 953         collection2.stop_threads()
 954
 955 def fuseProjectMkdirTestHelper1(mounttmp):
 956     class Test(unittest.TestCase):
 957         def runTest(self):
 958             os.mkdir(os.path.join(mounttmp, "testcollection"))
 959             with self.assertRaises(OSError):
 960                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 961     Test().runTest()
 962
 963 def fuseProjectMkdirTestHelper2(mounttmp):
 964     class Test(unittest.TestCase):
 965         def runTest(self):
 966             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 967                 f.write("Hello world!")
 968             with self.assertRaises(OSError):
 969                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 970             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 971             with self.assertRaises(OSError):
 972                 os.remove(os.path.join(mounttmp, "testcollection"))
 973             os.rmdir(os.path.join(mounttmp, "testcollection"))
 974     Test().runTest()
 975
 976 class FuseProjectMkdirRmdirTest(MountTestBase):
 977     def runTest(self):
 978         self.make_mount(fuse.ProjectDirectory,
 979                         project_object=self.api.users().current().execute())
 980
 981         d1 = llfuse.listdir(self.mounttmp)
 982         self.assertNotIn('testcollection', d1)
 983
 984         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
 985
 986         d1 = llfuse.listdir(self.mounttmp)
 987         self.assertIn('testcollection', d1)
 988
 989         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
 990
 991         d1 = llfuse.listdir(self.mounttmp)
 992         self.assertNotIn('testcollection', d1)
 993
 994
 995 def fuseProjectMvTestHelper1(mounttmp):
 996     class Test(unittest.TestCase):
 997         def runTest(self):
 998             d1 = llfuse.listdir(mounttmp)
 999             self.assertNotIn('testcollection', d1)
1000
1001             os.mkdir(os.path.join(mounttmp, "testcollection"))
1002
1003             d1 = llfuse.listdir(mounttmp)
1004             self.assertIn('testcollection', d1)
1005
1006             with self.assertRaises(OSError):
1007                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1008
1009             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1010
1011             d1 = llfuse.listdir(mounttmp)
1012             self.assertNotIn('testcollection', d1)
1013
1014             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1015             self.assertIn('testcollection', d1)
1016
1017     Test().runTest()
1018
1019 class FuseProjectMvTest(MountTestBase):
1020     def runTest(self):
1021         self.make_mount(fuse.ProjectDirectory,
1022                         project_object=self.api.users().current().execute())
1023
1024         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1025
1026
1027 def fuseFsyncTestHelper(mounttmp, k):
1028     class Test(unittest.TestCase):
1029         def runTest(self):
1030             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1031             os.fsync(fd)
1032             os.close(fd)
1033
1034     Test().runTest()
1035
1036 class FuseFsyncTest(FuseMagicTest):
1037     def runTest(self):
1038         self.make_mount(fuse.MagicDirectory)
1039         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1040
1041
1042 class MagicDirApiError(FuseMagicTest):
1043     def setUp(self):
1044         api = mock.MagicMock()
1045         super(MagicDirApiError, self).setUp(api=api)
1046         api.collections().get().execute.side_effect = iter([Exception('API fail'), {"manifest_text": self.test_manifest}])
1047         api.keep.get.side_effect = Exception('Keep fail')
1048
1049     def runTest(self):
1050         self.make_mount(fuse.MagicDirectory)
1051
1052         self.operations.inodes.inode_cache.cap = 1
1053         self.operations.inodes.inode_cache.min_entries = 2
1054
1055         with self.assertRaises(OSError):
1056             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1057
1058         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1059
1060
1061 class FuseUnitTest(unittest.TestCase):
1062     def test_sanitize_filename(self):
1063         acceptable = [
1064             "foo.txt",
1065             ".foo",
1066             "..foo",
1067             "...",
1068             "foo...",
1069             "foo..",
1070             "foo.",
1071             "-",
1072             "\x01\x02\x03",
1073             ]
1074         unacceptable = [
1075             "f\00",
1076             "\00\00",
1077             "/foo",
1078             "foo/",
1079             "//",
1080             ]
1081         for f in acceptable:
1082             self.assertEqual(f, fuse.sanitize_filename(f))
1083         for f in unacceptable:
1084             self.assertNotEqual(f, fuse.sanitize_filename(f))
1085             # The sanitized filename should be the same length, though.
1086             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1087         # Special cases
1088         self.assertEqual("_", fuse.sanitize_filename(""))
1089         self.assertEqual("_", fuse.sanitize_filename("."))
1090         self.assertEqual("__", fuse.sanitize_filename(".."))
1091
1092
1093 class FuseMagicTestPDHOnly(MountTestBase):
1094     def setUp(self, api=None):
1095         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1096
1097         cw = arvados.CollectionWriter()
1098
1099         cw.start_new_file('thing1.txt')
1100         cw.write("data 1")
1101
1102         self.testcollection = cw.finish()
1103         self.test_manifest = cw.manifest_text()
1104         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1105         self.testcollectionuuid = str(created['uuid'])
1106
1107     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1108         if skip_pdh_only is True:
1109             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1110         else:
1111             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1112
1113         mount_ls = llfuse.listdir(self.mounttmp)
1114         self.assertIn('README', mount_ls)
1115         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1116                              arvados.util.uuid_pattern.match(fn)
1117                              for fn in mount_ls),
1118                          "new FUSE MagicDirectory lists Collection")
1119
1120         # look up using pdh should succeed in all cases
1121         self.assertDirContents(self.testcollection, ['thing1.txt'])
1122         self.assertDirContents(os.path.join('by_id', self.testcollection),
1123                                ['thing1.txt'])
1124         mount_ls = llfuse.listdir(self.mounttmp)
1125         self.assertIn('README', mount_ls)
1126         self.assertIn(self.testcollection, mount_ls)
1127         self.assertIn(self.testcollection,
1128                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1129
1130         files = {}
1131         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1132
1133         for k, v in files.items():
1134             with open(os.path.join(self.mounttmp, k)) as f:
1135                 self.assertEqual(v, f.read())
1136
1137         # look up using uuid should fail when pdh_only is set
1138         if pdh_only is True:
1139             with self.assertRaises(OSError):
1140                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1141                                ['thing1.txt'])
1142         else:
1143             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1144                                ['thing1.txt'])
1145
1146     def test_with_pdh_only_true(self):
1147         self.verify_pdh_only(pdh_only=True)
1148
1149     def test_with_pdh_only_false(self):
1150         self.verify_pdh_only(pdh_only=False)
1151
1152     def test_with_default_by_id(self):
1153         self.verify_pdh_only(skip_pdh_only=True)