services/fuse/tests/test_mount.py

   1 import arvados
   2 import arvados.safeapi
   3 import arvados_fuse as fuse
   4 import glob
   5 import json
   6 import llfuse
   7 import os
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tempfile
  12 import threading
  13 import time
  14 import unittest
  15 import logging
  16 import multiprocessing
  17 import run_test_server
  18
  19 logger = logging.getLogger('arvados.arv-mount')
  20
  21 class MountTestBase(unittest.TestCase):
  22     def setUp(self):
  23         # The underlying C implementation of open() makes a fstat() syscall
  24         # with the GIL still held.  When the GETATTR message comes back to
  25         # llfuse (which in these tests is in the same interpreter process) it
  26         # can't acquire the GIL, so it can't service the fstat() call, so it
  27         # deadlocks.  The workaround is to run some of our test code in a
  28         # separate process.  Forturnately the multiprocessing module makes this
  29         # relatively easy.
  30         self.pool = multiprocessing.Pool(1)
  31
  32         self.keeptmp = tempfile.mkdtemp()
  33         os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
  34         self.mounttmp = tempfile.mkdtemp()
  35         run_test_server.run()
  36         run_test_server.authorize_with("admin")
  37         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
  38
  39     def make_mount(self, root_class, **root_kwargs):
  40         self.operations = fuse.Operations(os.getuid(), os.getgid())
  41         self.operations.inodes.add_entry(root_class(
  42             llfuse.ROOT_INODE, self.operations.inodes, self.api, 0, **root_kwargs))
  43         llfuse.init(self.operations, self.mounttmp, [])
  44         threading.Thread(None, llfuse.main).start()
  45         # wait until the driver is finished initializing
  46         self.operations.initlock.wait()
  47         return self.operations.inodes[llfuse.ROOT_INODE]
  48
  49     def tearDown(self):
  50         self.pool.close()
  51         self.pool = None
  52
  53         # llfuse.close is buggy, so use fusermount instead.
  54         #llfuse.close(unmount=True)
  55         count = 0
  56         success = 1
  57         while (count < 9 and success != 0):
  58           success = subprocess.call(["fusermount", "-u", self.mounttmp])
  59           time.sleep(0.5)
  60           count += 1
  61
  62         os.rmdir(self.mounttmp)
  63         shutil.rmtree(self.keeptmp)
  64         run_test_server.reset()
  65
  66     def assertDirContents(self, subdir, expect_content):
  67         path = self.mounttmp
  68         if subdir:
  69             path = os.path.join(path, subdir)
  70         self.assertEqual(sorted(expect_content), sorted(llfuse.listdir(path)))
  71
  72
  73 class FuseMountTest(MountTestBase):
  74     def setUp(self):
  75         super(FuseMountTest, self).setUp()
  76
  77         cw = arvados.CollectionWriter()
  78
  79         cw.start_new_file('thing1.txt')
  80         cw.write("data 1")
  81         cw.start_new_file('thing2.txt')
  82         cw.write("data 2")
  83         cw.start_new_stream('dir1')
  84
  85         cw.start_new_file('thing3.txt')
  86         cw.write("data 3")
  87         cw.start_new_file('thing4.txt')
  88         cw.write("data 4")
  89
  90         cw.start_new_stream('dir2')
  91         cw.start_new_file('thing5.txt')
  92         cw.write("data 5")
  93         cw.start_new_file('thing6.txt')
  94         cw.write("data 6")
  95
  96         cw.start_new_stream('dir2/dir3')
  97         cw.start_new_file('thing7.txt')
  98         cw.write("data 7")
  99
 100         cw.start_new_file('thing8.txt')
 101         cw.write("data 8")
 102
 103         cw.start_new_stream('edgecases')
 104         for f in ":/./../.../-/*/\x01\\/ ".split("/"):
 105             cw.start_new_file(f)
 106             cw.write('x')
 107
 108         for f in ":/../.../-/*/\x01\\/ ".split("/"):
 109             cw.start_new_stream('edgecases/dirs/' + f)
 110             cw.start_new_file('x/x')
 111             cw.write('x')
 112
 113         self.testcollection = cw.finish()
 114         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
 115
 116     def runTest(self):
 117         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
 118
 119         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 120                                       'edgecases', 'dir1', 'dir2'])
 121         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 122         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 123         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 124         self.assertDirContents('edgecases',
 125                                "dirs/:/_/__/.../-/*/\x01\\/ ".split("/"))
 126         self.assertDirContents('edgecases/dirs',
 127                                ":/__/.../-/*/\x01\\/ ".split("/"))
 128
 129         files = {'thing1.txt': 'data 1',
 130                  'thing2.txt': 'data 2',
 131                  'dir1/thing3.txt': 'data 3',
 132                  'dir1/thing4.txt': 'data 4',
 133                  'dir2/thing5.txt': 'data 5',
 134                  'dir2/thing6.txt': 'data 6',
 135                  'dir2/dir3/thing7.txt': 'data 7',
 136                  'dir2/dir3/thing8.txt': 'data 8'}
 137
 138         for k, v in files.items():
 139             with open(os.path.join(self.mounttmp, k)) as f:
 140                 self.assertEqual(v, f.read())
 141
 142
 143 class FuseNoAPITest(MountTestBase):
 144     def setUp(self):
 145         super(FuseNoAPITest, self).setUp()
 146         keep = arvados.keep.KeepClient(local_store=self.keeptmp)
 147         self.file_data = "API-free text\n"
 148         self.file_loc = keep.put(self.file_data)
 149         self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
 150                 self.file_loc, len(self.file_data)))
 151
 152     def runTest(self):
 153         self.make_mount(fuse.MagicDirectory)
 154         self.assertDirContents(self.coll_loc, ['api-free.txt'])
 155         with open(os.path.join(
 156                 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
 157             actual = keep_file.read(-1)
 158         self.assertEqual(self.file_data, actual)
 159
 160
 161 class FuseMagicTest(MountTestBase):
 162     def setUp(self):
 163         super(FuseMagicTest, self).setUp()
 164
 165         cw = arvados.CollectionWriter()
 166
 167         cw.start_new_file('thing1.txt')
 168         cw.write("data 1")
 169
 170         self.testcollection = cw.finish()
 171         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
 172
 173     def runTest(self):
 174         self.make_mount(fuse.MagicDirectory)
 175
 176         mount_ls = llfuse.listdir(self.mounttmp)
 177         self.assertIn('README', mount_ls)
 178         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 179                              arvados.util.uuid_pattern.match(fn)
 180                              for fn in mount_ls),
 181                          "new FUSE MagicDirectory lists Collection")
 182         self.assertDirContents(self.testcollection, ['thing1.txt'])
 183         self.assertDirContents(os.path.join('by_id', self.testcollection),
 184                                ['thing1.txt'])
 185         mount_ls = llfuse.listdir(self.mounttmp)
 186         self.assertIn('README', mount_ls)
 187         self.assertIn(self.testcollection, mount_ls)
 188         self.assertIn(self.testcollection,
 189                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 190
 191         files = {}
 192         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 193
 194         for k, v in files.items():
 195             with open(os.path.join(self.mounttmp, k)) as f:
 196                 self.assertEqual(v, f.read())
 197
 198
 199 class FuseTagsTest(MountTestBase):
 200     def runTest(self):
 201         self.make_mount(fuse.TagsDirectory)
 202
 203         d1 = llfuse.listdir(self.mounttmp)
 204         d1.sort()
 205         self.assertEqual(['foo_tag'], d1)
 206
 207         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 208         d2.sort()
 209         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 210
 211         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 212         d3.sort()
 213         self.assertEqual(['foo'], d3)
 214
 215
 216 class FuseTagsUpdateTest(MountTestBase):
 217     def tag_collection(self, coll_uuid, tag_name):
 218         return self.api.links().create(
 219             body={'link': {'head_uuid': coll_uuid,
 220                            'link_class': 'tag',
 221                            'name': tag_name,
 222         }}).execute()
 223
 224     def runTest(self):
 225         self.make_mount(fuse.TagsDirectory, poll_time=1)
 226
 227         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 228
 229         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 230         self.tag_collection(bar_uuid, 'fuse_test_tag')
 231         time.sleep(1)
 232         self.assertIn('fuse_test_tag', llfuse.listdir(self.mounttmp))
 233         self.assertDirContents('fuse_test_tag', [bar_uuid])
 234
 235         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 236         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 237         time.sleep(1)
 238         self.assertDirContents('fuse_test_tag', [bar_uuid, baz_uuid])
 239
 240         self.api.links().delete(uuid=l['uuid']).execute()
 241         time.sleep(1)
 242         self.assertDirContents('fuse_test_tag', [bar_uuid])
 243
 244
 245 class FuseSharedTest(MountTestBase):
 246     def runTest(self):
 247         self.make_mount(fuse.SharedDirectory,
 248                         exclude=self.api.users().current().execute()['uuid'])
 249
 250         # shared_dirs is a list of the directories exposed
 251         # by fuse.SharedDirectory (i.e. any object visible
 252         # to the current user)
 253         shared_dirs = llfuse.listdir(self.mounttmp)
 254         shared_dirs.sort()
 255         self.assertIn('FUSE User', shared_dirs)
 256
 257         # fuse_user_objs is a list of the objects owned by the FUSE
 258         # test user (which present as files in the 'FUSE User'
 259         # directory)
 260         fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
 261         fuse_user_objs.sort()
 262         self.assertEqual(['FUSE Test Project',                    # project owned by user
 263                           'collection #1 owned by FUSE',          # collection owned by user
 264                           'collection #2 owned by FUSE',          # collection owned by user
 265                           'pipeline instance owned by FUSE.pipelineInstance',  # pipeline instance owned by user
 266                       ], fuse_user_objs)
 267
 268         # test_proj_files is a list of the files in the FUSE Test Project.
 269         test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
 270         test_proj_files.sort()
 271         self.assertEqual(['collection in FUSE project',
 272                           'pipeline instance in FUSE project.pipelineInstance',
 273                           'pipeline template in FUSE project.pipelineTemplate'
 274                       ], test_proj_files)
 275
 276         # Double check that we can open and read objects in this folder as a file,
 277         # and that its contents are what we expect.
 278         pipeline_template_path = os.path.join(
 279                 self.mounttmp,
 280                 'FUSE User',
 281                 'FUSE Test Project',
 282                 'pipeline template in FUSE project.pipelineTemplate')
 283         with open(pipeline_template_path) as f:
 284             j = json.load(f)
 285             self.assertEqual("pipeline template in FUSE project", j['name'])
 286
 287         # check mtime on template
 288         st = os.stat(pipeline_template_path)
 289         self.assertEqual(st.st_mtime, 1397493304)
 290
 291         # check mtime on collection
 292         st = os.stat(os.path.join(
 293                 self.mounttmp,
 294                 'FUSE User',
 295                 'collection #1 owned by FUSE'))
 296         self.assertEqual(st.st_mtime, 1391448174)
 297
 298
 299 class FuseHomeTest(MountTestBase):
 300     def runTest(self):
 301         self.make_mount(fuse.ProjectDirectory,
 302                         project_object=self.api.users().current().execute())
 303
 304         d1 = llfuse.listdir(self.mounttmp)
 305         self.assertIn('Unrestricted public data', d1)
 306
 307         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 308         public_project = run_test_server.fixture('groups')[
 309             'anonymously_accessible_project']
 310         found_in = 0
 311         found_not_in = 0
 312         for name, item in run_test_server.fixture('collections').iteritems():
 313             if 'name' not in item:
 314                 pass
 315             elif item['owner_uuid'] == public_project['uuid']:
 316                 self.assertIn(item['name'], d2)
 317                 found_in += 1
 318             else:
 319                 # Artificial assumption here: there is no public
 320                 # collection fixture with the same name as a
 321                 # non-public collection.
 322                 self.assertNotIn(item['name'], d2)
 323                 found_not_in += 1
 324         self.assertNotEqual(0, found_in)
 325         self.assertNotEqual(0, found_not_in)
 326
 327         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 328         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 329
 330
 331 def fuseModifyFileTestHelper1(mounttmp):
 332     class Test(unittest.TestCase):
 333         def runTest(self):
 334             d1 = llfuse.listdir(mounttmp)
 335             self.assertEqual(["file1.txt"], d1)
 336             with open(os.path.join(mounttmp, "file1.txt")) as f:
 337                 self.assertEqual("blub", f.read())
 338     Test().runTest()
 339
 340 def fuseModifyFileTestHelper2(mounttmp):
 341     class Test(unittest.TestCase):
 342         def runTest(self):
 343             d1 = llfuse.listdir(mounttmp)
 344             self.assertEqual(["file1.txt"], d1)
 345             with open(os.path.join(mounttmp, "file1.txt")) as f:
 346                 self.assertEqual("plnp", f.read())
 347     Test().runTest()
 348
 349 class FuseModifyFileTest(MountTestBase):
 350     def runTest(self):
 351         collection = arvados.collection.Collection(api_client=self.api)
 352         with collection.open("file1.txt", "w") as f:
 353             f.write("blub")
 354
 355         collection.save_new()
 356
 357         m = self.make_mount(fuse.CollectionDirectory)
 358         with llfuse.lock:
 359             m.new_collection(collection.api_response(), collection)
 360
 361         self.pool.apply(fuseModifyFileTestHelper1, (self.mounttmp,))
 362
 363         with collection.open("file1.txt", "w") as f:
 364             f.write("plnp")
 365
 366         self.pool.apply(fuseModifyFileTestHelper2, (self.mounttmp,))
 367
 368
 369 class FuseAddFileToCollectionTest(MountTestBase):
 370     def runTest(self):
 371         collection = arvados.collection.Collection(api_client=self.api)
 372         with collection.open("file1.txt", "w") as f:
 373             f.write("blub")
 374
 375         collection.save_new()
 376
 377         m = self.make_mount(fuse.CollectionDirectory)
 378         with llfuse.lock:
 379             m.new_collection(collection.api_response(), collection)
 380
 381         d1 = llfuse.listdir(self.mounttmp)
 382         self.assertEqual(["file1.txt"], d1)
 383
 384         with collection.open("file2.txt", "w") as f:
 385             f.write("plnp")
 386
 387         d1 = llfuse.listdir(self.mounttmp)
 388         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 389
 390
 391 class FuseRemoveFileFromCollectionTest(MountTestBase):
 392     def runTest(self):
 393         collection = arvados.collection.Collection(api_client=self.api)
 394         with collection.open("file1.txt", "w") as f:
 395             f.write("blub")
 396
 397         with collection.open("file2.txt", "w") as f:
 398             f.write("plnp")
 399
 400         collection.save_new()
 401
 402         m = self.make_mount(fuse.CollectionDirectory)
 403         with llfuse.lock:
 404             m.new_collection(collection.api_response(), collection)
 405
 406         d1 = llfuse.listdir(self.mounttmp)
 407         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 408
 409         collection.remove("file2.txt")
 410
 411         d1 = llfuse.listdir(self.mounttmp)
 412         self.assertEqual(["file1.txt"], d1)
 413
 414
 415 def fuseCreateFileTestHelper(mounttmp):
 416     class Test(unittest.TestCase):
 417         def runTest(self):
 418             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 419                 pass
 420     Test().runTest()
 421
 422 class FuseCreateFileTest(MountTestBase):
 423     def runTest(self):
 424         collection = arvados.collection.Collection(api_client=self.api)
 425         collection.save_new()
 426
 427         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 428         self.assertEqual(collection2["manifest_text"], "")
 429
 430         collection.save_new()
 431
 432         m = self.make_mount(fuse.CollectionDirectory)
 433         with llfuse.lock:
 434             m.new_collection(collection.api_response(), collection)
 435         self.assertTrue(m.writable())
 436
 437         self.assertNotIn("file1.txt", collection)
 438
 439         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 440
 441         self.assertIn("file1.txt", collection)
 442
 443         d1 = llfuse.listdir(self.mounttmp)
 444         self.assertEqual(["file1.txt"], d1)
 445
 446         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 447         self.assertRegexpMatches(collection2["manifest_text"],
 448             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A[a-f0-9]{40}@[a-f0-9]{8} 0:0:file1\.txt$')
 449
 450
 451 def fuseWriteFileTestHelper1(mounttmp):
 452     class Test(unittest.TestCase):
 453         def runTest(self):
 454             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 455                 f.write("Hello world!")
 456     Test().runTest()
 457
 458 def fuseWriteFileTestHelper2(mounttmp):
 459     class Test(unittest.TestCase):
 460         def runTest(self):
 461             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 462                 self.assertEqual(f.read(), "Hello world!")
 463     Test().runTest()
 464
 465 class FuseWriteFileTest(MountTestBase):
 466     def runTest(self):
 467         collection = arvados.collection.Collection(api_client=self.api)
 468         collection.save_new()
 469
 470         m = self.make_mount(fuse.CollectionDirectory)
 471         with llfuse.lock:
 472             m.new_collection(collection.api_response(), collection)
 473         self.assertTrue(m.writable())
 474
 475         self.assertNotIn("file1.txt", collection)
 476
 477         self.pool.apply(fuseWriteFileTestHelper1, (self.mounttmp,))
 478
 479         with collection.open("file1.txt") as f:
 480             self.assertEqual(f.read(), "Hello world!")
 481
 482         self.pool.apply(fuseWriteFileTestHelper2, (self.mounttmp,))
 483
 484         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 485         self.assertRegexpMatches(collection2["manifest_text"],
 486             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
 487
 488
 489 def fuseUpdateFileTestHelper(mounttmp):
 490     class Test(unittest.TestCase):
 491         def runTest(self):
 492             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 493                 f.write("Hello world!")
 494
 495             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 496                 fr = f.read()
 497                 self.assertEqual(fr, "Hello world!")
 498                 f.seek(0)
 499                 f.write("Hola mundo!")
 500                 f.seek(0)
 501                 fr = f.read()
 502                 self.assertEqual(fr, "Hola mundo!!")
 503
 504             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 505                 self.assertEqual(f.read(), "Hola mundo!!")
 506
 507     Test().runTest()
 508
 509 class FuseUpdateFileTest(MountTestBase):
 510     def runTest(self):
 511         collection = arvados.collection.Collection(api_client=self.api)
 512         collection.save_new()
 513
 514         m = self.make_mount(fuse.CollectionDirectory)
 515         with llfuse.lock:
 516             m.new_collection(collection.api_response(), collection)
 517         self.assertTrue(m.writable())
 518
 519         # See note in FuseWriteFileTest
 520         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 521
 522         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 523         self.assertRegexpMatches(collection2["manifest_text"],
 524             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A[a-f0-9]{40}@[a-f0-9]{8} 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:11:file1\.txt 22:1:file1\.txt$')
 525
 526
 527 def fuseMkdirTestHelper(mounttmp):
 528     class Test(unittest.TestCase):
 529         def runTest(self):
 530             with self.assertRaises(IOError):
 531                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 532                     f.write("Hello world!")
 533
 534             os.mkdir(os.path.join(mounttmp, "testdir"))
 535
 536             with self.assertRaises(OSError):
 537                 os.mkdir(os.path.join(mounttmp, "testdir"))
 538
 539             d1 = llfuse.listdir(mounttmp)
 540             self.assertEqual(["testdir"], d1)
 541
 542             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 543                 f.write("Hello world!")
 544
 545             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 546             self.assertEqual(["file1.txt"], d1)
 547
 548     Test().runTest()
 549
 550 class FuseMkdirTest(MountTestBase):
 551     def runTest(self):
 552         collection = arvados.collection.Collection(api_client=self.api)
 553         collection.save_new()
 554
 555         m = self.make_mount(fuse.CollectionDirectory)
 556         with llfuse.lock:
 557             m.new_collection(collection.api_response(), collection)
 558         self.assertTrue(m.writable())
 559
 560         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 561
 562         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 563         self.assertRegexpMatches(collection2["manifest_text"],
 564             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
 565
 566
 567 def fuseRmTestHelper1(mounttmp):
 568     class Test(unittest.TestCase):
 569         def runTest(self):
 570             os.mkdir(os.path.join(mounttmp, "testdir"))
 571
 572             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 573                 f.write("Hello world!")
 574
 575     Test().runTest()
 576
 577 def fuseRmTestHelper2(mounttmp):
 578     class Test(unittest.TestCase):
 579         def runTest(self):
 580             # Can't delete because it's not empty
 581             with self.assertRaises(OSError):
 582                 os.rmdir(os.path.join(mounttmp, "testdir"))
 583
 584             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 585             self.assertEqual(["file1.txt"], d1)
 586
 587             # Delete file
 588             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 589
 590             # Make sure it's empty
 591             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 592             self.assertEqual([], d1)
 593
 594             # Try to delete it again
 595             with self.assertRaises(OSError):
 596                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 597
 598     Test().runTest()
 599
 600 def fuseRmTestHelper3(mounttmp):
 601     class Test(unittest.TestCase):
 602         def runTest(self):
 603             # Should be able to delete now that it is empty
 604             os.rmdir(os.path.join(mounttmp, "testdir"))
 605
 606             # Make sure it's empty
 607             d1 = llfuse.listdir(os.path.join(mounttmp))
 608             self.assertEqual([], d1)
 609
 610             # Try to delete it again
 611             with self.assertRaises(OSError):
 612                 os.rmdir(os.path.join(mounttmp, "testdir"))
 613
 614     Test().runTest()
 615
 616 class FuseRmTest(MountTestBase):
 617     def runTest(self):
 618         collection = arvados.collection.Collection(api_client=self.api)
 619         collection.save_new()
 620
 621         m = self.make_mount(fuse.CollectionDirectory)
 622         with llfuse.lock:
 623             m.new_collection(collection.api_response(), collection)
 624         self.assertTrue(m.writable())
 625
 626         self.pool.apply(fuseRmTestHelper1, (self.mounttmp,))
 627
 628         # Starting manifest
 629         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 630         self.assertRegexpMatches(collection2["manifest_text"],
 631             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
 632         self.pool.apply(fuseRmTestHelper2, (self.mounttmp,))
 633
 634         # Can't have empty directories :-( so manifest will be empty.
 635         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 636         self.assertEqual(collection2["manifest_text"], "")
 637
 638         self.pool.apply(fuseRmTestHelper3, (self.mounttmp,))
 639
 640         # manifest should be empty now.
 641         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 642         self.assertEqual(collection2["manifest_text"], "")
 643
 644
 645 def fuseMvFileTestHelper1(mounttmp):
 646     class Test(unittest.TestCase):
 647         def runTest(self):
 648             os.mkdir(os.path.join(mounttmp, "testdir"))
 649
 650             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 651                 f.write("Hello world!")
 652
 653     Test().runTest()
 654
 655 def fuseMvFileTestHelper2(mounttmp):
 656     class Test(unittest.TestCase):
 657         def runTest(self):
 658             d1 = llfuse.listdir(os.path.join(mounttmp))
 659             self.assertEqual(["testdir"], d1)
 660             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 661             self.assertEqual(["file1.txt"], d1)
 662
 663             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 664
 665             d1 = llfuse.listdir(os.path.join(mounttmp))
 666             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 667             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 668             self.assertEqual([], d1)
 669
 670     Test().runTest()
 671
 672 class FuseMvFileTest(MountTestBase):
 673     def runTest(self):
 674         collection = arvados.collection.Collection(api_client=self.api)
 675         collection.save_new()
 676
 677         m = self.make_mount(fuse.CollectionDirectory)
 678         with llfuse.lock:
 679             m.new_collection(collection.api_response(), collection)
 680         self.assertTrue(m.writable())
 681
 682         self.pool.apply(fuseMvFileTestHelper1, (self.mounttmp,))
 683
 684         # Starting manifest
 685         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 686         self.assertRegexpMatches(collection2["manifest_text"],
 687             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
 688
 689         self.pool.apply(fuseMvFileTestHelper2, (self.mounttmp,))
 690
 691         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 692         self.assertRegexpMatches(collection2["manifest_text"],
 693             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
 694
 695
 696 def fuseRenameTestHelper(mounttmp):
 697     class Test(unittest.TestCase):
 698         def runTest(self):
 699             os.mkdir(os.path.join(mounttmp, "testdir"))
 700
 701             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 702                 f.write("Hello world!")
 703
 704     Test().runTest()
 705
 706 class FuseRenameTest(MountTestBase):
 707     def runTest(self):
 708         collection = arvados.collection.Collection(api_client=self.api)
 709         collection.save_new()
 710
 711         m = self.make_mount(fuse.CollectionDirectory)
 712         with llfuse.lock:
 713             m.new_collection(collection.api_response(), collection)
 714         self.assertTrue(m.writable())
 715
 716         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 717
 718         # Starting manifest
 719         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 720         self.assertRegexpMatches(collection2["manifest_text"],
 721             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
 722
 723         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 724         self.assertEqual(["testdir"], d1)
 725         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 726         self.assertEqual(["file1.txt"], d1)
 727
 728         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 729
 730         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 731         self.assertEqual(["testdir2"], sorted(d1))
 732         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 733         self.assertEqual(["file1.txt"], d1)
 734
 735         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 736         self.assertRegexpMatches(collection2["manifest_text"],
 737             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
 738
 739
 740 class FuseUpdateFromEventTest(MountTestBase):
 741     def runTest(self):
 742         arvados.logger.setLevel(logging.DEBUG)
 743
 744         collection = arvados.collection.Collection(api_client=self.api)
 745         collection.save_new()
 746
 747         m = self.make_mount(fuse.CollectionDirectory)
 748         with llfuse.lock:
 749             m.new_collection(collection.api_response(), collection)
 750
 751         self.operations.listen_for_events(self.api)
 752
 753         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 754         self.assertEqual([], sorted(d1))
 755
 756         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 757             with collection2.open("file1.txt", "w") as f:
 758                 f.write("foo")
 759
 760         time.sleep(1)
 761
 762         # should show up via event bus notify
 763
 764         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 765         self.assertEqual(["file1.txt"], sorted(d1))
 766
 767
 768 def fuseFileConflictTestHelper(mounttmp):
 769     class Test(unittest.TestCase):
 770         def runTest(self):
 771             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 772                 f.write("bar")
 773
 774             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 775             self.assertEqual(len(d1), 2)
 776
 777             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 778                 self.assertEqual(f.read(), "bar")
 779
 780             self.assertRegexpMatches(d1[1],
 781                 r'file1\.txt~conflict-\d\d\d\d-\d\d-\d\d-\d\d:\d\d:\d\d~')
 782
 783             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 784                 self.assertEqual(f.read(), "foo")
 785
 786     Test().runTest()
 787
 788 class FuseFileConflictTest(MountTestBase):
 789     def runTest(self):
 790         collection = arvados.collection.Collection(api_client=self.api)
 791         collection.save_new()
 792
 793         m = self.make_mount(fuse.CollectionDirectory)
 794         with llfuse.lock:
 795             m.new_collection(collection.api_response(), collection)
 796
 797         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 798         self.assertEqual([], sorted(d1))
 799
 800         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 801             with collection2.open("file1.txt", "w") as f:
 802                 f.write("foo")
 803
 804         # See comment in FuseWriteFileTest
 805         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 806
 807
 808 def fuseUnlinkOpenFileTest(mounttmp):
 809     class Test(unittest.TestCase):
 810         def runTest(self):
 811             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 812                 f.write("foo")
 813
 814                 d1 = llfuse.listdir(os.path.join(mounttmp))
 815                 self.assertEqual(["file1.txt"], sorted(d1))
 816
 817                 os.remove(os.path.join(mounttmp, "file1.txt"))
 818
 819                 d1 = llfuse.listdir(os.path.join(mounttmp))
 820                 self.assertEqual([], sorted(d1))
 821
 822                 f.seek(0)
 823                 self.assertEqual(f.read(), "foo")
 824                 f.write("bar")
 825
 826                 f.seek(0)
 827                 self.assertEqual(f.read(), "foobar")
 828
 829     Test().runTest()
 830
 831 class FuseUnlinkOpenFileTest(MountTestBase):
 832     def runTest(self):
 833         collection = arvados.collection.Collection(api_client=self.api)
 834         collection.save_new()
 835
 836         m = self.make_mount(fuse.CollectionDirectory)
 837         with llfuse.lock:
 838             m.new_collection(collection.api_response(), collection)
 839
 840         # See comment in FuseWriteFileTest
 841         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 842
 843         self.assertEqual(collection.manifest_text(), "")
 844
 845
 846 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 847     class Test(unittest.TestCase):
 848         def runTest(self):
 849             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 850                 f.write("Hello world!")
 851
 852             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 853             self.assertEqual(["file1.txt"], sorted(d1))
 854             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 855             self.assertEqual([], sorted(d1))
 856
 857     Test().runTest()
 858
 859 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 860     class Test(unittest.TestCase):
 861         def runTest(self):
 862             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 863
 864             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 865             self.assertEqual([], sorted(d1))
 866             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 867             self.assertEqual(["file2.txt"], sorted(d1))
 868
 869     Test().runTest()
 870
 871 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 872     def runTest(self):
 873         collection1 = arvados.collection.Collection(api_client=self.api)
 874         collection1.save_new()
 875
 876         collection2 = arvados.collection.Collection(api_client=self.api)
 877         collection2.save_new()
 878
 879         m = self.make_mount(fuse.MagicDirectory)
 880
 881         # See comment in FuseWriteFileTest
 882         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 883                                                   collection1.manifest_locator(),
 884                                                   collection2.manifest_locator()))
 885
 886         collection1.update()
 887         collection2.update()
 888
 889         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
 890         self.assertEqual(collection2.manifest_text(), "")
 891
 892         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 893                                                   collection1.manifest_locator(),
 894                                                   collection2.manifest_locator()))
 895
 896         collection1.update()
 897         collection2.update()
 898
 899         self.assertEqual(collection1.manifest_text(), "")
 900         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file2\.txt$")
 901
 902
 903 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 904     class Test(unittest.TestCase):
 905         def runTest(self):
 906             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 907             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 908                 f.write("Hello world!")
 909
 910             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 911             self.assertEqual(["testdir"], sorted(d1))
 912             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 913             self.assertEqual(["file1.txt"], sorted(d1))
 914
 915             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 916             self.assertEqual([], sorted(d1))
 917
 918     Test().runTest()
 919
 920
 921 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 922     class Test(unittest.TestCase):
 923         def runTest(self):
 924             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 925
 926             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 927             self.assertEqual([], sorted(d1))
 928
 929             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 930             self.assertEqual(["testdir2"], sorted(d1))
 931             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 932             self.assertEqual(["file1.txt"], sorted(d1))
 933
 934             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 935                 self.assertEqual(f.read(), "Hello world!")
 936
 937     Test().runTest()
 938
 939 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 940     def runTest(self):
 941         collection1 = arvados.collection.Collection(api_client=self.api)
 942         collection1.save_new()
 943
 944         collection2 = arvados.collection.Collection(api_client=self.api)
 945         collection2.save_new()
 946
 947         m = self.make_mount(fuse.MagicDirectory)
 948
 949         # See comment in FuseWriteFileTest
 950         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 951                                                   collection1.manifest_locator(),
 952                                                   collection2.manifest_locator()))
 953
 954         collection1.update()
 955         collection2.update()
 956
 957         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
 958         self.assertEqual(collection2.manifest_text(), "")
 959
 960         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 961                                                   collection1.manifest_locator(),
 962                                                   collection2.manifest_locator()))
 963
 964         collection1.update()
 965         collection2.update()
 966
 967         self.assertEqual(collection1.manifest_text(), "")
 968         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
 969
 970
 971 def fuseProjectMkdirTestHelper1(mounttmp):
 972     class Test(unittest.TestCase):
 973         def runTest(self):
 974             os.mkdir(os.path.join(mounttmp, "testcollection"))
 975             with self.assertRaises(OSError):
 976                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 977     Test().runTest()
 978
 979 def fuseProjectMkdirTestHelper2(mounttmp):
 980     class Test(unittest.TestCase):
 981         def runTest(self):
 982             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 983                 f.write("Hello world!")
 984             with self.assertRaises(OSError):
 985                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 986             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 987             with self.assertRaises(OSError):
 988                 os.remove(os.path.join(mounttmp, "testcollection"))
 989             os.rmdir(os.path.join(mounttmp, "testcollection"))
 990     Test().runTest()
 991
 992 class FuseProjectMkdirRmdirTest(MountTestBase):
 993     def runTest(self):
 994         self.make_mount(fuse.ProjectDirectory,
 995                         project_object=self.api.users().current().execute())
 996
 997         d1 = llfuse.listdir(self.mounttmp)
 998         self.assertNotIn('testcollection', d1)
 999
1000         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
1001
1002         d1 = llfuse.listdir(self.mounttmp)
1003         self.assertIn('testcollection', d1)
1004
1005         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
1006
1007         d1 = llfuse.listdir(self.mounttmp)
1008         self.assertNotIn('testcollection', d1)
1009
1010
1011 def fuseProjectMvTestHelper1(mounttmp):
1012     class Test(unittest.TestCase):
1013         def runTest(self):
1014             d1 = llfuse.listdir(mounttmp)
1015             self.assertNotIn('testcollection', d1)
1016
1017             os.mkdir(os.path.join(mounttmp, "testcollection"))
1018
1019             d1 = llfuse.listdir(mounttmp)
1020             self.assertIn('testcollection', d1)
1021
1022             with self.assertRaises(OSError):
1023                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1024
1025             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1026
1027             d1 = llfuse.listdir(mounttmp)
1028             self.assertNotIn('testcollection', d1)
1029
1030             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1031             self.assertIn('testcollection', d1)
1032
1033     Test().runTest()
1034
1035 class FuseProjectMvTest(MountTestBase):
1036     def runTest(self):
1037         self.make_mount(fuse.ProjectDirectory,
1038                         project_object=self.api.users().current().execute())
1039
1040         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1041
1042
1043 class FuseUnitTest(unittest.TestCase):
1044     def test_sanitize_filename(self):
1045         acceptable = [
1046             "foo.txt",
1047             ".foo",
1048             "..foo",
1049             "...",
1050             "foo...",
1051             "foo..",
1052             "foo.",
1053             "-",
1054             "\x01\x02\x03",
1055             ]
1056         unacceptable = [
1057             "f\00",
1058             "\00\00",
1059             "/foo",
1060             "foo/",
1061             "//",
1062             ]
1063         for f in acceptable:
1064             self.assertEqual(f, fuse.sanitize_filename(f))
1065         for f in unacceptable:
1066             self.assertNotEqual(f, fuse.sanitize_filename(f))
1067             # The sanitized filename should be the same length, though.
1068             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1069         # Special cases
1070         self.assertEqual("_", fuse.sanitize_filename(""))
1071         self.assertEqual("_", fuse.sanitize_filename("."))
1072         self.assertEqual("__", fuse.sanitize_filename(".."))