services/fuse/tests/test_mount.py

   1 import arvados
   2 import arvados.safeapi
   3 import arvados_fuse as fuse
   4 import glob
   5 import json
   6 import llfuse
   7 import os
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tempfile
  12 import threading
  13 import time
  14 import unittest
  15 import logging
  16 import multiprocessing
  17 import run_test_server
  18
  19 from mount_test_base import MountTestBase
  20
  21 logger = logging.getLogger('arvados.arv-mount')
  22
  23
  24 class FuseMountTest(MountTestBase):
  25     def setUp(self):
  26         super(FuseMountTest, self).setUp()
  27
  28         cw = arvados.CollectionWriter()
  29
  30         cw.start_new_file('thing1.txt')
  31         cw.write("data 1")
  32         cw.start_new_file('thing2.txt')
  33         cw.write("data 2")
  34         cw.start_new_stream('dir1')
  35
  36         cw.start_new_file('thing3.txt')
  37         cw.write("data 3")
  38         cw.start_new_file('thing4.txt')
  39         cw.write("data 4")
  40
  41         cw.start_new_stream('dir2')
  42         cw.start_new_file('thing5.txt')
  43         cw.write("data 5")
  44         cw.start_new_file('thing6.txt')
  45         cw.write("data 6")
  46
  47         cw.start_new_stream('dir2/dir3')
  48         cw.start_new_file('thing7.txt')
  49         cw.write("data 7")
  50
  51         cw.start_new_file('thing8.txt')
  52         cw.write("data 8")
  53
  54         cw.start_new_stream('edgecases')
  55         for f in ":/./../.../-/*/\x01\\/ ".split("/"):
  56             cw.start_new_file(f)
  57             cw.write('x')
  58
  59         for f in ":/../.../-/*/\x01\\/ ".split("/"):
  60             cw.start_new_stream('edgecases/dirs/' + f)
  61             cw.start_new_file('x/x')
  62             cw.write('x')
  63
  64         self.testcollection = cw.finish()
  65         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  66
  67     def runTest(self):
  68         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  69
  70         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
  71                                       'edgecases', 'dir1', 'dir2'])
  72         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
  73         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
  74         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
  75         self.assertDirContents('edgecases',
  76                                "dirs/:/_/__/.../-/*/\x01\\/ ".split("/"))
  77         self.assertDirContents('edgecases/dirs',
  78                                ":/__/.../-/*/\x01\\/ ".split("/"))
  79
  80         files = {'thing1.txt': 'data 1',
  81                  'thing2.txt': 'data 2',
  82                  'dir1/thing3.txt': 'data 3',
  83                  'dir1/thing4.txt': 'data 4',
  84                  'dir2/thing5.txt': 'data 5',
  85                  'dir2/thing6.txt': 'data 6',
  86                  'dir2/dir3/thing7.txt': 'data 7',
  87                  'dir2/dir3/thing8.txt': 'data 8'}
  88
  89         for k, v in files.items():
  90             with open(os.path.join(self.mounttmp, k)) as f:
  91                 self.assertEqual(v, f.read())
  92
  93
  94 class FuseNoAPITest(MountTestBase):
  95     def setUp(self):
  96         super(FuseNoAPITest, self).setUp()
  97         keep = arvados.keep.KeepClient(local_store=self.keeptmp)
  98         self.file_data = "API-free text\n"
  99         self.file_loc = keep.put(self.file_data)
 100         self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
 101                 self.file_loc, len(self.file_data)))
 102
 103     def runTest(self):
 104         self.make_mount(fuse.MagicDirectory)
 105         self.assertDirContents(self.coll_loc, ['api-free.txt'])
 106         with open(os.path.join(
 107                 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
 108             actual = keep_file.read(-1)
 109         self.assertEqual(self.file_data, actual)
 110
 111
 112 class FuseMagicTest(MountTestBase):
 113     def setUp(self):
 114         super(FuseMagicTest, self).setUp()
 115
 116         cw = arvados.CollectionWriter()
 117
 118         cw.start_new_file('thing1.txt')
 119         cw.write("data 1")
 120
 121         self.testcollection = cw.finish()
 122         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
 123
 124     def runTest(self):
 125         self.make_mount(fuse.MagicDirectory)
 126
 127         mount_ls = llfuse.listdir(self.mounttmp)
 128         self.assertIn('README', mount_ls)
 129         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 130                              arvados.util.uuid_pattern.match(fn)
 131                              for fn in mount_ls),
 132                          "new FUSE MagicDirectory lists Collection")
 133         self.assertDirContents(self.testcollection, ['thing1.txt'])
 134         self.assertDirContents(os.path.join('by_id', self.testcollection),
 135                                ['thing1.txt'])
 136         mount_ls = llfuse.listdir(self.mounttmp)
 137         self.assertIn('README', mount_ls)
 138         self.assertIn(self.testcollection, mount_ls)
 139         self.assertIn(self.testcollection,
 140                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 141
 142         files = {}
 143         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 144
 145         for k, v in files.items():
 146             with open(os.path.join(self.mounttmp, k)) as f:
 147                 self.assertEqual(v, f.read())
 148
 149
 150 class FuseTagsTest(MountTestBase):
 151     def runTest(self):
 152         self.make_mount(fuse.TagsDirectory)
 153
 154         d1 = llfuse.listdir(self.mounttmp)
 155         d1.sort()
 156         self.assertEqual(['foo_tag'], d1)
 157
 158         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 159         d2.sort()
 160         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 161
 162         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 163         d3.sort()
 164         self.assertEqual(['foo'], d3)
 165
 166
 167 class FuseTagsUpdateTest(MountTestBase):
 168     def tag_collection(self, coll_uuid, tag_name):
 169         return self.api.links().create(
 170             body={'link': {'head_uuid': coll_uuid,
 171                            'link_class': 'tag',
 172                            'name': tag_name,
 173         }}).execute()
 174
 175     def runTest(self):
 176         self.make_mount(fuse.TagsDirectory, poll_time=1)
 177
 178         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 179
 180         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 181         self.tag_collection(bar_uuid, 'fuse_test_tag')
 182         time.sleep(1)
 183         self.assertIn('fuse_test_tag', llfuse.listdir(self.mounttmp))
 184         self.assertDirContents('fuse_test_tag', [bar_uuid])
 185
 186         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 187         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 188         time.sleep(1)
 189         self.assertDirContents('fuse_test_tag', [bar_uuid, baz_uuid])
 190
 191         self.api.links().delete(uuid=l['uuid']).execute()
 192         time.sleep(1)
 193         self.assertDirContents('fuse_test_tag', [bar_uuid])
 194
 195
 196 class FuseSharedTest(MountTestBase):
 197     def runTest(self):
 198         self.make_mount(fuse.SharedDirectory,
 199                         exclude=self.api.users().current().execute()['uuid'])
 200
 201         # shared_dirs is a list of the directories exposed
 202         # by fuse.SharedDirectory (i.e. any object visible
 203         # to the current user)
 204         shared_dirs = llfuse.listdir(self.mounttmp)
 205         shared_dirs.sort()
 206         self.assertIn('FUSE User', shared_dirs)
 207
 208         # fuse_user_objs is a list of the objects owned by the FUSE
 209         # test user (which present as files in the 'FUSE User'
 210         # directory)
 211         fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
 212         fuse_user_objs.sort()
 213         self.assertEqual(['FUSE Test Project',                    # project owned by user
 214                           'collection #1 owned by FUSE',          # collection owned by user
 215                           'collection #2 owned by FUSE',          # collection owned by user
 216                           'pipeline instance owned by FUSE.pipelineInstance',  # pipeline instance owned by user
 217                       ], fuse_user_objs)
 218
 219         # test_proj_files is a list of the files in the FUSE Test Project.
 220         test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
 221         test_proj_files.sort()
 222         self.assertEqual(['collection in FUSE project',
 223                           'pipeline instance in FUSE project.pipelineInstance',
 224                           'pipeline template in FUSE project.pipelineTemplate'
 225                       ], test_proj_files)
 226
 227         # Double check that we can open and read objects in this folder as a file,
 228         # and that its contents are what we expect.
 229         pipeline_template_path = os.path.join(
 230                 self.mounttmp,
 231                 'FUSE User',
 232                 'FUSE Test Project',
 233                 'pipeline template in FUSE project.pipelineTemplate')
 234         with open(pipeline_template_path) as f:
 235             j = json.load(f)
 236             self.assertEqual("pipeline template in FUSE project", j['name'])
 237
 238         # check mtime on template
 239         st = os.stat(pipeline_template_path)
 240         self.assertEqual(st.st_mtime, 1397493304)
 241
 242         # check mtime on collection
 243         st = os.stat(os.path.join(
 244                 self.mounttmp,
 245                 'FUSE User',
 246                 'collection #1 owned by FUSE'))
 247         self.assertEqual(st.st_mtime, 1391448174)
 248
 249
 250 class FuseHomeTest(MountTestBase):
 251     def runTest(self):
 252         self.make_mount(fuse.ProjectDirectory,
 253                         project_object=self.api.users().current().execute())
 254
 255         d1 = llfuse.listdir(self.mounttmp)
 256         self.assertIn('Unrestricted public data', d1)
 257
 258         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 259         public_project = run_test_server.fixture('groups')[
 260             'anonymously_accessible_project']
 261         found_in = 0
 262         found_not_in = 0
 263         for name, item in run_test_server.fixture('collections').iteritems():
 264             if 'name' not in item:
 265                 pass
 266             elif item['owner_uuid'] == public_project['uuid']:
 267                 self.assertIn(item['name'], d2)
 268                 found_in += 1
 269             else:
 270                 # Artificial assumption here: there is no public
 271                 # collection fixture with the same name as a
 272                 # non-public collection.
 273                 self.assertNotIn(item['name'], d2)
 274                 found_not_in += 1
 275         self.assertNotEqual(0, found_in)
 276         self.assertNotEqual(0, found_not_in)
 277
 278         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 279         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 280
 281
 282 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 283     class Test(unittest.TestCase):
 284         def runTest(self):
 285             d1 = llfuse.listdir(mounttmp)
 286             self.assertEqual(["file1.txt"], d1)
 287             with open(os.path.join(mounttmp, "file1.txt")) as f:
 288                 self.assertEqual("blub", f.read())
 289     Test().runTest()
 290
 291 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 292     class Test(unittest.TestCase):
 293         def runTest(self):
 294             d1 = llfuse.listdir(mounttmp)
 295             self.assertEqual(["file1.txt"], d1)
 296             with open(os.path.join(mounttmp, "file1.txt")) as f:
 297                 self.assertEqual("plnp", f.read())
 298     Test().runTest()
 299
 300 class FuseModifyFileTest(MountTestBase):
 301     def runTest(self):
 302         collection = arvados.collection.Collection(api_client=self.api)
 303         with collection.open("file1.txt", "w") as f:
 304             f.write("blub")
 305
 306         collection.save_new()
 307
 308         m = self.make_mount(fuse.CollectionDirectory)
 309         with llfuse.lock:
 310             m.new_collection(collection.api_response(), collection)
 311
 312         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 313
 314         with collection.open("file1.txt", "w") as f:
 315             f.write("plnp")
 316
 317         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 318
 319
 320 class FuseAddFileToCollectionTest(MountTestBase):
 321     def runTest(self):
 322         collection = arvados.collection.Collection(api_client=self.api)
 323         with collection.open("file1.txt", "w") as f:
 324             f.write("blub")
 325
 326         collection.save_new()
 327
 328         m = self.make_mount(fuse.CollectionDirectory)
 329         with llfuse.lock:
 330             m.new_collection(collection.api_response(), collection)
 331
 332         d1 = llfuse.listdir(self.mounttmp)
 333         self.assertEqual(["file1.txt"], d1)
 334
 335         with collection.open("file2.txt", "w") as f:
 336             f.write("plnp")
 337
 338         d1 = llfuse.listdir(self.mounttmp)
 339         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 340
 341
 342 class FuseRemoveFileFromCollectionTest(MountTestBase):
 343     def runTest(self):
 344         collection = arvados.collection.Collection(api_client=self.api)
 345         with collection.open("file1.txt", "w") as f:
 346             f.write("blub")
 347
 348         with collection.open("file2.txt", "w") as f:
 349             f.write("plnp")
 350
 351         collection.save_new()
 352
 353         m = self.make_mount(fuse.CollectionDirectory)
 354         with llfuse.lock:
 355             m.new_collection(collection.api_response(), collection)
 356
 357         d1 = llfuse.listdir(self.mounttmp)
 358         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 359
 360         collection.remove("file2.txt")
 361
 362         d1 = llfuse.listdir(self.mounttmp)
 363         self.assertEqual(["file1.txt"], d1)
 364
 365
 366 def fuseCreateFileTestHelper(mounttmp):
 367     class Test(unittest.TestCase):
 368         def runTest(self):
 369             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 370                 pass
 371     Test().runTest()
 372
 373 class FuseCreateFileTest(MountTestBase):
 374     def runTest(self):
 375         collection = arvados.collection.Collection(api_client=self.api)
 376         collection.save_new()
 377
 378         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 379         self.assertEqual(collection2["manifest_text"], "")
 380
 381         collection.save_new()
 382
 383         m = self.make_mount(fuse.CollectionDirectory)
 384         with llfuse.lock:
 385             m.new_collection(collection.api_response(), collection)
 386         self.assertTrue(m.writable())
 387
 388         self.assertNotIn("file1.txt", collection)
 389
 390         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 391
 392         self.assertIn("file1.txt", collection)
 393
 394         d1 = llfuse.listdir(self.mounttmp)
 395         self.assertEqual(["file1.txt"], d1)
 396
 397         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 398         self.assertRegexpMatches(collection2["manifest_text"],
 399             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 400
 401
 402 def fuseWriteFileTestHelperWriteFile(mounttmp):
 403     class Test(unittest.TestCase):
 404         def runTest(self):
 405             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 406                 f.write("Hello world!")
 407     Test().runTest()
 408
 409 def fuseWriteFileTestHelperReadFile(mounttmp):
 410     class Test(unittest.TestCase):
 411         def runTest(self):
 412             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 413                 self.assertEqual(f.read(), "Hello world!")
 414     Test().runTest()
 415
 416 class FuseWriteFileTest(MountTestBase):
 417     def runTest(self):
 418         collection = arvados.collection.Collection(api_client=self.api)
 419         collection.save_new()
 420
 421         m = self.make_mount(fuse.CollectionDirectory)
 422         with llfuse.lock:
 423             m.new_collection(collection.api_response(), collection)
 424         self.assertTrue(m.writable())
 425
 426         self.assertNotIn("file1.txt", collection)
 427
 428         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 429
 430         with collection.open("file1.txt") as f:
 431             self.assertEqual(f.read(), "Hello world!")
 432
 433         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 434
 435         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 436         self.assertRegexpMatches(collection2["manifest_text"],
 437             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 438
 439
 440 def fuseUpdateFileTestHelper(mounttmp):
 441     class Test(unittest.TestCase):
 442         def runTest(self):
 443             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 444                 f.write("Hello world!")
 445
 446             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 447                 fr = f.read()
 448                 self.assertEqual(fr, "Hello world!")
 449                 f.seek(0)
 450                 f.write("Hola mundo!")
 451                 f.seek(0)
 452                 fr = f.read()
 453                 self.assertEqual(fr, "Hola mundo!!")
 454
 455             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 456                 self.assertEqual(f.read(), "Hola mundo!!")
 457
 458     Test().runTest()
 459
 460 class FuseUpdateFileTest(MountTestBase):
 461     def runTest(self):
 462         collection = arvados.collection.Collection(api_client=self.api)
 463         collection.save_new()
 464
 465         m = self.make_mount(fuse.CollectionDirectory)
 466         with llfuse.lock:
 467             m.new_collection(collection.api_response(), collection)
 468         self.assertTrue(m.writable())
 469
 470         # See note in MountTestBase.setUp
 471         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 472
 473         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 474         self.assertRegexpMatches(collection2["manifest_text"],
 475             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 476
 477
 478 def fuseMkdirTestHelper(mounttmp):
 479     class Test(unittest.TestCase):
 480         def runTest(self):
 481             with self.assertRaises(IOError):
 482                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 483                     f.write("Hello world!")
 484
 485             os.mkdir(os.path.join(mounttmp, "testdir"))
 486
 487             with self.assertRaises(OSError):
 488                 os.mkdir(os.path.join(mounttmp, "testdir"))
 489
 490             d1 = llfuse.listdir(mounttmp)
 491             self.assertEqual(["testdir"], d1)
 492
 493             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 494                 f.write("Hello world!")
 495
 496             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 497             self.assertEqual(["file1.txt"], d1)
 498
 499     Test().runTest()
 500
 501 class FuseMkdirTest(MountTestBase):
 502     def runTest(self):
 503         collection = arvados.collection.Collection(api_client=self.api)
 504         collection.save_new()
 505
 506         m = self.make_mount(fuse.CollectionDirectory)
 507         with llfuse.lock:
 508             m.new_collection(collection.api_response(), collection)
 509         self.assertTrue(m.writable())
 510
 511         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 512
 513         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 514         self.assertRegexpMatches(collection2["manifest_text"],
 515             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 516
 517
 518 def fuseRmTestHelperWriteFile(mounttmp):
 519     class Test(unittest.TestCase):
 520         def runTest(self):
 521             os.mkdir(os.path.join(mounttmp, "testdir"))
 522
 523             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 524                 f.write("Hello world!")
 525
 526     Test().runTest()
 527
 528 def fuseRmTestHelperDeleteFile(mounttmp):
 529     class Test(unittest.TestCase):
 530         def runTest(self):
 531             # Can't delete because it's not empty
 532             with self.assertRaises(OSError):
 533                 os.rmdir(os.path.join(mounttmp, "testdir"))
 534
 535             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 536             self.assertEqual(["file1.txt"], d1)
 537
 538             # Delete file
 539             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 540
 541             # Make sure it's empty
 542             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 543             self.assertEqual([], d1)
 544
 545             # Try to delete it again
 546             with self.assertRaises(OSError):
 547                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 548
 549     Test().runTest()
 550
 551 def fuseRmTestHelperRmdir(mounttmp):
 552     class Test(unittest.TestCase):
 553         def runTest(self):
 554             # Should be able to delete now that it is empty
 555             os.rmdir(os.path.join(mounttmp, "testdir"))
 556
 557             # Make sure it's empty
 558             d1 = llfuse.listdir(os.path.join(mounttmp))
 559             self.assertEqual([], d1)
 560
 561             # Try to delete it again
 562             with self.assertRaises(OSError):
 563                 os.rmdir(os.path.join(mounttmp, "testdir"))
 564
 565     Test().runTest()
 566
 567 class FuseRmTest(MountTestBase):
 568     def runTest(self):
 569         collection = arvados.collection.Collection(api_client=self.api)
 570         collection.save_new()
 571
 572         m = self.make_mount(fuse.CollectionDirectory)
 573         with llfuse.lock:
 574             m.new_collection(collection.api_response(), collection)
 575         self.assertTrue(m.writable())
 576
 577         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 578
 579         # Starting manifest
 580         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 581         self.assertRegexpMatches(collection2["manifest_text"],
 582             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 583         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 584
 585         # Can't have empty directories :-( so manifest will be empty.
 586         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 587         self.assertEqual(collection2["manifest_text"], "")
 588
 589         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 590
 591         # manifest should be empty now.
 592         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 593         self.assertEqual(collection2["manifest_text"], "")
 594
 595
 596 def fuseMvFileTestHelperWriteFile(mounttmp):
 597     class Test(unittest.TestCase):
 598         def runTest(self):
 599             os.mkdir(os.path.join(mounttmp, "testdir"))
 600
 601             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 602                 f.write("Hello world!")
 603
 604     Test().runTest()
 605
 606 def fuseMvFileTestHelperMoveFile(mounttmp):
 607     class Test(unittest.TestCase):
 608         def runTest(self):
 609             d1 = llfuse.listdir(os.path.join(mounttmp))
 610             self.assertEqual(["testdir"], d1)
 611             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 612             self.assertEqual(["file1.txt"], d1)
 613
 614             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 615
 616             d1 = llfuse.listdir(os.path.join(mounttmp))
 617             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 618             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 619             self.assertEqual([], d1)
 620
 621     Test().runTest()
 622
 623 class FuseMvFileTest(MountTestBase):
 624     def runTest(self):
 625         collection = arvados.collection.Collection(api_client=self.api)
 626         collection.save_new()
 627
 628         m = self.make_mount(fuse.CollectionDirectory)
 629         with llfuse.lock:
 630             m.new_collection(collection.api_response(), collection)
 631         self.assertTrue(m.writable())
 632
 633         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 634
 635         # Starting manifest
 636         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 637         self.assertRegexpMatches(collection2["manifest_text"],
 638             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 639
 640         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 641
 642         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 643         self.assertRegexpMatches(collection2["manifest_text"],
 644             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 645
 646
 647 def fuseRenameTestHelper(mounttmp):
 648     class Test(unittest.TestCase):
 649         def runTest(self):
 650             os.mkdir(os.path.join(mounttmp, "testdir"))
 651
 652             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 653                 f.write("Hello world!")
 654
 655     Test().runTest()
 656
 657 class FuseRenameTest(MountTestBase):
 658     def runTest(self):
 659         collection = arvados.collection.Collection(api_client=self.api)
 660         collection.save_new()
 661
 662         m = self.make_mount(fuse.CollectionDirectory)
 663         with llfuse.lock:
 664             m.new_collection(collection.api_response(), collection)
 665         self.assertTrue(m.writable())
 666
 667         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 668
 669         # Starting manifest
 670         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 671         self.assertRegexpMatches(collection2["manifest_text"],
 672             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 673
 674         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 675         self.assertEqual(["testdir"], d1)
 676         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 677         self.assertEqual(["file1.txt"], d1)
 678
 679         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 680
 681         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 682         self.assertEqual(["testdir2"], sorted(d1))
 683         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 684         self.assertEqual(["file1.txt"], d1)
 685
 686         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 687         self.assertRegexpMatches(collection2["manifest_text"],
 688             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 689
 690
 691 class FuseUpdateFromEventTest(MountTestBase):
 692     def runTest(self):
 693         collection = arvados.collection.Collection(api_client=self.api)
 694         collection.save_new()
 695
 696         m = self.make_mount(fuse.CollectionDirectory)
 697         with llfuse.lock:
 698             m.new_collection(collection.api_response(), collection)
 699
 700         self.operations.listen_for_events(self.api)
 701
 702         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 703         self.assertEqual([], sorted(d1))
 704
 705         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 706             with collection2.open("file1.txt", "w") as f:
 707                 f.write("foo")
 708
 709         time.sleep(1)
 710
 711         # should show up via event bus notify
 712
 713         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 714         self.assertEqual(["file1.txt"], sorted(d1))
 715
 716
 717 def fuseFileConflictTestHelper(mounttmp):
 718     class Test(unittest.TestCase):
 719         def runTest(self):
 720             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 721                 f.write("bar")
 722
 723             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 724             self.assertEqual(len(d1), 2)
 725
 726             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 727                 self.assertEqual(f.read(), "bar")
 728
 729             self.assertRegexpMatches(d1[1],
 730                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 731
 732             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 733                 self.assertEqual(f.read(), "foo")
 734
 735     Test().runTest()
 736
 737 class FuseFileConflictTest(MountTestBase):
 738     def runTest(self):
 739         collection = arvados.collection.Collection(api_client=self.api)
 740         collection.save_new()
 741
 742         m = self.make_mount(fuse.CollectionDirectory)
 743         with llfuse.lock:
 744             m.new_collection(collection.api_response(), collection)
 745
 746         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 747         self.assertEqual([], sorted(d1))
 748
 749         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 750             with collection2.open("file1.txt", "w") as f:
 751                 f.write("foo")
 752
 753         # See note in MountTestBase.setUp
 754         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 755
 756
 757 def fuseUnlinkOpenFileTest(mounttmp):
 758     class Test(unittest.TestCase):
 759         def runTest(self):
 760             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 761                 f.write("foo")
 762
 763                 d1 = llfuse.listdir(os.path.join(mounttmp))
 764                 self.assertEqual(["file1.txt"], sorted(d1))
 765
 766                 os.remove(os.path.join(mounttmp, "file1.txt"))
 767
 768                 d1 = llfuse.listdir(os.path.join(mounttmp))
 769                 self.assertEqual([], sorted(d1))
 770
 771                 f.seek(0)
 772                 self.assertEqual(f.read(), "foo")
 773                 f.write("bar")
 774
 775                 f.seek(0)
 776                 self.assertEqual(f.read(), "foobar")
 777
 778     Test().runTest()
 779
 780 class FuseUnlinkOpenFileTest(MountTestBase):
 781     def runTest(self):
 782         collection = arvados.collection.Collection(api_client=self.api)
 783         collection.save_new()
 784
 785         m = self.make_mount(fuse.CollectionDirectory)
 786         with llfuse.lock:
 787             m.new_collection(collection.api_response(), collection)
 788
 789         # See note in MountTestBase.setUp
 790         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 791
 792         self.assertEqual(collection.manifest_text(), "")
 793
 794
 795 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 796     class Test(unittest.TestCase):
 797         def runTest(self):
 798             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 799                 f.write("Hello world!")
 800
 801             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 802             self.assertEqual(["file1.txt"], sorted(d1))
 803             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 804             self.assertEqual([], sorted(d1))
 805
 806     Test().runTest()
 807
 808 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 809     class Test(unittest.TestCase):
 810         def runTest(self):
 811             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 812
 813             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 814             self.assertEqual([], sorted(d1))
 815             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 816             self.assertEqual(["file2.txt"], sorted(d1))
 817
 818     Test().runTest()
 819
 820 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 821     def runTest(self):
 822         collection1 = arvados.collection.Collection(api_client=self.api)
 823         collection1.save_new()
 824
 825         collection2 = arvados.collection.Collection(api_client=self.api)
 826         collection2.save_new()
 827
 828         m = self.make_mount(fuse.MagicDirectory)
 829
 830         # See note in MountTestBase.setUp
 831         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 832                                                   collection1.manifest_locator(),
 833                                                   collection2.manifest_locator()))
 834
 835         collection1.update()
 836         collection2.update()
 837
 838         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 839         self.assertEqual(collection2.manifest_text(), "")
 840
 841         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 842                                                   collection1.manifest_locator(),
 843                                                   collection2.manifest_locator()))
 844
 845         collection1.update()
 846         collection2.update()
 847
 848         self.assertEqual(collection1.manifest_text(), "")
 849         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 850
 851         collection1.stop_threads()
 852         collection2.stop_threads()
 853
 854
 855 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 856     class Test(unittest.TestCase):
 857         def runTest(self):
 858             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 859             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 860                 f.write("Hello world!")
 861
 862             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 863             self.assertEqual(["testdir"], sorted(d1))
 864             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 865             self.assertEqual(["file1.txt"], sorted(d1))
 866
 867             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 868             self.assertEqual([], sorted(d1))
 869
 870     Test().runTest()
 871
 872
 873 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 874     class Test(unittest.TestCase):
 875         def runTest(self):
 876             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 877
 878             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 879             self.assertEqual([], sorted(d1))
 880
 881             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 882             self.assertEqual(["testdir2"], sorted(d1))
 883             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 884             self.assertEqual(["file1.txt"], sorted(d1))
 885
 886             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 887                 self.assertEqual(f.read(), "Hello world!")
 888
 889     Test().runTest()
 890
 891 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 892     def runTest(self):
 893         collection1 = arvados.collection.Collection(api_client=self.api)
 894         collection1.save_new()
 895
 896         collection2 = arvados.collection.Collection(api_client=self.api)
 897         collection2.save_new()
 898
 899         m = self.make_mount(fuse.MagicDirectory)
 900
 901         # See note in MountTestBase.setUp
 902         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 903                                                   collection1.manifest_locator(),
 904                                                   collection2.manifest_locator()))
 905
 906         collection1.update()
 907         collection2.update()
 908
 909         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 910         self.assertEqual(collection2.manifest_text(), "")
 911
 912         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 913                                                   collection1.manifest_locator(),
 914                                                   collection2.manifest_locator()))
 915
 916         collection1.update()
 917         collection2.update()
 918
 919         self.assertEqual(collection1.manifest_text(), "")
 920         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 921
 922         collection1.stop_threads()
 923         collection2.stop_threads()
 924
 925 def fuseProjectMkdirTestHelper1(mounttmp):
 926     class Test(unittest.TestCase):
 927         def runTest(self):
 928             os.mkdir(os.path.join(mounttmp, "testcollection"))
 929             with self.assertRaises(OSError):
 930                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 931     Test().runTest()
 932
 933 def fuseProjectMkdirTestHelper2(mounttmp):
 934     class Test(unittest.TestCase):
 935         def runTest(self):
 936             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 937                 f.write("Hello world!")
 938             with self.assertRaises(OSError):
 939                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 940             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 941             with self.assertRaises(OSError):
 942                 os.remove(os.path.join(mounttmp, "testcollection"))
 943             os.rmdir(os.path.join(mounttmp, "testcollection"))
 944     Test().runTest()
 945
 946 class FuseProjectMkdirRmdirTest(MountTestBase):
 947     def runTest(self):
 948         self.make_mount(fuse.ProjectDirectory,
 949                         project_object=self.api.users().current().execute())
 950
 951         d1 = llfuse.listdir(self.mounttmp)
 952         self.assertNotIn('testcollection', d1)
 953
 954         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
 955
 956         d1 = llfuse.listdir(self.mounttmp)
 957         self.assertIn('testcollection', d1)
 958
 959         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
 960
 961         d1 = llfuse.listdir(self.mounttmp)
 962         self.assertNotIn('testcollection', d1)
 963
 964
 965 def fuseProjectMvTestHelper1(mounttmp):
 966     class Test(unittest.TestCase):
 967         def runTest(self):
 968             d1 = llfuse.listdir(mounttmp)
 969             self.assertNotIn('testcollection', d1)
 970
 971             os.mkdir(os.path.join(mounttmp, "testcollection"))
 972
 973             d1 = llfuse.listdir(mounttmp)
 974             self.assertIn('testcollection', d1)
 975
 976             with self.assertRaises(OSError):
 977                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
 978
 979             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
 980
 981             d1 = llfuse.listdir(mounttmp)
 982             self.assertNotIn('testcollection', d1)
 983
 984             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
 985             self.assertIn('testcollection', d1)
 986
 987     Test().runTest()
 988
 989 class FuseProjectMvTest(MountTestBase):
 990     def runTest(self):
 991         self.make_mount(fuse.ProjectDirectory,
 992                         project_object=self.api.users().current().execute())
 993
 994         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
 995
 996
 997 class FuseUnitTest(unittest.TestCase):
 998     def test_sanitize_filename(self):
 999         acceptable = [
1000             "foo.txt",
1001             ".foo",
1002             "..foo",
1003             "...",
1004             "foo...",
1005             "foo..",
1006             "foo.",
1007             "-",
1008             "\x01\x02\x03",
1009             ]
1010         unacceptable = [
1011             "f\00",
1012             "\00\00",
1013             "/foo",
1014             "foo/",
1015             "//",
1016             ]
1017         for f in acceptable:
1018             self.assertEqual(f, fuse.sanitize_filename(f))
1019         for f in unacceptable:
1020             self.assertNotEqual(f, fuse.sanitize_filename(f))
1021             # The sanitized filename should be the same length, though.
1022             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1023         # Special cases
1024         self.assertEqual("_", fuse.sanitize_filename(""))
1025         self.assertEqual("_", fuse.sanitize_filename("."))
1026         self.assertEqual("__", fuse.sanitize_filename(".."))