services/fuse/tests/test_mount.py

   1 import arvados
   2 import arvados.safeapi
   3 import arvados_fuse as fuse
   4 import glob
   5 import json
   6 import llfuse
   7 import os
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tempfile
  12 import threading
  13 import time
  14 import unittest
  15 import logging
  16 import multiprocessing
  17 import run_test_server
  18 import mock
  19
  20 from mount_test_base import MountTestBase
  21
  22 logger = logging.getLogger('arvados.arv-mount')
  23
  24
  25 class FuseMountTest(MountTestBase):
  26     def setUp(self):
  27         super(FuseMountTest, self).setUp()
  28
  29         cw = arvados.CollectionWriter()
  30
  31         cw.start_new_file('thing1.txt')
  32         cw.write("data 1")
  33         cw.start_new_file('thing2.txt')
  34         cw.write("data 2")
  35         cw.start_new_stream('dir1')
  36
  37         cw.start_new_file('thing3.txt')
  38         cw.write("data 3")
  39         cw.start_new_file('thing4.txt')
  40         cw.write("data 4")
  41
  42         cw.start_new_stream('dir2')
  43         cw.start_new_file('thing5.txt')
  44         cw.write("data 5")
  45         cw.start_new_file('thing6.txt')
  46         cw.write("data 6")
  47
  48         cw.start_new_stream('dir2/dir3')
  49         cw.start_new_file('thing7.txt')
  50         cw.write("data 7")
  51
  52         cw.start_new_file('thing8.txt')
  53         cw.write("data 8")
  54
  55         cw.start_new_stream('edgecases')
  56         for f in ":/./../.../-/*/\x01\\/ ".split("/"):
  57             cw.start_new_file(f)
  58             cw.write('x')
  59
  60         for f in ":/../.../-/*/\x01\\/ ".split("/"):
  61             cw.start_new_stream('edgecases/dirs/' + f)
  62             cw.start_new_file('x/x')
  63             cw.write('x')
  64
  65         self.testcollection = cw.finish()
  66         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  67
  68     def runTest(self):
  69         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  70
  71         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
  72                                       'edgecases', 'dir1', 'dir2'])
  73         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
  74         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
  75         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
  76         self.assertDirContents('edgecases',
  77                                "dirs/:/_/__/.../-/*/\x01\\/ ".split("/"))
  78         self.assertDirContents('edgecases/dirs',
  79                                ":/__/.../-/*/\x01\\/ ".split("/"))
  80
  81         files = {'thing1.txt': 'data 1',
  82                  'thing2.txt': 'data 2',
  83                  'dir1/thing3.txt': 'data 3',
  84                  'dir1/thing4.txt': 'data 4',
  85                  'dir2/thing5.txt': 'data 5',
  86                  'dir2/thing6.txt': 'data 6',
  87                  'dir2/dir3/thing7.txt': 'data 7',
  88                  'dir2/dir3/thing8.txt': 'data 8'}
  89
  90         for k, v in files.items():
  91             with open(os.path.join(self.mounttmp, k)) as f:
  92                 self.assertEqual(v, f.read())
  93
  94
  95 class FuseNoAPITest(MountTestBase):
  96     def setUp(self):
  97         super(FuseNoAPITest, self).setUp()
  98         keep = arvados.keep.KeepClient(local_store=self.keeptmp)
  99         self.file_data = "API-free text\n"
 100         self.file_loc = keep.put(self.file_data)
 101         self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
 102                 self.file_loc, len(self.file_data)))
 103
 104     def runTest(self):
 105         self.make_mount(fuse.MagicDirectory)
 106         self.assertDirContents(self.coll_loc, ['api-free.txt'])
 107         with open(os.path.join(
 108                 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
 109             actual = keep_file.read(-1)
 110         self.assertEqual(self.file_data, actual)
 111
 112
 113 class FuseMagicTest(MountTestBase):
 114     def setUp(self, api=None):
 115         super(FuseMagicTest, self).setUp(api=api)
 116
 117         cw = arvados.CollectionWriter()
 118
 119         cw.start_new_file('thing1.txt')
 120         cw.write("data 1")
 121
 122         self.testcollection = cw.finish()
 123         self.test_manifest = cw.manifest_text()
 124         self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 125
 126     def runTest(self):
 127         self.make_mount(fuse.MagicDirectory)
 128
 129         mount_ls = llfuse.listdir(self.mounttmp)
 130         self.assertIn('README', mount_ls)
 131         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 132                              arvados.util.uuid_pattern.match(fn)
 133                              for fn in mount_ls),
 134                          "new FUSE MagicDirectory lists Collection")
 135         self.assertDirContents(self.testcollection, ['thing1.txt'])
 136         self.assertDirContents(os.path.join('by_id', self.testcollection),
 137                                ['thing1.txt'])
 138         mount_ls = llfuse.listdir(self.mounttmp)
 139         self.assertIn('README', mount_ls)
 140         self.assertIn(self.testcollection, mount_ls)
 141         self.assertIn(self.testcollection,
 142                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 143
 144         files = {}
 145         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 146
 147         for k, v in files.items():
 148             with open(os.path.join(self.mounttmp, k)) as f:
 149                 self.assertEqual(v, f.read())
 150
 151
 152 class FuseTagsTest(MountTestBase):
 153     def runTest(self):
 154         self.make_mount(fuse.TagsDirectory)
 155
 156         d1 = llfuse.listdir(self.mounttmp)
 157         d1.sort()
 158         self.assertEqual(['foo_tag'], d1)
 159
 160         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 161         d2.sort()
 162         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 163
 164         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 165         d3.sort()
 166         self.assertEqual(['foo'], d3)
 167
 168
 169 class FuseTagsUpdateTest(MountTestBase):
 170     def tag_collection(self, coll_uuid, tag_name):
 171         return self.api.links().create(
 172             body={'link': {'head_uuid': coll_uuid,
 173                            'link_class': 'tag',
 174                            'name': tag_name,
 175         }}).execute()
 176
 177     def runTest(self):
 178         self.make_mount(fuse.TagsDirectory, poll_time=1)
 179
 180         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 181
 182         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 183         self.tag_collection(bar_uuid, 'fuse_test_tag')
 184         time.sleep(1)
 185         self.assertIn('fuse_test_tag', llfuse.listdir(self.mounttmp))
 186         self.assertDirContents('fuse_test_tag', [bar_uuid])
 187
 188         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 189         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 190         time.sleep(1)
 191         self.assertDirContents('fuse_test_tag', [bar_uuid, baz_uuid])
 192
 193         self.api.links().delete(uuid=l['uuid']).execute()
 194         time.sleep(1)
 195         self.assertDirContents('fuse_test_tag', [bar_uuid])
 196
 197
 198 class FuseSharedTest(MountTestBase):
 199     def runTest(self):
 200         self.make_mount(fuse.SharedDirectory,
 201                         exclude=self.api.users().current().execute()['uuid'])
 202
 203         # shared_dirs is a list of the directories exposed
 204         # by fuse.SharedDirectory (i.e. any object visible
 205         # to the current user)
 206         shared_dirs = llfuse.listdir(self.mounttmp)
 207         shared_dirs.sort()
 208         self.assertIn('FUSE User', shared_dirs)
 209
 210         # fuse_user_objs is a list of the objects owned by the FUSE
 211         # test user (which present as files in the 'FUSE User'
 212         # directory)
 213         fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
 214         fuse_user_objs.sort()
 215         self.assertEqual(['FUSE Test Project',                    # project owned by user
 216                           'collection #1 owned by FUSE',          # collection owned by user
 217                           'collection #2 owned by FUSE',          # collection owned by user
 218                           'pipeline instance owned by FUSE.pipelineInstance',  # pipeline instance owned by user
 219                       ], fuse_user_objs)
 220
 221         # test_proj_files is a list of the files in the FUSE Test Project.
 222         test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
 223         test_proj_files.sort()
 224         self.assertEqual(['collection in FUSE project',
 225                           'pipeline instance in FUSE project.pipelineInstance',
 226                           'pipeline template in FUSE project.pipelineTemplate'
 227                       ], test_proj_files)
 228
 229         # Double check that we can open and read objects in this folder as a file,
 230         # and that its contents are what we expect.
 231         pipeline_template_path = os.path.join(
 232                 self.mounttmp,
 233                 'FUSE User',
 234                 'FUSE Test Project',
 235                 'pipeline template in FUSE project.pipelineTemplate')
 236         with open(pipeline_template_path) as f:
 237             j = json.load(f)
 238             self.assertEqual("pipeline template in FUSE project", j['name'])
 239
 240         # check mtime on template
 241         st = os.stat(pipeline_template_path)
 242         self.assertEqual(st.st_mtime, 1397493304)
 243
 244         # check mtime on collection
 245         st = os.stat(os.path.join(
 246                 self.mounttmp,
 247                 'FUSE User',
 248                 'collection #1 owned by FUSE'))
 249         self.assertEqual(st.st_mtime, 1391448174)
 250
 251
 252 class FuseHomeTest(MountTestBase):
 253     def runTest(self):
 254         self.make_mount(fuse.ProjectDirectory,
 255                         project_object=self.api.users().current().execute())
 256
 257         d1 = llfuse.listdir(self.mounttmp)
 258         self.assertIn('Unrestricted public data', d1)
 259
 260         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 261         public_project = run_test_server.fixture('groups')[
 262             'anonymously_accessible_project']
 263         found_in = 0
 264         found_not_in = 0
 265         for name, item in run_test_server.fixture('collections').iteritems():
 266             if 'name' not in item:
 267                 pass
 268             elif item['owner_uuid'] == public_project['uuid']:
 269                 self.assertIn(item['name'], d2)
 270                 found_in += 1
 271             else:
 272                 # Artificial assumption here: there is no public
 273                 # collection fixture with the same name as a
 274                 # non-public collection.
 275                 self.assertNotIn(item['name'], d2)
 276                 found_not_in += 1
 277         self.assertNotEqual(0, found_in)
 278         self.assertNotEqual(0, found_not_in)
 279
 280         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 281         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 282
 283
 284 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 285     class Test(unittest.TestCase):
 286         def runTest(self):
 287             d1 = llfuse.listdir(mounttmp)
 288             self.assertEqual(["file1.txt"], d1)
 289             with open(os.path.join(mounttmp, "file1.txt")) as f:
 290                 self.assertEqual("blub", f.read())
 291     Test().runTest()
 292
 293 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 294     class Test(unittest.TestCase):
 295         def runTest(self):
 296             d1 = llfuse.listdir(mounttmp)
 297             self.assertEqual(["file1.txt"], d1)
 298             with open(os.path.join(mounttmp, "file1.txt")) as f:
 299                 self.assertEqual("plnp", f.read())
 300     Test().runTest()
 301
 302 class FuseModifyFileTest(MountTestBase):
 303     def runTest(self):
 304         collection = arvados.collection.Collection(api_client=self.api)
 305         with collection.open("file1.txt", "w") as f:
 306             f.write("blub")
 307
 308         collection.save_new()
 309
 310         m = self.make_mount(fuse.CollectionDirectory)
 311         with llfuse.lock:
 312             m.new_collection(collection.api_response(), collection)
 313
 314         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 315
 316         with collection.open("file1.txt", "w") as f:
 317             f.write("plnp")
 318
 319         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 320
 321
 322 class FuseAddFileToCollectionTest(MountTestBase):
 323     def runTest(self):
 324         collection = arvados.collection.Collection(api_client=self.api)
 325         with collection.open("file1.txt", "w") as f:
 326             f.write("blub")
 327
 328         collection.save_new()
 329
 330         m = self.make_mount(fuse.CollectionDirectory)
 331         with llfuse.lock:
 332             m.new_collection(collection.api_response(), collection)
 333
 334         d1 = llfuse.listdir(self.mounttmp)
 335         self.assertEqual(["file1.txt"], d1)
 336
 337         with collection.open("file2.txt", "w") as f:
 338             f.write("plnp")
 339
 340         d1 = llfuse.listdir(self.mounttmp)
 341         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 342
 343
 344 class FuseRemoveFileFromCollectionTest(MountTestBase):
 345     def runTest(self):
 346         collection = arvados.collection.Collection(api_client=self.api)
 347         with collection.open("file1.txt", "w") as f:
 348             f.write("blub")
 349
 350         with collection.open("file2.txt", "w") as f:
 351             f.write("plnp")
 352
 353         collection.save_new()
 354
 355         m = self.make_mount(fuse.CollectionDirectory)
 356         with llfuse.lock:
 357             m.new_collection(collection.api_response(), collection)
 358
 359         d1 = llfuse.listdir(self.mounttmp)
 360         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 361
 362         collection.remove("file2.txt")
 363
 364         d1 = llfuse.listdir(self.mounttmp)
 365         self.assertEqual(["file1.txt"], d1)
 366
 367
 368 def fuseCreateFileTestHelper(mounttmp):
 369     class Test(unittest.TestCase):
 370         def runTest(self):
 371             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 372                 pass
 373     Test().runTest()
 374
 375 class FuseCreateFileTest(MountTestBase):
 376     def runTest(self):
 377         collection = arvados.collection.Collection(api_client=self.api)
 378         collection.save_new()
 379
 380         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 381         self.assertEqual(collection2["manifest_text"], "")
 382
 383         collection.save_new()
 384
 385         m = self.make_mount(fuse.CollectionDirectory)
 386         with llfuse.lock:
 387             m.new_collection(collection.api_response(), collection)
 388         self.assertTrue(m.writable())
 389
 390         self.assertNotIn("file1.txt", collection)
 391
 392         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 393
 394         self.assertIn("file1.txt", collection)
 395
 396         d1 = llfuse.listdir(self.mounttmp)
 397         self.assertEqual(["file1.txt"], d1)
 398
 399         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 400         self.assertRegexpMatches(collection2["manifest_text"],
 401             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 402
 403
 404 def fuseWriteFileTestHelperWriteFile(mounttmp):
 405     class Test(unittest.TestCase):
 406         def runTest(self):
 407             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 408                 f.write("Hello world!")
 409     Test().runTest()
 410
 411 def fuseWriteFileTestHelperReadFile(mounttmp):
 412     class Test(unittest.TestCase):
 413         def runTest(self):
 414             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 415                 self.assertEqual(f.read(), "Hello world!")
 416     Test().runTest()
 417
 418 class FuseWriteFileTest(MountTestBase):
 419     def runTest(self):
 420         collection = arvados.collection.Collection(api_client=self.api)
 421         collection.save_new()
 422
 423         m = self.make_mount(fuse.CollectionDirectory)
 424         with llfuse.lock:
 425             m.new_collection(collection.api_response(), collection)
 426         self.assertTrue(m.writable())
 427
 428         self.assertNotIn("file1.txt", collection)
 429
 430         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 431
 432         with collection.open("file1.txt") as f:
 433             self.assertEqual(f.read(), "Hello world!")
 434
 435         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 436
 437         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 438         self.assertRegexpMatches(collection2["manifest_text"],
 439             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 440
 441
 442 def fuseUpdateFileTestHelper(mounttmp):
 443     class Test(unittest.TestCase):
 444         def runTest(self):
 445             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 446                 f.write("Hello world!")
 447
 448             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 449                 fr = f.read()
 450                 self.assertEqual(fr, "Hello world!")
 451                 f.seek(0)
 452                 f.write("Hola mundo!")
 453                 f.seek(0)
 454                 fr = f.read()
 455                 self.assertEqual(fr, "Hola mundo!!")
 456
 457             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 458                 self.assertEqual(f.read(), "Hola mundo!!")
 459
 460     Test().runTest()
 461
 462 class FuseUpdateFileTest(MountTestBase):
 463     def runTest(self):
 464         collection = arvados.collection.Collection(api_client=self.api)
 465         collection.save_new()
 466
 467         m = self.make_mount(fuse.CollectionDirectory)
 468         with llfuse.lock:
 469             m.new_collection(collection.api_response(), collection)
 470         self.assertTrue(m.writable())
 471
 472         # See note in MountTestBase.setUp
 473         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 474
 475         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 476         self.assertRegexpMatches(collection2["manifest_text"],
 477             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 478
 479
 480 def fuseMkdirTestHelper(mounttmp):
 481     class Test(unittest.TestCase):
 482         def runTest(self):
 483             with self.assertRaises(IOError):
 484                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 485                     f.write("Hello world!")
 486
 487             os.mkdir(os.path.join(mounttmp, "testdir"))
 488
 489             with self.assertRaises(OSError):
 490                 os.mkdir(os.path.join(mounttmp, "testdir"))
 491
 492             d1 = llfuse.listdir(mounttmp)
 493             self.assertEqual(["testdir"], d1)
 494
 495             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 496                 f.write("Hello world!")
 497
 498             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 499             self.assertEqual(["file1.txt"], d1)
 500
 501     Test().runTest()
 502
 503 class FuseMkdirTest(MountTestBase):
 504     def runTest(self):
 505         collection = arvados.collection.Collection(api_client=self.api)
 506         collection.save_new()
 507
 508         m = self.make_mount(fuse.CollectionDirectory)
 509         with llfuse.lock:
 510             m.new_collection(collection.api_response(), collection)
 511         self.assertTrue(m.writable())
 512
 513         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 514
 515         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 516         self.assertRegexpMatches(collection2["manifest_text"],
 517             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 518
 519
 520 def fuseRmTestHelperWriteFile(mounttmp):
 521     class Test(unittest.TestCase):
 522         def runTest(self):
 523             os.mkdir(os.path.join(mounttmp, "testdir"))
 524
 525             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 526                 f.write("Hello world!")
 527
 528     Test().runTest()
 529
 530 def fuseRmTestHelperDeleteFile(mounttmp):
 531     class Test(unittest.TestCase):
 532         def runTest(self):
 533             # Can't delete because it's not empty
 534             with self.assertRaises(OSError):
 535                 os.rmdir(os.path.join(mounttmp, "testdir"))
 536
 537             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 538             self.assertEqual(["file1.txt"], d1)
 539
 540             # Delete file
 541             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 542
 543             # Make sure it's empty
 544             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 545             self.assertEqual([], d1)
 546
 547             # Try to delete it again
 548             with self.assertRaises(OSError):
 549                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 550
 551     Test().runTest()
 552
 553 def fuseRmTestHelperRmdir(mounttmp):
 554     class Test(unittest.TestCase):
 555         def runTest(self):
 556             # Should be able to delete now that it is empty
 557             os.rmdir(os.path.join(mounttmp, "testdir"))
 558
 559             # Make sure it's empty
 560             d1 = llfuse.listdir(os.path.join(mounttmp))
 561             self.assertEqual([], d1)
 562
 563             # Try to delete it again
 564             with self.assertRaises(OSError):
 565                 os.rmdir(os.path.join(mounttmp, "testdir"))
 566
 567     Test().runTest()
 568
 569 class FuseRmTest(MountTestBase):
 570     def runTest(self):
 571         collection = arvados.collection.Collection(api_client=self.api)
 572         collection.save_new()
 573
 574         m = self.make_mount(fuse.CollectionDirectory)
 575         with llfuse.lock:
 576             m.new_collection(collection.api_response(), collection)
 577         self.assertTrue(m.writable())
 578
 579         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 580
 581         # Starting manifest
 582         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 583         self.assertRegexpMatches(collection2["manifest_text"],
 584             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 585         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 586
 587         # Can't have empty directories :-( so manifest will be empty.
 588         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 589         self.assertEqual(collection2["manifest_text"], "")
 590
 591         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 592
 593         # manifest should be empty now.
 594         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 595         self.assertEqual(collection2["manifest_text"], "")
 596
 597
 598 def fuseMvFileTestHelperWriteFile(mounttmp):
 599     class Test(unittest.TestCase):
 600         def runTest(self):
 601             os.mkdir(os.path.join(mounttmp, "testdir"))
 602
 603             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 604                 f.write("Hello world!")
 605
 606     Test().runTest()
 607
 608 def fuseMvFileTestHelperMoveFile(mounttmp):
 609     class Test(unittest.TestCase):
 610         def runTest(self):
 611             d1 = llfuse.listdir(os.path.join(mounttmp))
 612             self.assertEqual(["testdir"], d1)
 613             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 614             self.assertEqual(["file1.txt"], d1)
 615
 616             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 617
 618             d1 = llfuse.listdir(os.path.join(mounttmp))
 619             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 620             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 621             self.assertEqual([], d1)
 622
 623     Test().runTest()
 624
 625 class FuseMvFileTest(MountTestBase):
 626     def runTest(self):
 627         collection = arvados.collection.Collection(api_client=self.api)
 628         collection.save_new()
 629
 630         m = self.make_mount(fuse.CollectionDirectory)
 631         with llfuse.lock:
 632             m.new_collection(collection.api_response(), collection)
 633         self.assertTrue(m.writable())
 634
 635         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 636
 637         # Starting manifest
 638         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 639         self.assertRegexpMatches(collection2["manifest_text"],
 640             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 641
 642         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 643
 644         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 645         self.assertRegexpMatches(collection2["manifest_text"],
 646             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 647
 648
 649 def fuseRenameTestHelper(mounttmp):
 650     class Test(unittest.TestCase):
 651         def runTest(self):
 652             os.mkdir(os.path.join(mounttmp, "testdir"))
 653
 654             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 655                 f.write("Hello world!")
 656
 657     Test().runTest()
 658
 659 class FuseRenameTest(MountTestBase):
 660     def runTest(self):
 661         collection = arvados.collection.Collection(api_client=self.api)
 662         collection.save_new()
 663
 664         m = self.make_mount(fuse.CollectionDirectory)
 665         with llfuse.lock:
 666             m.new_collection(collection.api_response(), collection)
 667         self.assertTrue(m.writable())
 668
 669         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 670
 671         # Starting manifest
 672         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 673         self.assertRegexpMatches(collection2["manifest_text"],
 674             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 675
 676         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 677         self.assertEqual(["testdir"], d1)
 678         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 679         self.assertEqual(["file1.txt"], d1)
 680
 681         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 682
 683         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 684         self.assertEqual(["testdir2"], sorted(d1))
 685         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 686         self.assertEqual(["file1.txt"], d1)
 687
 688         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 689         self.assertRegexpMatches(collection2["manifest_text"],
 690             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 691
 692
 693 class FuseUpdateFromEventTest(MountTestBase):
 694     def runTest(self):
 695         collection = arvados.collection.Collection(api_client=self.api)
 696         collection.save_new()
 697
 698         m = self.make_mount(fuse.CollectionDirectory)
 699         with llfuse.lock:
 700             m.new_collection(collection.api_response(), collection)
 701
 702         self.operations.listen_for_events(self.api)
 703
 704         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 705         self.assertEqual([], sorted(d1))
 706
 707         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 708             with collection2.open("file1.txt", "w") as f:
 709                 f.write("foo")
 710
 711         time.sleep(1)
 712
 713         # should show up via event bus notify
 714
 715         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 716         self.assertEqual(["file1.txt"], sorted(d1))
 717
 718
 719 def fuseFileConflictTestHelper(mounttmp):
 720     class Test(unittest.TestCase):
 721         def runTest(self):
 722             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 723                 f.write("bar")
 724
 725             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 726             self.assertEqual(len(d1), 2)
 727
 728             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 729                 self.assertEqual(f.read(), "bar")
 730
 731             self.assertRegexpMatches(d1[1],
 732                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 733
 734             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 735                 self.assertEqual(f.read(), "foo")
 736
 737     Test().runTest()
 738
 739 class FuseFileConflictTest(MountTestBase):
 740     def runTest(self):
 741         collection = arvados.collection.Collection(api_client=self.api)
 742         collection.save_new()
 743
 744         m = self.make_mount(fuse.CollectionDirectory)
 745         with llfuse.lock:
 746             m.new_collection(collection.api_response(), collection)
 747
 748         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 749         self.assertEqual([], sorted(d1))
 750
 751         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 752             with collection2.open("file1.txt", "w") as f:
 753                 f.write("foo")
 754
 755         # See note in MountTestBase.setUp
 756         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 757
 758
 759 def fuseUnlinkOpenFileTest(mounttmp):
 760     class Test(unittest.TestCase):
 761         def runTest(self):
 762             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 763                 f.write("foo")
 764
 765                 d1 = llfuse.listdir(os.path.join(mounttmp))
 766                 self.assertEqual(["file1.txt"], sorted(d1))
 767
 768                 os.remove(os.path.join(mounttmp, "file1.txt"))
 769
 770                 d1 = llfuse.listdir(os.path.join(mounttmp))
 771                 self.assertEqual([], sorted(d1))
 772
 773                 f.seek(0)
 774                 self.assertEqual(f.read(), "foo")
 775                 f.write("bar")
 776
 777                 f.seek(0)
 778                 self.assertEqual(f.read(), "foobar")
 779
 780     Test().runTest()
 781
 782 class FuseUnlinkOpenFileTest(MountTestBase):
 783     def runTest(self):
 784         collection = arvados.collection.Collection(api_client=self.api)
 785         collection.save_new()
 786
 787         m = self.make_mount(fuse.CollectionDirectory)
 788         with llfuse.lock:
 789             m.new_collection(collection.api_response(), collection)
 790
 791         # See note in MountTestBase.setUp
 792         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 793
 794         self.assertEqual(collection.manifest_text(), "")
 795
 796
 797 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 798     class Test(unittest.TestCase):
 799         def runTest(self):
 800             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 801                 f.write("Hello world!")
 802
 803             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 804             self.assertEqual(["file1.txt"], sorted(d1))
 805             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 806             self.assertEqual([], sorted(d1))
 807
 808     Test().runTest()
 809
 810 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 811     class Test(unittest.TestCase):
 812         def runTest(self):
 813             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 814
 815             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 816             self.assertEqual([], sorted(d1))
 817             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 818             self.assertEqual(["file2.txt"], sorted(d1))
 819
 820     Test().runTest()
 821
 822 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 823     def runTest(self):
 824         collection1 = arvados.collection.Collection(api_client=self.api)
 825         collection1.save_new()
 826
 827         collection2 = arvados.collection.Collection(api_client=self.api)
 828         collection2.save_new()
 829
 830         m = self.make_mount(fuse.MagicDirectory)
 831
 832         # See note in MountTestBase.setUp
 833         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 834                                                   collection1.manifest_locator(),
 835                                                   collection2.manifest_locator()))
 836
 837         collection1.update()
 838         collection2.update()
 839
 840         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 841         self.assertEqual(collection2.manifest_text(), "")
 842
 843         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 844                                                   collection1.manifest_locator(),
 845                                                   collection2.manifest_locator()))
 846
 847         collection1.update()
 848         collection2.update()
 849
 850         self.assertEqual(collection1.manifest_text(), "")
 851         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 852
 853         collection1.stop_threads()
 854         collection2.stop_threads()
 855
 856
 857 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 858     class Test(unittest.TestCase):
 859         def runTest(self):
 860             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 861             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 862                 f.write("Hello world!")
 863
 864             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 865             self.assertEqual(["testdir"], sorted(d1))
 866             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 867             self.assertEqual(["file1.txt"], sorted(d1))
 868
 869             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 870             self.assertEqual([], sorted(d1))
 871
 872     Test().runTest()
 873
 874
 875 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 876     class Test(unittest.TestCase):
 877         def runTest(self):
 878             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 879
 880             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 881             self.assertEqual([], sorted(d1))
 882
 883             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 884             self.assertEqual(["testdir2"], sorted(d1))
 885             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 886             self.assertEqual(["file1.txt"], sorted(d1))
 887
 888             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 889                 self.assertEqual(f.read(), "Hello world!")
 890
 891     Test().runTest()
 892
 893 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 894     def runTest(self):
 895         collection1 = arvados.collection.Collection(api_client=self.api)
 896         collection1.save_new()
 897
 898         collection2 = arvados.collection.Collection(api_client=self.api)
 899         collection2.save_new()
 900
 901         m = self.make_mount(fuse.MagicDirectory)
 902
 903         # See note in MountTestBase.setUp
 904         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 905                                                   collection1.manifest_locator(),
 906                                                   collection2.manifest_locator()))
 907
 908         collection1.update()
 909         collection2.update()
 910
 911         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 912         self.assertEqual(collection2.manifest_text(), "")
 913
 914         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 915                                                   collection1.manifest_locator(),
 916                                                   collection2.manifest_locator()))
 917
 918         collection1.update()
 919         collection2.update()
 920
 921         self.assertEqual(collection1.manifest_text(), "")
 922         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 923
 924         collection1.stop_threads()
 925         collection2.stop_threads()
 926
 927 def fuseProjectMkdirTestHelper1(mounttmp):
 928     class Test(unittest.TestCase):
 929         def runTest(self):
 930             os.mkdir(os.path.join(mounttmp, "testcollection"))
 931             with self.assertRaises(OSError):
 932                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 933     Test().runTest()
 934
 935 def fuseProjectMkdirTestHelper2(mounttmp):
 936     class Test(unittest.TestCase):
 937         def runTest(self):
 938             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 939                 f.write("Hello world!")
 940             with self.assertRaises(OSError):
 941                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 942             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 943             with self.assertRaises(OSError):
 944                 os.remove(os.path.join(mounttmp, "testcollection"))
 945             os.rmdir(os.path.join(mounttmp, "testcollection"))
 946     Test().runTest()
 947
 948 class FuseProjectMkdirRmdirTest(MountTestBase):
 949     def runTest(self):
 950         self.make_mount(fuse.ProjectDirectory,
 951                         project_object=self.api.users().current().execute())
 952
 953         d1 = llfuse.listdir(self.mounttmp)
 954         self.assertNotIn('testcollection', d1)
 955
 956         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
 957
 958         d1 = llfuse.listdir(self.mounttmp)
 959         self.assertIn('testcollection', d1)
 960
 961         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
 962
 963         d1 = llfuse.listdir(self.mounttmp)
 964         self.assertNotIn('testcollection', d1)
 965
 966
 967 def fuseProjectMvTestHelper1(mounttmp):
 968     class Test(unittest.TestCase):
 969         def runTest(self):
 970             d1 = llfuse.listdir(mounttmp)
 971             self.assertNotIn('testcollection', d1)
 972
 973             os.mkdir(os.path.join(mounttmp, "testcollection"))
 974
 975             d1 = llfuse.listdir(mounttmp)
 976             self.assertIn('testcollection', d1)
 977
 978             with self.assertRaises(OSError):
 979                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
 980
 981             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
 982
 983             d1 = llfuse.listdir(mounttmp)
 984             self.assertNotIn('testcollection', d1)
 985
 986             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
 987             self.assertIn('testcollection', d1)
 988
 989     Test().runTest()
 990
 991 class FuseProjectMvTest(MountTestBase):
 992     def runTest(self):
 993         self.make_mount(fuse.ProjectDirectory,
 994                         project_object=self.api.users().current().execute())
 995
 996         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
 997
 998
 999 def fuseFsyncTestHelper(mounttmp, k):
1000     class Test(unittest.TestCase):
1001         def runTest(self):
1002             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1003             os.fsync(fd)
1004             os.close(fd)
1005
1006     Test().runTest()
1007
1008 class FuseFsyncTest(FuseMagicTest):
1009     def runTest(self):
1010         self.make_mount(fuse.MagicDirectory)
1011         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1012
1013
1014 class MagicDirApiError(FuseMagicTest):
1015     def setUp(self):
1016         api = mock.MagicMock()
1017         super(MagicDirApiError, self).setUp(api=api)
1018         api.collections().get().execute.side_effect = iter([Exception('API fail'), {"manifest_text": self.test_manifest}])
1019         api.keep.get.side_effect = Exception('Keep fail')
1020
1021     def runTest(self):
1022         self.make_mount(fuse.MagicDirectory)
1023
1024         self.operations.inodes.inode_cache.cap = 1
1025         self.operations.inodes.inode_cache.min_entries = 2
1026
1027         with self.assertRaises(OSError):
1028             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1029
1030         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1031
1032
1033 class FuseUnitTest(unittest.TestCase):
1034     def test_sanitize_filename(self):
1035         acceptable = [
1036             "foo.txt",
1037             ".foo",
1038             "..foo",
1039             "...",
1040             "foo...",
1041             "foo..",
1042             "foo.",
1043             "-",
1044             "\x01\x02\x03",
1045             ]
1046         unacceptable = [
1047             "f\00",
1048             "\00\00",
1049             "/foo",
1050             "foo/",
1051             "//",
1052             ]
1053         for f in acceptable:
1054             self.assertEqual(f, fuse.sanitize_filename(f))
1055         for f in unacceptable:
1056             self.assertNotEqual(f, fuse.sanitize_filename(f))
1057             # The sanitized filename should be the same length, though.
1058             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1059         # Special cases
1060         self.assertEqual("_", fuse.sanitize_filename(""))
1061         self.assertEqual("_", fuse.sanitize_filename("."))
1062         self.assertEqual("__", fuse.sanitize_filename(".."))