services/fuse/tests/test_mount.py

   1 import arvados
   2 import arvados.safeapi
   3 import arvados_fuse as fuse
   4 import glob
   5 import json
   6 import llfuse
   7 import os
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tempfile
  12 import threading
  13 import time
  14 import unittest
  15 import logging
  16 import multiprocessing
  17 import run_test_server
  18 import mock
  19
  20 from mount_test_base import MountTestBase
  21
  22 logger = logging.getLogger('arvados.arv-mount')
  23
  24
  25 class FuseMountTest(MountTestBase):
  26     def setUp(self):
  27         super(FuseMountTest, self).setUp()
  28
  29         cw = arvados.CollectionWriter()
  30
  31         cw.start_new_file('thing1.txt')
  32         cw.write("data 1")
  33         cw.start_new_file('thing2.txt')
  34         cw.write("data 2")
  35
  36         cw.start_new_stream('dir1')
  37         cw.start_new_file('thing3.txt')
  38         cw.write("data 3")
  39         cw.start_new_file('thing4.txt')
  40         cw.write("data 4")
  41
  42         cw.start_new_stream('dir2')
  43         cw.start_new_file('thing5.txt')
  44         cw.write("data 5")
  45         cw.start_new_file('thing6.txt')
  46         cw.write("data 6")
  47
  48         cw.start_new_stream('dir2/dir3')
  49         cw.start_new_file('thing7.txt')
  50         cw.write("data 7")
  51
  52         cw.start_new_file('thing8.txt')
  53         cw.write("data 8")
  54
  55         cw.start_new_stream('edgecases')
  56         for f in ":/.../-/*/\x01\\/ ".split("/"):
  57             cw.start_new_file(f)
  58             cw.write('x')
  59
  60         for f in ":/.../-/*/\x01\\/ ".split("/"):
  61             cw.start_new_stream('edgecases/dirs/' + f)
  62             cw.start_new_file('x/x')
  63             cw.write('x')
  64
  65         self.testcollection = cw.finish()
  66         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  67
  68     def runTest(self):
  69         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  70
  71         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
  72                                       'edgecases', 'dir1', 'dir2'])
  73         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
  74         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
  75         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
  76         self.assertDirContents('edgecases',
  77                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
  78         self.assertDirContents('edgecases/dirs',
  79                                ":/.../-/*/\x01\\/ ".split("/"))
  80
  81         files = {'thing1.txt': 'data 1',
  82                  'thing2.txt': 'data 2',
  83                  'dir1/thing3.txt': 'data 3',
  84                  'dir1/thing4.txt': 'data 4',
  85                  'dir2/thing5.txt': 'data 5',
  86                  'dir2/thing6.txt': 'data 6',
  87                  'dir2/dir3/thing7.txt': 'data 7',
  88                  'dir2/dir3/thing8.txt': 'data 8'}
  89
  90         for k, v in files.items():
  91             with open(os.path.join(self.mounttmp, k)) as f:
  92                 self.assertEqual(v, f.read())
  93
  94
  95 class FuseNoAPITest(MountTestBase):
  96     def setUp(self):
  97         super(FuseNoAPITest, self).setUp()
  98         keep = arvados.keep.KeepClient(local_store=self.keeptmp)
  99         self.file_data = "API-free text\n"
 100         self.file_loc = keep.put(self.file_data)
 101         self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
 102                 self.file_loc, len(self.file_data)))
 103
 104     def runTest(self):
 105         self.make_mount(fuse.MagicDirectory)
 106         self.assertDirContents(self.coll_loc, ['api-free.txt'])
 107         with open(os.path.join(
 108                 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
 109             actual = keep_file.read(-1)
 110         self.assertEqual(self.file_data, actual)
 111
 112
 113 class FuseMagicTest(MountTestBase):
 114     def setUp(self, api=None):
 115         super(FuseMagicTest, self).setUp(api=api)
 116
 117         cw = arvados.CollectionWriter()
 118
 119         cw.start_new_file('thing1.txt')
 120         cw.write("data 1")
 121
 122         self.testcollection = cw.finish()
 123         self.test_manifest = cw.manifest_text()
 124         self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 125
 126     def runTest(self):
 127         self.make_mount(fuse.MagicDirectory)
 128
 129         mount_ls = llfuse.listdir(self.mounttmp)
 130         self.assertIn('README', mount_ls)
 131         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 132                              arvados.util.uuid_pattern.match(fn)
 133                              for fn in mount_ls),
 134                          "new FUSE MagicDirectory lists Collection")
 135         self.assertDirContents(self.testcollection, ['thing1.txt'])
 136         self.assertDirContents(os.path.join('by_id', self.testcollection),
 137                                ['thing1.txt'])
 138         mount_ls = llfuse.listdir(self.mounttmp)
 139         self.assertIn('README', mount_ls)
 140         self.assertIn(self.testcollection, mount_ls)
 141         self.assertIn(self.testcollection,
 142                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 143
 144         files = {}
 145         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 146
 147         for k, v in files.items():
 148             with open(os.path.join(self.mounttmp, k)) as f:
 149                 self.assertEqual(v, f.read())
 150
 151
 152 class FuseTagsTest(MountTestBase):
 153     def runTest(self):
 154         self.make_mount(fuse.TagsDirectory)
 155
 156         d1 = llfuse.listdir(self.mounttmp)
 157         d1.sort()
 158         self.assertEqual(['foo_tag'], d1)
 159
 160         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 161         d2.sort()
 162         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 163
 164         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 165         d3.sort()
 166         self.assertEqual(['foo'], d3)
 167
 168
 169 class FuseTagsUpdateTest(MountTestBase):
 170     def tag_collection(self, coll_uuid, tag_name):
 171         return self.api.links().create(
 172             body={'link': {'head_uuid': coll_uuid,
 173                            'link_class': 'tag',
 174                            'name': tag_name,
 175         }}).execute()
 176
 177     def runTest(self):
 178         self.make_mount(fuse.TagsDirectory, poll_time=1)
 179
 180         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 181
 182         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 183         self.tag_collection(bar_uuid, 'fuse_test_tag')
 184         time.sleep(1)
 185         self.assertIn('fuse_test_tag', llfuse.listdir(self.mounttmp))
 186         self.assertDirContents('fuse_test_tag', [bar_uuid])
 187
 188         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 189         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 190         time.sleep(1)
 191         self.assertDirContents('fuse_test_tag', [bar_uuid, baz_uuid])
 192
 193         self.api.links().delete(uuid=l['uuid']).execute()
 194         time.sleep(1)
 195         self.assertDirContents('fuse_test_tag', [bar_uuid])
 196
 197
 198 class FuseSharedTest(MountTestBase):
 199     def runTest(self):
 200         self.make_mount(fuse.SharedDirectory,
 201                         exclude=self.api.users().current().execute()['uuid'])
 202
 203         # shared_dirs is a list of the directories exposed
 204         # by fuse.SharedDirectory (i.e. any object visible
 205         # to the current user)
 206         shared_dirs = llfuse.listdir(self.mounttmp)
 207         shared_dirs.sort()
 208         self.assertIn('FUSE User', shared_dirs)
 209
 210         # fuse_user_objs is a list of the objects owned by the FUSE
 211         # test user (which present as files in the 'FUSE User'
 212         # directory)
 213         fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
 214         fuse_user_objs.sort()
 215         self.assertEqual(['FUSE Test Project',                    # project owned by user
 216                           'collection #1 owned by FUSE',          # collection owned by user
 217                           'collection #2 owned by FUSE',          # collection owned by user
 218                           'pipeline instance owned by FUSE.pipelineInstance',  # pipeline instance owned by user
 219                       ], fuse_user_objs)
 220
 221         # test_proj_files is a list of the files in the FUSE Test Project.
 222         test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
 223         test_proj_files.sort()
 224         self.assertEqual(['collection in FUSE project',
 225                           'pipeline instance in FUSE project.pipelineInstance',
 226                           'pipeline template in FUSE project.pipelineTemplate'
 227                       ], test_proj_files)
 228
 229         # Double check that we can open and read objects in this folder as a file,
 230         # and that its contents are what we expect.
 231         pipeline_template_path = os.path.join(
 232                 self.mounttmp,
 233                 'FUSE User',
 234                 'FUSE Test Project',
 235                 'pipeline template in FUSE project.pipelineTemplate')
 236         with open(pipeline_template_path) as f:
 237             j = json.load(f)
 238             self.assertEqual("pipeline template in FUSE project", j['name'])
 239
 240         # check mtime on template
 241         st = os.stat(pipeline_template_path)
 242         self.assertEqual(st.st_mtime, 1397493304)
 243
 244         # check mtime on collection
 245         st = os.stat(os.path.join(
 246                 self.mounttmp,
 247                 'FUSE User',
 248                 'collection #1 owned by FUSE'))
 249         self.assertEqual(st.st_mtime, 1391448174)
 250
 251
 252 class FuseHomeTest(MountTestBase):
 253     def runTest(self):
 254         self.make_mount(fuse.ProjectDirectory,
 255                         project_object=self.api.users().current().execute())
 256
 257         d1 = llfuse.listdir(self.mounttmp)
 258         self.assertIn('Unrestricted public data', d1)
 259
 260         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 261         public_project = run_test_server.fixture('groups')[
 262             'anonymously_accessible_project']
 263         found_in = 0
 264         found_not_in = 0
 265         for name, item in run_test_server.fixture('collections').iteritems():
 266             if 'name' not in item:
 267                 pass
 268             elif item['owner_uuid'] == public_project['uuid']:
 269                 self.assertIn(item['name'], d2)
 270                 found_in += 1
 271             else:
 272                 # Artificial assumption here: there is no public
 273                 # collection fixture with the same name as a
 274                 # non-public collection.
 275                 self.assertNotIn(item['name'], d2)
 276                 found_not_in += 1
 277         self.assertNotEqual(0, found_in)
 278         self.assertNotEqual(0, found_not_in)
 279
 280         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 281         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 282
 283
 284 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 285     class Test(unittest.TestCase):
 286         def runTest(self):
 287             d1 = llfuse.listdir(mounttmp)
 288             self.assertEqual(["file1.txt"], d1)
 289             with open(os.path.join(mounttmp, "file1.txt")) as f:
 290                 self.assertEqual("blub", f.read())
 291     Test().runTest()
 292
 293 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 294     class Test(unittest.TestCase):
 295         def runTest(self):
 296             d1 = llfuse.listdir(mounttmp)
 297             self.assertEqual(["file1.txt"], d1)
 298             with open(os.path.join(mounttmp, "file1.txt")) as f:
 299                 self.assertEqual("plnp", f.read())
 300     Test().runTest()
 301
 302 class FuseModifyFileTest(MountTestBase):
 303     def runTest(self):
 304         collection = arvados.collection.Collection(api_client=self.api)
 305         with collection.open("file1.txt", "w") as f:
 306             f.write("blub")
 307
 308         collection.save_new()
 309
 310         m = self.make_mount(fuse.CollectionDirectory)
 311         with llfuse.lock:
 312             m.new_collection(collection.api_response(), collection)
 313
 314         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 315
 316         with collection.open("file1.txt", "w") as f:
 317             f.write("plnp")
 318
 319         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 320
 321
 322 class FuseAddFileToCollectionTest(MountTestBase):
 323     def runTest(self):
 324         collection = arvados.collection.Collection(api_client=self.api)
 325         with collection.open("file1.txt", "w") as f:
 326             f.write("blub")
 327
 328         collection.save_new()
 329
 330         m = self.make_mount(fuse.CollectionDirectory)
 331         with llfuse.lock:
 332             m.new_collection(collection.api_response(), collection)
 333
 334         d1 = llfuse.listdir(self.mounttmp)
 335         self.assertEqual(["file1.txt"], d1)
 336
 337         with collection.open("file2.txt", "w") as f:
 338             f.write("plnp")
 339
 340         d1 = llfuse.listdir(self.mounttmp)
 341         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 342
 343
 344 class FuseRemoveFileFromCollectionTest(MountTestBase):
 345     def runTest(self):
 346         collection = arvados.collection.Collection(api_client=self.api)
 347         with collection.open("file1.txt", "w") as f:
 348             f.write("blub")
 349
 350         with collection.open("file2.txt", "w") as f:
 351             f.write("plnp")
 352
 353         collection.save_new()
 354
 355         m = self.make_mount(fuse.CollectionDirectory)
 356         with llfuse.lock:
 357             m.new_collection(collection.api_response(), collection)
 358
 359         d1 = llfuse.listdir(self.mounttmp)
 360         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 361
 362         collection.remove("file2.txt")
 363
 364         d1 = llfuse.listdir(self.mounttmp)
 365         self.assertEqual(["file1.txt"], d1)
 366
 367
 368 def fuseCreateFileTestHelper(mounttmp):
 369     class Test(unittest.TestCase):
 370         def runTest(self):
 371             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 372                 pass
 373     Test().runTest()
 374
 375 class FuseCreateFileTest(MountTestBase):
 376     def runTest(self):
 377         collection = arvados.collection.Collection(api_client=self.api)
 378         collection.save_new()
 379
 380         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 381         self.assertEqual(collection2["manifest_text"], "")
 382
 383         collection.save_new()
 384
 385         m = self.make_mount(fuse.CollectionDirectory)
 386         with llfuse.lock:
 387             m.new_collection(collection.api_response(), collection)
 388         self.assertTrue(m.writable())
 389
 390         self.assertNotIn("file1.txt", collection)
 391
 392         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 393
 394         self.assertIn("file1.txt", collection)
 395
 396         d1 = llfuse.listdir(self.mounttmp)
 397         self.assertEqual(["file1.txt"], d1)
 398
 399         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 400         self.assertRegexpMatches(collection2["manifest_text"],
 401             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 402
 403
 404 def fuseWriteFileTestHelperWriteFile(mounttmp):
 405     class Test(unittest.TestCase):
 406         def runTest(self):
 407             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 408                 f.write("Hello world!")
 409     Test().runTest()
 410
 411 def fuseWriteFileTestHelperReadFile(mounttmp):
 412     class Test(unittest.TestCase):
 413         def runTest(self):
 414             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 415                 self.assertEqual(f.read(), "Hello world!")
 416     Test().runTest()
 417
 418 class FuseWriteFileTest(MountTestBase):
 419     def runTest(self):
 420         collection = arvados.collection.Collection(api_client=self.api)
 421         collection.save_new()
 422
 423         m = self.make_mount(fuse.CollectionDirectory)
 424         with llfuse.lock:
 425             m.new_collection(collection.api_response(), collection)
 426         self.assertTrue(m.writable())
 427
 428         self.assertNotIn("file1.txt", collection)
 429
 430         self.assertEqual(0, self.operations.write_counter.get())
 431         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 432         self.assertEqual(12, self.operations.write_counter.get())
 433
 434         with collection.open("file1.txt") as f:
 435             self.assertEqual(f.read(), "Hello world!")
 436
 437         self.assertEqual(0, self.operations.read_counter.get())
 438         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 439         self.assertEqual(12, self.operations.read_counter.get())
 440
 441         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 442         self.assertRegexpMatches(collection2["manifest_text"],
 443             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 444
 445
 446 def fuseUpdateFileTestHelper(mounttmp):
 447     class Test(unittest.TestCase):
 448         def runTest(self):
 449             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 450                 f.write("Hello world!")
 451
 452             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 453                 fr = f.read()
 454                 self.assertEqual(fr, "Hello world!")
 455                 f.seek(0)
 456                 f.write("Hola mundo!")
 457                 f.seek(0)
 458                 fr = f.read()
 459                 self.assertEqual(fr, "Hola mundo!!")
 460
 461             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 462                 self.assertEqual(f.read(), "Hola mundo!!")
 463
 464     Test().runTest()
 465
 466 class FuseUpdateFileTest(MountTestBase):
 467     def runTest(self):
 468         collection = arvados.collection.Collection(api_client=self.api)
 469         collection.save_new()
 470
 471         m = self.make_mount(fuse.CollectionDirectory)
 472         with llfuse.lock:
 473             m.new_collection(collection.api_response(), collection)
 474         self.assertTrue(m.writable())
 475
 476         # See note in MountTestBase.setUp
 477         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 478
 479         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 480         self.assertRegexpMatches(collection2["manifest_text"],
 481             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 482
 483
 484 def fuseMkdirTestHelper(mounttmp):
 485     class Test(unittest.TestCase):
 486         def runTest(self):
 487             with self.assertRaises(IOError):
 488                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 489                     f.write("Hello world!")
 490
 491             os.mkdir(os.path.join(mounttmp, "testdir"))
 492
 493             with self.assertRaises(OSError):
 494                 os.mkdir(os.path.join(mounttmp, "testdir"))
 495
 496             d1 = llfuse.listdir(mounttmp)
 497             self.assertEqual(["testdir"], d1)
 498
 499             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 500                 f.write("Hello world!")
 501
 502             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 503             self.assertEqual(["file1.txt"], d1)
 504
 505     Test().runTest()
 506
 507 class FuseMkdirTest(MountTestBase):
 508     def runTest(self):
 509         collection = arvados.collection.Collection(api_client=self.api)
 510         collection.save_new()
 511
 512         m = self.make_mount(fuse.CollectionDirectory)
 513         with llfuse.lock:
 514             m.new_collection(collection.api_response(), collection)
 515         self.assertTrue(m.writable())
 516
 517         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 518
 519         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 520         self.assertRegexpMatches(collection2["manifest_text"],
 521             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 522
 523
 524 def fuseRmTestHelperWriteFile(mounttmp):
 525     class Test(unittest.TestCase):
 526         def runTest(self):
 527             os.mkdir(os.path.join(mounttmp, "testdir"))
 528
 529             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 530                 f.write("Hello world!")
 531
 532     Test().runTest()
 533
 534 def fuseRmTestHelperDeleteFile(mounttmp):
 535     class Test(unittest.TestCase):
 536         def runTest(self):
 537             # Can't delete because it's not empty
 538             with self.assertRaises(OSError):
 539                 os.rmdir(os.path.join(mounttmp, "testdir"))
 540
 541             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 542             self.assertEqual(["file1.txt"], d1)
 543
 544             # Delete file
 545             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 546
 547             # Make sure it's empty
 548             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 549             self.assertEqual([], d1)
 550
 551             # Try to delete it again
 552             with self.assertRaises(OSError):
 553                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 554
 555     Test().runTest()
 556
 557 def fuseRmTestHelperRmdir(mounttmp):
 558     class Test(unittest.TestCase):
 559         def runTest(self):
 560             # Should be able to delete now that it is empty
 561             os.rmdir(os.path.join(mounttmp, "testdir"))
 562
 563             # Make sure it's empty
 564             d1 = llfuse.listdir(os.path.join(mounttmp))
 565             self.assertEqual([], d1)
 566
 567             # Try to delete it again
 568             with self.assertRaises(OSError):
 569                 os.rmdir(os.path.join(mounttmp, "testdir"))
 570
 571     Test().runTest()
 572
 573 class FuseRmTest(MountTestBase):
 574     def runTest(self):
 575         collection = arvados.collection.Collection(api_client=self.api)
 576         collection.save_new()
 577
 578         m = self.make_mount(fuse.CollectionDirectory)
 579         with llfuse.lock:
 580             m.new_collection(collection.api_response(), collection)
 581         self.assertTrue(m.writable())
 582
 583         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 584
 585         # Starting manifest
 586         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 587         self.assertRegexpMatches(collection2["manifest_text"],
 588             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 589         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 590
 591         # Can't have empty directories :-( so manifest will be empty.
 592         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 593         self.assertEqual(collection2["manifest_text"], "")
 594
 595         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 596
 597         # manifest should be empty now.
 598         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 599         self.assertEqual(collection2["manifest_text"], "")
 600
 601
 602 def fuseMvFileTestHelperWriteFile(mounttmp):
 603     class Test(unittest.TestCase):
 604         def runTest(self):
 605             os.mkdir(os.path.join(mounttmp, "testdir"))
 606
 607             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 608                 f.write("Hello world!")
 609
 610     Test().runTest()
 611
 612 def fuseMvFileTestHelperMoveFile(mounttmp):
 613     class Test(unittest.TestCase):
 614         def runTest(self):
 615             d1 = llfuse.listdir(os.path.join(mounttmp))
 616             self.assertEqual(["testdir"], d1)
 617             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 618             self.assertEqual(["file1.txt"], d1)
 619
 620             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 621
 622             d1 = llfuse.listdir(os.path.join(mounttmp))
 623             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 624             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 625             self.assertEqual([], d1)
 626
 627     Test().runTest()
 628
 629 class FuseMvFileTest(MountTestBase):
 630     def runTest(self):
 631         collection = arvados.collection.Collection(api_client=self.api)
 632         collection.save_new()
 633
 634         m = self.make_mount(fuse.CollectionDirectory)
 635         with llfuse.lock:
 636             m.new_collection(collection.api_response(), collection)
 637         self.assertTrue(m.writable())
 638
 639         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 640
 641         # Starting manifest
 642         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 643         self.assertRegexpMatches(collection2["manifest_text"],
 644             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 645
 646         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 647
 648         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 649         self.assertRegexpMatches(collection2["manifest_text"],
 650             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 651
 652
 653 def fuseRenameTestHelper(mounttmp):
 654     class Test(unittest.TestCase):
 655         def runTest(self):
 656             os.mkdir(os.path.join(mounttmp, "testdir"))
 657
 658             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 659                 f.write("Hello world!")
 660
 661     Test().runTest()
 662
 663 class FuseRenameTest(MountTestBase):
 664     def runTest(self):
 665         collection = arvados.collection.Collection(api_client=self.api)
 666         collection.save_new()
 667
 668         m = self.make_mount(fuse.CollectionDirectory)
 669         with llfuse.lock:
 670             m.new_collection(collection.api_response(), collection)
 671         self.assertTrue(m.writable())
 672
 673         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 674
 675         # Starting manifest
 676         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 677         self.assertRegexpMatches(collection2["manifest_text"],
 678             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 679
 680         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 681         self.assertEqual(["testdir"], d1)
 682         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 683         self.assertEqual(["file1.txt"], d1)
 684
 685         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 686
 687         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 688         self.assertEqual(["testdir2"], sorted(d1))
 689         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 690         self.assertEqual(["file1.txt"], d1)
 691
 692         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 693         self.assertRegexpMatches(collection2["manifest_text"],
 694             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 695
 696
 697 class FuseUpdateFromEventTest(MountTestBase):
 698     def runTest(self):
 699         collection = arvados.collection.Collection(api_client=self.api)
 700         collection.save_new()
 701
 702         m = self.make_mount(fuse.CollectionDirectory)
 703         with llfuse.lock:
 704             m.new_collection(collection.api_response(), collection)
 705
 706         self.operations.listen_for_events(self.api)
 707
 708         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 709         self.assertEqual([], sorted(d1))
 710
 711         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 712             with collection2.open("file1.txt", "w") as f:
 713                 f.write("foo")
 714
 715         time.sleep(1)
 716
 717         # should show up via event bus notify
 718
 719         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 720         self.assertEqual(["file1.txt"], sorted(d1))
 721
 722
 723 def fuseFileConflictTestHelper(mounttmp):
 724     class Test(unittest.TestCase):
 725         def runTest(self):
 726             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 727                 f.write("bar")
 728
 729             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 730             self.assertEqual(len(d1), 2)
 731
 732             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 733                 self.assertEqual(f.read(), "bar")
 734
 735             self.assertRegexpMatches(d1[1],
 736                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 737
 738             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 739                 self.assertEqual(f.read(), "foo")
 740
 741     Test().runTest()
 742
 743 class FuseFileConflictTest(MountTestBase):
 744     def runTest(self):
 745         collection = arvados.collection.Collection(api_client=self.api)
 746         collection.save_new()
 747
 748         m = self.make_mount(fuse.CollectionDirectory)
 749         with llfuse.lock:
 750             m.new_collection(collection.api_response(), collection)
 751
 752         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 753         self.assertEqual([], sorted(d1))
 754
 755         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 756             with collection2.open("file1.txt", "w") as f:
 757                 f.write("foo")
 758
 759         # See note in MountTestBase.setUp
 760         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 761
 762
 763 def fuseUnlinkOpenFileTest(mounttmp):
 764     class Test(unittest.TestCase):
 765         def runTest(self):
 766             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 767                 f.write("foo")
 768
 769                 d1 = llfuse.listdir(os.path.join(mounttmp))
 770                 self.assertEqual(["file1.txt"], sorted(d1))
 771
 772                 os.remove(os.path.join(mounttmp, "file1.txt"))
 773
 774                 d1 = llfuse.listdir(os.path.join(mounttmp))
 775                 self.assertEqual([], sorted(d1))
 776
 777                 f.seek(0)
 778                 self.assertEqual(f.read(), "foo")
 779                 f.write("bar")
 780
 781                 f.seek(0)
 782                 self.assertEqual(f.read(), "foobar")
 783
 784     Test().runTest()
 785
 786 class FuseUnlinkOpenFileTest(MountTestBase):
 787     def runTest(self):
 788         collection = arvados.collection.Collection(api_client=self.api)
 789         collection.save_new()
 790
 791         m = self.make_mount(fuse.CollectionDirectory)
 792         with llfuse.lock:
 793             m.new_collection(collection.api_response(), collection)
 794
 795         # See note in MountTestBase.setUp
 796         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 797
 798         self.assertEqual(collection.manifest_text(), "")
 799
 800
 801 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 802     class Test(unittest.TestCase):
 803         def runTest(self):
 804             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 805                 f.write("Hello world!")
 806
 807             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 808             self.assertEqual(["file1.txt"], sorted(d1))
 809             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 810             self.assertEqual([], sorted(d1))
 811
 812     Test().runTest()
 813
 814 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 815     class Test(unittest.TestCase):
 816         def runTest(self):
 817             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 818
 819             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 820             self.assertEqual([], sorted(d1))
 821             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 822             self.assertEqual(["file2.txt"], sorted(d1))
 823
 824     Test().runTest()
 825
 826 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 827     def runTest(self):
 828         collection1 = arvados.collection.Collection(api_client=self.api)
 829         collection1.save_new()
 830
 831         collection2 = arvados.collection.Collection(api_client=self.api)
 832         collection2.save_new()
 833
 834         m = self.make_mount(fuse.MagicDirectory)
 835
 836         # See note in MountTestBase.setUp
 837         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 838                                                   collection1.manifest_locator(),
 839                                                   collection2.manifest_locator()))
 840
 841         collection1.update()
 842         collection2.update()
 843
 844         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 845         self.assertEqual(collection2.manifest_text(), "")
 846
 847         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 848                                                   collection1.manifest_locator(),
 849                                                   collection2.manifest_locator()))
 850
 851         collection1.update()
 852         collection2.update()
 853
 854         self.assertEqual(collection1.manifest_text(), "")
 855         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 856
 857         collection1.stop_threads()
 858         collection2.stop_threads()
 859
 860
 861 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 862     class Test(unittest.TestCase):
 863         def runTest(self):
 864             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 865             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 866                 f.write("Hello world!")
 867
 868             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 869             self.assertEqual(["testdir"], sorted(d1))
 870             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 871             self.assertEqual(["file1.txt"], sorted(d1))
 872
 873             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 874             self.assertEqual([], sorted(d1))
 875
 876     Test().runTest()
 877
 878
 879 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 880     class Test(unittest.TestCase):
 881         def runTest(self):
 882             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 883
 884             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 885             self.assertEqual([], sorted(d1))
 886
 887             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 888             self.assertEqual(["testdir2"], sorted(d1))
 889             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 890             self.assertEqual(["file1.txt"], sorted(d1))
 891
 892             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 893                 self.assertEqual(f.read(), "Hello world!")
 894
 895     Test().runTest()
 896
 897 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 898     def runTest(self):
 899         collection1 = arvados.collection.Collection(api_client=self.api)
 900         collection1.save_new()
 901
 902         collection2 = arvados.collection.Collection(api_client=self.api)
 903         collection2.save_new()
 904
 905         m = self.make_mount(fuse.MagicDirectory)
 906
 907         # See note in MountTestBase.setUp
 908         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 909                                                   collection1.manifest_locator(),
 910                                                   collection2.manifest_locator()))
 911
 912         collection1.update()
 913         collection2.update()
 914
 915         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 916         self.assertEqual(collection2.manifest_text(), "")
 917
 918         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 919                                                   collection1.manifest_locator(),
 920                                                   collection2.manifest_locator()))
 921
 922         collection1.update()
 923         collection2.update()
 924
 925         self.assertEqual(collection1.manifest_text(), "")
 926         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 927
 928         collection1.stop_threads()
 929         collection2.stop_threads()
 930
 931 def fuseProjectMkdirTestHelper1(mounttmp):
 932     class Test(unittest.TestCase):
 933         def runTest(self):
 934             os.mkdir(os.path.join(mounttmp, "testcollection"))
 935             with self.assertRaises(OSError):
 936                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 937     Test().runTest()
 938
 939 def fuseProjectMkdirTestHelper2(mounttmp):
 940     class Test(unittest.TestCase):
 941         def runTest(self):
 942             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 943                 f.write("Hello world!")
 944             with self.assertRaises(OSError):
 945                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 946             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 947             with self.assertRaises(OSError):
 948                 os.remove(os.path.join(mounttmp, "testcollection"))
 949             os.rmdir(os.path.join(mounttmp, "testcollection"))
 950     Test().runTest()
 951
 952 class FuseProjectMkdirRmdirTest(MountTestBase):
 953     def runTest(self):
 954         self.make_mount(fuse.ProjectDirectory,
 955                         project_object=self.api.users().current().execute())
 956
 957         d1 = llfuse.listdir(self.mounttmp)
 958         self.assertNotIn('testcollection', d1)
 959
 960         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
 961
 962         d1 = llfuse.listdir(self.mounttmp)
 963         self.assertIn('testcollection', d1)
 964
 965         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
 966
 967         d1 = llfuse.listdir(self.mounttmp)
 968         self.assertNotIn('testcollection', d1)
 969
 970
 971 def fuseProjectMvTestHelper1(mounttmp):
 972     class Test(unittest.TestCase):
 973         def runTest(self):
 974             d1 = llfuse.listdir(mounttmp)
 975             self.assertNotIn('testcollection', d1)
 976
 977             os.mkdir(os.path.join(mounttmp, "testcollection"))
 978
 979             d1 = llfuse.listdir(mounttmp)
 980             self.assertIn('testcollection', d1)
 981
 982             with self.assertRaises(OSError):
 983                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
 984
 985             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
 986
 987             d1 = llfuse.listdir(mounttmp)
 988             self.assertNotIn('testcollection', d1)
 989
 990             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
 991             self.assertIn('testcollection', d1)
 992
 993     Test().runTest()
 994
 995 class FuseProjectMvTest(MountTestBase):
 996     def runTest(self):
 997         self.make_mount(fuse.ProjectDirectory,
 998                         project_object=self.api.users().current().execute())
 999
1000         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1001
1002
1003 def fuseFsyncTestHelper(mounttmp, k):
1004     class Test(unittest.TestCase):
1005         def runTest(self):
1006             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1007             os.fsync(fd)
1008             os.close(fd)
1009
1010     Test().runTest()
1011
1012 class FuseFsyncTest(FuseMagicTest):
1013     def runTest(self):
1014         self.make_mount(fuse.MagicDirectory)
1015         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1016
1017
1018 class MagicDirApiError(FuseMagicTest):
1019     def setUp(self):
1020         api = mock.MagicMock()
1021         super(MagicDirApiError, self).setUp(api=api)
1022         api.collections().get().execute.side_effect = iter([Exception('API fail'), {"manifest_text": self.test_manifest}])
1023         api.keep.get.side_effect = Exception('Keep fail')
1024
1025     def runTest(self):
1026         self.make_mount(fuse.MagicDirectory)
1027
1028         self.operations.inodes.inode_cache.cap = 1
1029         self.operations.inodes.inode_cache.min_entries = 2
1030
1031         with self.assertRaises(OSError):
1032             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1033
1034         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1035
1036
1037 class FuseUnitTest(unittest.TestCase):
1038     def test_sanitize_filename(self):
1039         acceptable = [
1040             "foo.txt",
1041             ".foo",
1042             "..foo",
1043             "...",
1044             "foo...",
1045             "foo..",
1046             "foo.",
1047             "-",
1048             "\x01\x02\x03",
1049             ]
1050         unacceptable = [
1051             "f\00",
1052             "\00\00",
1053             "/foo",
1054             "foo/",
1055             "//",
1056             ]
1057         for f in acceptable:
1058             self.assertEqual(f, fuse.sanitize_filename(f))
1059         for f in unacceptable:
1060             self.assertNotEqual(f, fuse.sanitize_filename(f))
1061             # The sanitized filename should be the same length, though.
1062             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1063         # Special cases
1064         self.assertEqual("_", fuse.sanitize_filename(""))
1065         self.assertEqual("_", fuse.sanitize_filename("."))
1066         self.assertEqual("__", fuse.sanitize_filename(".."))
1067
1068
1069 class FuseMagicTestPDHOnly(MountTestBase):
1070     def setUp(self, api=None):
1071         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1072
1073         cw = arvados.CollectionWriter()
1074
1075         cw.start_new_file('thing1.txt')
1076         cw.write("data 1")
1077
1078         self.testcollection = cw.finish()
1079         self.test_manifest = cw.manifest_text()
1080         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1081         self.testcollectionuuid = str(created['uuid'])
1082
1083     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1084         if skip_pdh_only is True:
1085             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1086         else:
1087             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1088
1089         mount_ls = llfuse.listdir(self.mounttmp)
1090         self.assertIn('README', mount_ls)
1091         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1092                              arvados.util.uuid_pattern.match(fn)
1093                              for fn in mount_ls),
1094                          "new FUSE MagicDirectory lists Collection")
1095
1096         # look up using pdh should succeed in all cases
1097         self.assertDirContents(self.testcollection, ['thing1.txt'])
1098         self.assertDirContents(os.path.join('by_id', self.testcollection),
1099                                ['thing1.txt'])
1100         mount_ls = llfuse.listdir(self.mounttmp)
1101         self.assertIn('README', mount_ls)
1102         self.assertIn(self.testcollection, mount_ls)
1103         self.assertIn(self.testcollection,
1104                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1105
1106         files = {}
1107         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1108
1109         for k, v in files.items():
1110             with open(os.path.join(self.mounttmp, k)) as f:
1111                 self.assertEqual(v, f.read())
1112
1113         # look up using uuid should fail when pdh_only is set
1114         if pdh_only is True:
1115             with self.assertRaises(OSError):
1116                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1117                                ['thing1.txt'])
1118         else:
1119             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1120                                ['thing1.txt'])
1121
1122     def test_with_pdh_only_true(self):
1123         self.verify_pdh_only(pdh_only=True)
1124
1125     def test_with_pdh_only_false(self):
1126         self.verify_pdh_only(pdh_only=False)
1127
1128     def test_with_default_by_id(self):
1129         self.verify_pdh_only(skip_pdh_only=True)