services/fuse/tests/test_mount.py

   1 import json
   2 import llfuse
   3 import logging
   4 import mock
   5 import os
   6 import subprocess
   7 import time
   8 import unittest
   9
  10 import arvados
  11 import arvados_fuse as fuse
  12 import run_test_server
  13
  14 from mount_test_base import MountTestBase
  15
  16 logger = logging.getLogger('arvados.arv-mount')
  17
  18
  19 class AssertWithTimeout(object):
  20     """Allow some time for an assertion to pass."""
  21
  22     def __init__(self, timeout=0):
  23         self.timeout = timeout
  24
  25     def __iter__(self):
  26         self.deadline = time.time() + self.timeout
  27         self.done = False
  28         return self
  29
  30     def next(self):
  31         if self.done:
  32             raise StopIteration
  33         return self.attempt
  34
  35     def attempt(self, fn, *args, **kwargs):
  36         try:
  37             fn(*args, **kwargs)
  38         except AssertionError:
  39             if time.time() > self.deadline:
  40                 raise
  41             time.sleep(0.1)
  42         else:
  43             self.done = True
  44
  45
  46 class FuseMountTest(MountTestBase):
  47     def setUp(self):
  48         super(FuseMountTest, self).setUp()
  49
  50         cw = arvados.CollectionWriter()
  51
  52         cw.start_new_file('thing1.txt')
  53         cw.write("data 1")
  54         cw.start_new_file('thing2.txt')
  55         cw.write("data 2")
  56
  57         cw.start_new_stream('dir1')
  58         cw.start_new_file('thing3.txt')
  59         cw.write("data 3")
  60         cw.start_new_file('thing4.txt')
  61         cw.write("data 4")
  62
  63         cw.start_new_stream('dir2')
  64         cw.start_new_file('thing5.txt')
  65         cw.write("data 5")
  66         cw.start_new_file('thing6.txt')
  67         cw.write("data 6")
  68
  69         cw.start_new_stream('dir2/dir3')
  70         cw.start_new_file('thing7.txt')
  71         cw.write("data 7")
  72
  73         cw.start_new_file('thing8.txt')
  74         cw.write("data 8")
  75
  76         cw.start_new_stream('edgecases')
  77         for f in ":/.../-/*/\x01\\/ ".split("/"):
  78             cw.start_new_file(f)
  79             cw.write('x')
  80
  81         for f in ":/.../-/*/\x01\\/ ".split("/"):
  82             cw.start_new_stream('edgecases/dirs/' + f)
  83             cw.start_new_file('x/x')
  84             cw.write('x')
  85
  86         self.testcollection = cw.finish()
  87         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  88
  89     def runTest(self):
  90         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  91
  92         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
  93                                       'edgecases', 'dir1', 'dir2'])
  94         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
  95         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
  96         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
  97         self.assertDirContents('edgecases',
  98                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
  99         self.assertDirContents('edgecases/dirs',
 100                                ":/.../-/*/\x01\\/ ".split("/"))
 101
 102         files = {'thing1.txt': 'data 1',
 103                  'thing2.txt': 'data 2',
 104                  'dir1/thing3.txt': 'data 3',
 105                  'dir1/thing4.txt': 'data 4',
 106                  'dir2/thing5.txt': 'data 5',
 107                  'dir2/thing6.txt': 'data 6',
 108                  'dir2/dir3/thing7.txt': 'data 7',
 109                  'dir2/dir3/thing8.txt': 'data 8'}
 110
 111         for k, v in files.items():
 112             with open(os.path.join(self.mounttmp, k)) as f:
 113                 self.assertEqual(v, f.read())
 114
 115
 116 class FuseNoAPITest(MountTestBase):
 117     def setUp(self):
 118         super(FuseNoAPITest, self).setUp()
 119         keep = arvados.keep.KeepClient(local_store=self.keeptmp)
 120         self.file_data = "API-free text\n"
 121         self.file_loc = keep.put(self.file_data)
 122         self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
 123                 self.file_loc, len(self.file_data)))
 124
 125     def runTest(self):
 126         self.make_mount(fuse.MagicDirectory)
 127         self.assertDirContents(self.coll_loc, ['api-free.txt'])
 128         with open(os.path.join(
 129                 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
 130             actual = keep_file.read(-1)
 131         self.assertEqual(self.file_data, actual)
 132
 133
 134 class FuseMagicTest(MountTestBase):
 135     def setUp(self, api=None):
 136         super(FuseMagicTest, self).setUp(api=api)
 137
 138         cw = arvados.CollectionWriter()
 139
 140         cw.start_new_file('thing1.txt')
 141         cw.write("data 1")
 142
 143         self.testcollection = cw.finish()
 144         self.test_manifest = cw.manifest_text()
 145         self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 146
 147     def runTest(self):
 148         self.make_mount(fuse.MagicDirectory)
 149
 150         mount_ls = llfuse.listdir(self.mounttmp)
 151         self.assertIn('README', mount_ls)
 152         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 153                              arvados.util.uuid_pattern.match(fn)
 154                              for fn in mount_ls),
 155                          "new FUSE MagicDirectory lists Collection")
 156         self.assertDirContents(self.testcollection, ['thing1.txt'])
 157         self.assertDirContents(os.path.join('by_id', self.testcollection),
 158                                ['thing1.txt'])
 159         mount_ls = llfuse.listdir(self.mounttmp)
 160         self.assertIn('README', mount_ls)
 161         self.assertIn(self.testcollection, mount_ls)
 162         self.assertIn(self.testcollection,
 163                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 164
 165         files = {}
 166         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 167
 168         for k, v in files.items():
 169             with open(os.path.join(self.mounttmp, k)) as f:
 170                 self.assertEqual(v, f.read())
 171
 172
 173 class FuseTagsTest(MountTestBase):
 174     def runTest(self):
 175         self.make_mount(fuse.TagsDirectory)
 176
 177         d1 = llfuse.listdir(self.mounttmp)
 178         d1.sort()
 179         self.assertEqual(['foo_tag'], d1)
 180
 181         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 182         d2.sort()
 183         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 184
 185         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 186         d3.sort()
 187         self.assertEqual(['foo'], d3)
 188
 189
 190 class FuseTagsUpdateTest(MountTestBase):
 191     def tag_collection(self, coll_uuid, tag_name):
 192         return self.api.links().create(
 193             body={'link': {'head_uuid': coll_uuid,
 194                            'link_class': 'tag',
 195                            'name': tag_name,
 196         }}).execute()
 197
 198     def runTest(self):
 199         self.make_mount(fuse.TagsDirectory, poll_time=1)
 200
 201         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 202
 203         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 204         self.tag_collection(bar_uuid, 'fuse_test_tag')
 205         for attempt in AssertWithTimeout(10):
 206             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 207         self.assertDirContents('fuse_test_tag', [bar_uuid])
 208
 209         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 210         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 211         for attempt in AssertWithTimeout(10):
 212             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 213
 214         self.api.links().delete(uuid=l['uuid']).execute()
 215         for attempt in AssertWithTimeout(10):
 216             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 217
 218
 219 class FuseSharedTest(MountTestBase):
 220     def runTest(self):
 221         self.make_mount(fuse.SharedDirectory,
 222                         exclude=self.api.users().current().execute()['uuid'])
 223
 224         # shared_dirs is a list of the directories exposed
 225         # by fuse.SharedDirectory (i.e. any object visible
 226         # to the current user)
 227         shared_dirs = llfuse.listdir(self.mounttmp)
 228         shared_dirs.sort()
 229         self.assertIn('FUSE User', shared_dirs)
 230
 231         # fuse_user_objs is a list of the objects owned by the FUSE
 232         # test user (which present as files in the 'FUSE User'
 233         # directory)
 234         fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
 235         fuse_user_objs.sort()
 236         self.assertEqual(['FUSE Test Project',                    # project owned by user
 237                           'collection #1 owned by FUSE',          # collection owned by user
 238                           'collection #2 owned by FUSE',          # collection owned by user
 239                           'pipeline instance owned by FUSE.pipelineInstance',  # pipeline instance owned by user
 240                       ], fuse_user_objs)
 241
 242         # test_proj_files is a list of the files in the FUSE Test Project.
 243         test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
 244         test_proj_files.sort()
 245         self.assertEqual(['collection in FUSE project',
 246                           'pipeline instance in FUSE project.pipelineInstance',
 247                           'pipeline template in FUSE project.pipelineTemplate'
 248                       ], test_proj_files)
 249
 250         # Double check that we can open and read objects in this folder as a file,
 251         # and that its contents are what we expect.
 252         pipeline_template_path = os.path.join(
 253                 self.mounttmp,
 254                 'FUSE User',
 255                 'FUSE Test Project',
 256                 'pipeline template in FUSE project.pipelineTemplate')
 257         with open(pipeline_template_path) as f:
 258             j = json.load(f)
 259             self.assertEqual("pipeline template in FUSE project", j['name'])
 260
 261         # check mtime on template
 262         st = os.stat(pipeline_template_path)
 263         try:
 264             mtime = st.st_mtime_ns / 1000000000
 265         except AttributeError:
 266             mtime = st.st_mtime
 267         self.assertEqual(mtime, 1397493304)
 268
 269         # check mtime on collection
 270         st = os.stat(os.path.join(
 271                 self.mounttmp,
 272                 'FUSE User',
 273                 'collection #1 owned by FUSE'))
 274         try:
 275             mtime = st.st_mtime_ns / 1000000000
 276         except AttributeError:
 277             mtime = st.st_mtime
 278         self.assertEqual(mtime, 1391448174)
 279
 280
 281 class FuseHomeTest(MountTestBase):
 282     def runTest(self):
 283         self.make_mount(fuse.ProjectDirectory,
 284                         project_object=self.api.users().current().execute())
 285
 286         d1 = llfuse.listdir(self.mounttmp)
 287         self.assertIn('Unrestricted public data', d1)
 288
 289         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 290         public_project = run_test_server.fixture('groups')[
 291             'anonymously_accessible_project']
 292         found_in = 0
 293         found_not_in = 0
 294         for name, item in run_test_server.fixture('collections').iteritems():
 295             if 'name' not in item:
 296                 pass
 297             elif item['owner_uuid'] == public_project['uuid']:
 298                 self.assertIn(item['name'], d2)
 299                 found_in += 1
 300             else:
 301                 # Artificial assumption here: there is no public
 302                 # collection fixture with the same name as a
 303                 # non-public collection.
 304                 self.assertNotIn(item['name'], d2)
 305                 found_not_in += 1
 306         self.assertNotEqual(0, found_in)
 307         self.assertNotEqual(0, found_not_in)
 308
 309         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 310         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 311
 312
 313 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 314     class Test(unittest.TestCase):
 315         def runTest(self):
 316             d1 = llfuse.listdir(mounttmp)
 317             self.assertEqual(["file1.txt"], d1)
 318             with open(os.path.join(mounttmp, "file1.txt")) as f:
 319                 self.assertEqual("blub", f.read())
 320     Test().runTest()
 321
 322 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 323     class Test(unittest.TestCase):
 324         def runTest(self):
 325             d1 = llfuse.listdir(mounttmp)
 326             self.assertEqual(["file1.txt"], d1)
 327             with open(os.path.join(mounttmp, "file1.txt")) as f:
 328                 self.assertEqual("plnp", f.read())
 329     Test().runTest()
 330
 331 class FuseModifyFileTest(MountTestBase):
 332     def runTest(self):
 333         collection = arvados.collection.Collection(api_client=self.api)
 334         with collection.open("file1.txt", "w") as f:
 335             f.write("blub")
 336
 337         collection.save_new()
 338
 339         m = self.make_mount(fuse.CollectionDirectory)
 340         with llfuse.lock:
 341             m.new_collection(collection.api_response(), collection)
 342
 343         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 344
 345         with collection.open("file1.txt", "w") as f:
 346             f.write("plnp")
 347
 348         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 349
 350
 351 class FuseAddFileToCollectionTest(MountTestBase):
 352     def runTest(self):
 353         collection = arvados.collection.Collection(api_client=self.api)
 354         with collection.open("file1.txt", "w") as f:
 355             f.write("blub")
 356
 357         collection.save_new()
 358
 359         m = self.make_mount(fuse.CollectionDirectory)
 360         with llfuse.lock:
 361             m.new_collection(collection.api_response(), collection)
 362
 363         d1 = llfuse.listdir(self.mounttmp)
 364         self.assertEqual(["file1.txt"], d1)
 365
 366         with collection.open("file2.txt", "w") as f:
 367             f.write("plnp")
 368
 369         d1 = llfuse.listdir(self.mounttmp)
 370         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 371
 372
 373 class FuseRemoveFileFromCollectionTest(MountTestBase):
 374     def runTest(self):
 375         collection = arvados.collection.Collection(api_client=self.api)
 376         with collection.open("file1.txt", "w") as f:
 377             f.write("blub")
 378
 379         with collection.open("file2.txt", "w") as f:
 380             f.write("plnp")
 381
 382         collection.save_new()
 383
 384         m = self.make_mount(fuse.CollectionDirectory)
 385         with llfuse.lock:
 386             m.new_collection(collection.api_response(), collection)
 387
 388         d1 = llfuse.listdir(self.mounttmp)
 389         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 390
 391         collection.remove("file2.txt")
 392
 393         d1 = llfuse.listdir(self.mounttmp)
 394         self.assertEqual(["file1.txt"], d1)
 395
 396
 397 def fuseCreateFileTestHelper(mounttmp):
 398     class Test(unittest.TestCase):
 399         def runTest(self):
 400             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 401                 pass
 402     Test().runTest()
 403
 404 class FuseCreateFileTest(MountTestBase):
 405     def runTest(self):
 406         collection = arvados.collection.Collection(api_client=self.api)
 407         collection.save_new()
 408
 409         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 410         self.assertEqual(collection2["manifest_text"], "")
 411
 412         collection.save_new()
 413
 414         m = self.make_mount(fuse.CollectionDirectory)
 415         with llfuse.lock:
 416             m.new_collection(collection.api_response(), collection)
 417         self.assertTrue(m.writable())
 418
 419         self.assertNotIn("file1.txt", collection)
 420
 421         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 422
 423         self.assertIn("file1.txt", collection)
 424
 425         d1 = llfuse.listdir(self.mounttmp)
 426         self.assertEqual(["file1.txt"], d1)
 427
 428         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 429         self.assertRegexpMatches(collection2["manifest_text"],
 430             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 431
 432
 433 def fuseWriteFileTestHelperWriteFile(mounttmp):
 434     class Test(unittest.TestCase):
 435         def runTest(self):
 436             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 437                 f.write("Hello world!")
 438     Test().runTest()
 439
 440 def fuseWriteFileTestHelperReadFile(mounttmp):
 441     class Test(unittest.TestCase):
 442         def runTest(self):
 443             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 444                 self.assertEqual(f.read(), "Hello world!")
 445     Test().runTest()
 446
 447 class FuseWriteFileTest(MountTestBase):
 448     def runTest(self):
 449         collection = arvados.collection.Collection(api_client=self.api)
 450         collection.save_new()
 451
 452         m = self.make_mount(fuse.CollectionDirectory)
 453         with llfuse.lock:
 454             m.new_collection(collection.api_response(), collection)
 455         self.assertTrue(m.writable())
 456
 457         self.assertNotIn("file1.txt", collection)
 458
 459         self.assertEqual(0, self.operations.write_counter.get())
 460         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 461         self.assertEqual(12, self.operations.write_counter.get())
 462
 463         with collection.open("file1.txt") as f:
 464             self.assertEqual(f.read(), "Hello world!")
 465
 466         self.assertEqual(0, self.operations.read_counter.get())
 467         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 468         self.assertEqual(12, self.operations.read_counter.get())
 469
 470         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 471         self.assertRegexpMatches(collection2["manifest_text"],
 472             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 473
 474
 475 def fuseUpdateFileTestHelper(mounttmp):
 476     class Test(unittest.TestCase):
 477         def runTest(self):
 478             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 479                 f.write("Hello world!")
 480
 481             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 482                 fr = f.read()
 483                 self.assertEqual(fr, "Hello world!")
 484                 f.seek(0)
 485                 f.write("Hola mundo!")
 486                 f.seek(0)
 487                 fr = f.read()
 488                 self.assertEqual(fr, "Hola mundo!!")
 489
 490             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 491                 self.assertEqual(f.read(), "Hola mundo!!")
 492
 493     Test().runTest()
 494
 495 class FuseUpdateFileTest(MountTestBase):
 496     def runTest(self):
 497         collection = arvados.collection.Collection(api_client=self.api)
 498         collection.save_new()
 499
 500         m = self.make_mount(fuse.CollectionDirectory)
 501         with llfuse.lock:
 502             m.new_collection(collection.api_response(), collection)
 503         self.assertTrue(m.writable())
 504
 505         # See note in MountTestBase.setUp
 506         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 507
 508         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 509         self.assertRegexpMatches(collection2["manifest_text"],
 510             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 511
 512
 513 def fuseMkdirTestHelper(mounttmp):
 514     class Test(unittest.TestCase):
 515         def runTest(self):
 516             with self.assertRaises(IOError):
 517                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 518                     f.write("Hello world!")
 519
 520             os.mkdir(os.path.join(mounttmp, "testdir"))
 521
 522             with self.assertRaises(OSError):
 523                 os.mkdir(os.path.join(mounttmp, "testdir"))
 524
 525             d1 = llfuse.listdir(mounttmp)
 526             self.assertEqual(["testdir"], d1)
 527
 528             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 529                 f.write("Hello world!")
 530
 531             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 532             self.assertEqual(["file1.txt"], d1)
 533
 534     Test().runTest()
 535
 536 class FuseMkdirTest(MountTestBase):
 537     def runTest(self):
 538         collection = arvados.collection.Collection(api_client=self.api)
 539         collection.save_new()
 540
 541         m = self.make_mount(fuse.CollectionDirectory)
 542         with llfuse.lock:
 543             m.new_collection(collection.api_response(), collection)
 544         self.assertTrue(m.writable())
 545
 546         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 547
 548         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 549         self.assertRegexpMatches(collection2["manifest_text"],
 550             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 551
 552
 553 def fuseRmTestHelperWriteFile(mounttmp):
 554     class Test(unittest.TestCase):
 555         def runTest(self):
 556             os.mkdir(os.path.join(mounttmp, "testdir"))
 557
 558             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 559                 f.write("Hello world!")
 560
 561     Test().runTest()
 562
 563 def fuseRmTestHelperDeleteFile(mounttmp):
 564     class Test(unittest.TestCase):
 565         def runTest(self):
 566             # Can't delete because it's not empty
 567             with self.assertRaises(OSError):
 568                 os.rmdir(os.path.join(mounttmp, "testdir"))
 569
 570             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 571             self.assertEqual(["file1.txt"], d1)
 572
 573             # Delete file
 574             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 575
 576             # Make sure it's empty
 577             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 578             self.assertEqual([], d1)
 579
 580             # Try to delete it again
 581             with self.assertRaises(OSError):
 582                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 583
 584     Test().runTest()
 585
 586 def fuseRmTestHelperRmdir(mounttmp):
 587     class Test(unittest.TestCase):
 588         def runTest(self):
 589             # Should be able to delete now that it is empty
 590             os.rmdir(os.path.join(mounttmp, "testdir"))
 591
 592             # Make sure it's empty
 593             d1 = llfuse.listdir(os.path.join(mounttmp))
 594             self.assertEqual([], d1)
 595
 596             # Try to delete it again
 597             with self.assertRaises(OSError):
 598                 os.rmdir(os.path.join(mounttmp, "testdir"))
 599
 600     Test().runTest()
 601
 602 class FuseRmTest(MountTestBase):
 603     def runTest(self):
 604         collection = arvados.collection.Collection(api_client=self.api)
 605         collection.save_new()
 606
 607         m = self.make_mount(fuse.CollectionDirectory)
 608         with llfuse.lock:
 609             m.new_collection(collection.api_response(), collection)
 610         self.assertTrue(m.writable())
 611
 612         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 613
 614         # Starting manifest
 615         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 616         self.assertRegexpMatches(collection2["manifest_text"],
 617             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 618         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 619
 620         # Can't have empty directories :-( so manifest will be empty.
 621         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 622         self.assertEqual(collection2["manifest_text"], "")
 623
 624         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 625
 626         # manifest should be empty now.
 627         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 628         self.assertEqual(collection2["manifest_text"], "")
 629
 630
 631 def fuseMvFileTestHelperWriteFile(mounttmp):
 632     class Test(unittest.TestCase):
 633         def runTest(self):
 634             os.mkdir(os.path.join(mounttmp, "testdir"))
 635
 636             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 637                 f.write("Hello world!")
 638
 639     Test().runTest()
 640
 641 def fuseMvFileTestHelperMoveFile(mounttmp):
 642     class Test(unittest.TestCase):
 643         def runTest(self):
 644             d1 = llfuse.listdir(os.path.join(mounttmp))
 645             self.assertEqual(["testdir"], d1)
 646             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 647             self.assertEqual(["file1.txt"], d1)
 648
 649             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 650
 651             d1 = llfuse.listdir(os.path.join(mounttmp))
 652             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 653             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 654             self.assertEqual([], d1)
 655
 656     Test().runTest()
 657
 658 class FuseMvFileTest(MountTestBase):
 659     def runTest(self):
 660         collection = arvados.collection.Collection(api_client=self.api)
 661         collection.save_new()
 662
 663         m = self.make_mount(fuse.CollectionDirectory)
 664         with llfuse.lock:
 665             m.new_collection(collection.api_response(), collection)
 666         self.assertTrue(m.writable())
 667
 668         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 669
 670         # Starting manifest
 671         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 672         self.assertRegexpMatches(collection2["manifest_text"],
 673             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 674
 675         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 676
 677         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 678         self.assertRegexpMatches(collection2["manifest_text"],
 679             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 680
 681
 682 def fuseRenameTestHelper(mounttmp):
 683     class Test(unittest.TestCase):
 684         def runTest(self):
 685             os.mkdir(os.path.join(mounttmp, "testdir"))
 686
 687             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 688                 f.write("Hello world!")
 689
 690     Test().runTest()
 691
 692 class FuseRenameTest(MountTestBase):
 693     def runTest(self):
 694         collection = arvados.collection.Collection(api_client=self.api)
 695         collection.save_new()
 696
 697         m = self.make_mount(fuse.CollectionDirectory)
 698         with llfuse.lock:
 699             m.new_collection(collection.api_response(), collection)
 700         self.assertTrue(m.writable())
 701
 702         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 703
 704         # Starting manifest
 705         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 706         self.assertRegexpMatches(collection2["manifest_text"],
 707             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 708
 709         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 710         self.assertEqual(["testdir"], d1)
 711         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 712         self.assertEqual(["file1.txt"], d1)
 713
 714         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 715
 716         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 717         self.assertEqual(["testdir2"], sorted(d1))
 718         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 719         self.assertEqual(["file1.txt"], d1)
 720
 721         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 722         self.assertRegexpMatches(collection2["manifest_text"],
 723             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 724
 725
 726 class FuseUpdateFromEventTest(MountTestBase):
 727     def runTest(self):
 728         collection = arvados.collection.Collection(api_client=self.api)
 729         collection.save_new()
 730
 731         m = self.make_mount(fuse.CollectionDirectory)
 732         with llfuse.lock:
 733             m.new_collection(collection.api_response(), collection)
 734
 735         self.operations.listen_for_events()
 736
 737         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 738         self.assertEqual([], sorted(d1))
 739
 740         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 741             with collection2.open("file1.txt", "w") as f:
 742                 f.write("foo")
 743
 744         for attempt in AssertWithTimeout(10):
 745             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 746
 747
 748 def fuseFileConflictTestHelper(mounttmp):
 749     class Test(unittest.TestCase):
 750         def runTest(self):
 751             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 752                 f.write("bar")
 753
 754             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 755             self.assertEqual(len(d1), 2)
 756
 757             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 758                 self.assertEqual(f.read(), "bar")
 759
 760             self.assertRegexpMatches(d1[1],
 761                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 762
 763             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 764                 self.assertEqual(f.read(), "foo")
 765
 766     Test().runTest()
 767
 768 class FuseFileConflictTest(MountTestBase):
 769     def runTest(self):
 770         collection = arvados.collection.Collection(api_client=self.api)
 771         collection.save_new()
 772
 773         m = self.make_mount(fuse.CollectionDirectory)
 774         with llfuse.lock:
 775             m.new_collection(collection.api_response(), collection)
 776
 777         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 778         self.assertEqual([], sorted(d1))
 779
 780         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 781             with collection2.open("file1.txt", "w") as f:
 782                 f.write("foo")
 783
 784         # See note in MountTestBase.setUp
 785         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 786
 787
 788 def fuseUnlinkOpenFileTest(mounttmp):
 789     class Test(unittest.TestCase):
 790         def runTest(self):
 791             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 792                 f.write("foo")
 793
 794                 d1 = llfuse.listdir(os.path.join(mounttmp))
 795                 self.assertEqual(["file1.txt"], sorted(d1))
 796
 797                 os.remove(os.path.join(mounttmp, "file1.txt"))
 798
 799                 d1 = llfuse.listdir(os.path.join(mounttmp))
 800                 self.assertEqual([], sorted(d1))
 801
 802                 f.seek(0)
 803                 self.assertEqual(f.read(), "foo")
 804                 f.write("bar")
 805
 806                 f.seek(0)
 807                 self.assertEqual(f.read(), "foobar")
 808
 809     Test().runTest()
 810
 811 class FuseUnlinkOpenFileTest(MountTestBase):
 812     def runTest(self):
 813         collection = arvados.collection.Collection(api_client=self.api)
 814         collection.save_new()
 815
 816         m = self.make_mount(fuse.CollectionDirectory)
 817         with llfuse.lock:
 818             m.new_collection(collection.api_response(), collection)
 819
 820         # See note in MountTestBase.setUp
 821         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 822
 823         self.assertEqual(collection.manifest_text(), "")
 824
 825
 826 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 827     class Test(unittest.TestCase):
 828         def runTest(self):
 829             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 830                 f.write("Hello world!")
 831
 832             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 833             self.assertEqual(["file1.txt"], sorted(d1))
 834             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 835             self.assertEqual([], sorted(d1))
 836
 837     Test().runTest()
 838
 839 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 840     class Test(unittest.TestCase):
 841         def runTest(self):
 842             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 843
 844             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 845             self.assertEqual([], sorted(d1))
 846             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 847             self.assertEqual(["file2.txt"], sorted(d1))
 848
 849     Test().runTest()
 850
 851 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 852     def runTest(self):
 853         collection1 = arvados.collection.Collection(api_client=self.api)
 854         collection1.save_new()
 855
 856         collection2 = arvados.collection.Collection(api_client=self.api)
 857         collection2.save_new()
 858
 859         m = self.make_mount(fuse.MagicDirectory)
 860
 861         # See note in MountTestBase.setUp
 862         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 863                                                   collection1.manifest_locator(),
 864                                                   collection2.manifest_locator()))
 865
 866         collection1.update()
 867         collection2.update()
 868
 869         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 870         self.assertEqual(collection2.manifest_text(), "")
 871
 872         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 873                                                   collection1.manifest_locator(),
 874                                                   collection2.manifest_locator()))
 875
 876         collection1.update()
 877         collection2.update()
 878
 879         self.assertEqual(collection1.manifest_text(), "")
 880         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 881
 882         collection1.stop_threads()
 883         collection2.stop_threads()
 884
 885
 886 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 887     class Test(unittest.TestCase):
 888         def runTest(self):
 889             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 890             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 891                 f.write("Hello world!")
 892
 893             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 894             self.assertEqual(["testdir"], sorted(d1))
 895             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 896             self.assertEqual(["file1.txt"], sorted(d1))
 897
 898             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 899             self.assertEqual([], sorted(d1))
 900
 901     Test().runTest()
 902
 903
 904 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 905     class Test(unittest.TestCase):
 906         def runTest(self):
 907             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 908
 909             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 910             self.assertEqual([], sorted(d1))
 911
 912             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 913             self.assertEqual(["testdir2"], sorted(d1))
 914             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 915             self.assertEqual(["file1.txt"], sorted(d1))
 916
 917             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 918                 self.assertEqual(f.read(), "Hello world!")
 919
 920     Test().runTest()
 921
 922 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 923     def runTest(self):
 924         collection1 = arvados.collection.Collection(api_client=self.api)
 925         collection1.save_new()
 926
 927         collection2 = arvados.collection.Collection(api_client=self.api)
 928         collection2.save_new()
 929
 930         m = self.make_mount(fuse.MagicDirectory)
 931
 932         # See note in MountTestBase.setUp
 933         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 934                                                   collection1.manifest_locator(),
 935                                                   collection2.manifest_locator()))
 936
 937         collection1.update()
 938         collection2.update()
 939
 940         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 941         self.assertEqual(collection2.manifest_text(), "")
 942
 943         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 944                                                   collection1.manifest_locator(),
 945                                                   collection2.manifest_locator()))
 946
 947         collection1.update()
 948         collection2.update()
 949
 950         self.assertEqual(collection1.manifest_text(), "")
 951         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 952
 953         collection1.stop_threads()
 954         collection2.stop_threads()
 955
 956 def fuseProjectMkdirTestHelper1(mounttmp):
 957     class Test(unittest.TestCase):
 958         def runTest(self):
 959             os.mkdir(os.path.join(mounttmp, "testcollection"))
 960             with self.assertRaises(OSError):
 961                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 962     Test().runTest()
 963
 964 def fuseProjectMkdirTestHelper2(mounttmp):
 965     class Test(unittest.TestCase):
 966         def runTest(self):
 967             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 968                 f.write("Hello world!")
 969             with self.assertRaises(OSError):
 970                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 971             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 972             with self.assertRaises(OSError):
 973                 os.remove(os.path.join(mounttmp, "testcollection"))
 974             os.rmdir(os.path.join(mounttmp, "testcollection"))
 975     Test().runTest()
 976
 977 class FuseProjectMkdirRmdirTest(MountTestBase):
 978     def runTest(self):
 979         self.make_mount(fuse.ProjectDirectory,
 980                         project_object=self.api.users().current().execute())
 981
 982         d1 = llfuse.listdir(self.mounttmp)
 983         self.assertNotIn('testcollection', d1)
 984
 985         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
 986
 987         d1 = llfuse.listdir(self.mounttmp)
 988         self.assertIn('testcollection', d1)
 989
 990         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
 991
 992         d1 = llfuse.listdir(self.mounttmp)
 993         self.assertNotIn('testcollection', d1)
 994
 995
 996 def fuseProjectMvTestHelper1(mounttmp):
 997     class Test(unittest.TestCase):
 998         def runTest(self):
 999             d1 = llfuse.listdir(mounttmp)
1000             self.assertNotIn('testcollection', d1)
1001
1002             os.mkdir(os.path.join(mounttmp, "testcollection"))
1003
1004             d1 = llfuse.listdir(mounttmp)
1005             self.assertIn('testcollection', d1)
1006
1007             with self.assertRaises(OSError):
1008                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1009
1010             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1011
1012             d1 = llfuse.listdir(mounttmp)
1013             self.assertNotIn('testcollection', d1)
1014
1015             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1016             self.assertIn('testcollection', d1)
1017
1018     Test().runTest()
1019
1020 class FuseProjectMvTest(MountTestBase):
1021     def runTest(self):
1022         self.make_mount(fuse.ProjectDirectory,
1023                         project_object=self.api.users().current().execute())
1024
1025         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1026
1027
1028 def fuseFsyncTestHelper(mounttmp, k):
1029     class Test(unittest.TestCase):
1030         def runTest(self):
1031             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1032             os.fsync(fd)
1033             os.close(fd)
1034
1035     Test().runTest()
1036
1037 class FuseFsyncTest(FuseMagicTest):
1038     def runTest(self):
1039         self.make_mount(fuse.MagicDirectory)
1040         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1041
1042
1043 class MagicDirApiError(FuseMagicTest):
1044     def setUp(self):
1045         api = mock.MagicMock()
1046         super(MagicDirApiError, self).setUp(api=api)
1047         api.collections().get().execute.side_effect = iter([Exception('API fail'), {"manifest_text": self.test_manifest}])
1048         api.keep.get.side_effect = Exception('Keep fail')
1049
1050     def runTest(self):
1051         self.make_mount(fuse.MagicDirectory)
1052
1053         self.operations.inodes.inode_cache.cap = 1
1054         self.operations.inodes.inode_cache.min_entries = 2
1055
1056         with self.assertRaises(OSError):
1057             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1058
1059         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1060
1061
1062 class FuseUnitTest(unittest.TestCase):
1063     def test_sanitize_filename(self):
1064         acceptable = [
1065             "foo.txt",
1066             ".foo",
1067             "..foo",
1068             "...",
1069             "foo...",
1070             "foo..",
1071             "foo.",
1072             "-",
1073             "\x01\x02\x03",
1074             ]
1075         unacceptable = [
1076             "f\00",
1077             "\00\00",
1078             "/foo",
1079             "foo/",
1080             "//",
1081             ]
1082         for f in acceptable:
1083             self.assertEqual(f, fuse.sanitize_filename(f))
1084         for f in unacceptable:
1085             self.assertNotEqual(f, fuse.sanitize_filename(f))
1086             # The sanitized filename should be the same length, though.
1087             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1088         # Special cases
1089         self.assertEqual("_", fuse.sanitize_filename(""))
1090         self.assertEqual("_", fuse.sanitize_filename("."))
1091         self.assertEqual("__", fuse.sanitize_filename(".."))
1092
1093
1094 class FuseMagicTestPDHOnly(MountTestBase):
1095     def setUp(self, api=None):
1096         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1097
1098         cw = arvados.CollectionWriter()
1099
1100         cw.start_new_file('thing1.txt')
1101         cw.write("data 1")
1102
1103         self.testcollection = cw.finish()
1104         self.test_manifest = cw.manifest_text()
1105         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1106         self.testcollectionuuid = str(created['uuid'])
1107
1108     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1109         if skip_pdh_only is True:
1110             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1111         else:
1112             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1113
1114         mount_ls = llfuse.listdir(self.mounttmp)
1115         self.assertIn('README', mount_ls)
1116         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1117                              arvados.util.uuid_pattern.match(fn)
1118                              for fn in mount_ls),
1119                          "new FUSE MagicDirectory lists Collection")
1120
1121         # look up using pdh should succeed in all cases
1122         self.assertDirContents(self.testcollection, ['thing1.txt'])
1123         self.assertDirContents(os.path.join('by_id', self.testcollection),
1124                                ['thing1.txt'])
1125         mount_ls = llfuse.listdir(self.mounttmp)
1126         self.assertIn('README', mount_ls)
1127         self.assertIn(self.testcollection, mount_ls)
1128         self.assertIn(self.testcollection,
1129                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1130
1131         files = {}
1132         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1133
1134         for k, v in files.items():
1135             with open(os.path.join(self.mounttmp, k)) as f:
1136                 self.assertEqual(v, f.read())
1137
1138         # look up using uuid should fail when pdh_only is set
1139         if pdh_only is True:
1140             with self.assertRaises(OSError):
1141                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1142                                ['thing1.txt'])
1143         else:
1144             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1145                                ['thing1.txt'])
1146
1147     def test_with_pdh_only_true(self):
1148         self.verify_pdh_only(pdh_only=True)
1149
1150     def test_with_pdh_only_false(self):
1151         self.verify_pdh_only(pdh_only=False)
1152
1153     def test_with_default_by_id(self):
1154         self.verify_pdh_only(skip_pdh_only=True)