services/fuse/tests/test_mount.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 import json
   6 import llfuse
   7 import logging
   8 import mock
   9 import os
  10 import subprocess
  11 import time
  12 import unittest
  13
  14 import arvados
  15 import arvados_fuse as fuse
  16 import run_test_server
  17
  18 from mount_test_base import MountTestBase
  19
  20 logger = logging.getLogger('arvados.arv-mount')
  21
  22
  23 class AssertWithTimeout(object):
  24     """Allow some time for an assertion to pass."""
  25
  26     def __init__(self, timeout=0):
  27         self.timeout = timeout
  28
  29     def __iter__(self):
  30         self.deadline = time.time() + self.timeout
  31         self.done = False
  32         return self
  33
  34     def next(self):
  35         if self.done:
  36             raise StopIteration
  37         return self.attempt
  38
  39     def attempt(self, fn, *args, **kwargs):
  40         try:
  41             fn(*args, **kwargs)
  42         except AssertionError:
  43             if time.time() > self.deadline:
  44                 raise
  45             time.sleep(0.1)
  46         else:
  47             self.done = True
  48
  49
  50 class FuseMountTest(MountTestBase):
  51     def setUp(self):
  52         super(FuseMountTest, self).setUp()
  53
  54         cw = arvados.CollectionWriter()
  55
  56         cw.start_new_file('thing1.txt')
  57         cw.write("data 1")
  58         cw.start_new_file('thing2.txt')
  59         cw.write("data 2")
  60
  61         cw.start_new_stream('dir1')
  62         cw.start_new_file('thing3.txt')
  63         cw.write("data 3")
  64         cw.start_new_file('thing4.txt')
  65         cw.write("data 4")
  66
  67         cw.start_new_stream('dir2')
  68         cw.start_new_file('thing5.txt')
  69         cw.write("data 5")
  70         cw.start_new_file('thing6.txt')
  71         cw.write("data 6")
  72
  73         cw.start_new_stream('dir2/dir3')
  74         cw.start_new_file('thing7.txt')
  75         cw.write("data 7")
  76
  77         cw.start_new_file('thing8.txt')
  78         cw.write("data 8")
  79
  80         cw.start_new_stream('edgecases')
  81         for f in ":/.../-/*/\x01\\/ ".split("/"):
  82             cw.start_new_file(f)
  83             cw.write('x')
  84
  85         for f in ":/.../-/*/\x01\\/ ".split("/"):
  86             cw.start_new_stream('edgecases/dirs/' + f)
  87             cw.start_new_file('x/x')
  88             cw.write('x')
  89
  90         self.testcollection = cw.finish()
  91         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  92
  93     def runTest(self):
  94         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  95
  96         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
  97                                       'edgecases', 'dir1', 'dir2'])
  98         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
  99         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 100         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 101         self.assertDirContents('edgecases',
 102                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
 103         self.assertDirContents('edgecases/dirs',
 104                                ":/.../-/*/\x01\\/ ".split("/"))
 105
 106         files = {'thing1.txt': 'data 1',
 107                  'thing2.txt': 'data 2',
 108                  'dir1/thing3.txt': 'data 3',
 109                  'dir1/thing4.txt': 'data 4',
 110                  'dir2/thing5.txt': 'data 5',
 111                  'dir2/thing6.txt': 'data 6',
 112                  'dir2/dir3/thing7.txt': 'data 7',
 113                  'dir2/dir3/thing8.txt': 'data 8'}
 114
 115         for k, v in files.items():
 116             with open(os.path.join(self.mounttmp, k)) as f:
 117                 self.assertEqual(v, f.read())
 118
 119
 120 class FuseNoAPITest(MountTestBase):
 121     def setUp(self):
 122         super(FuseNoAPITest, self).setUp()
 123         keep = arvados.keep.KeepClient(local_store=self.keeptmp)
 124         self.file_data = "API-free text\n"
 125         self.file_loc = keep.put(self.file_data)
 126         self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
 127                 self.file_loc, len(self.file_data)))
 128
 129     def runTest(self):
 130         self.make_mount(fuse.MagicDirectory)
 131         self.assertDirContents(self.coll_loc, ['api-free.txt'])
 132         with open(os.path.join(
 133                 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
 134             actual = keep_file.read(-1)
 135         self.assertEqual(self.file_data, actual)
 136
 137
 138 class FuseMagicTest(MountTestBase):
 139     def setUp(self, api=None):
 140         super(FuseMagicTest, self).setUp(api=api)
 141
 142         cw = arvados.CollectionWriter()
 143
 144         cw.start_new_file('thing1.txt')
 145         cw.write("data 1")
 146
 147         self.testcollection = cw.finish()
 148         self.test_manifest = cw.manifest_text()
 149         self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 150
 151     def runTest(self):
 152         self.make_mount(fuse.MagicDirectory)
 153
 154         mount_ls = llfuse.listdir(self.mounttmp)
 155         self.assertIn('README', mount_ls)
 156         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 157                              arvados.util.uuid_pattern.match(fn)
 158                              for fn in mount_ls),
 159                          "new FUSE MagicDirectory lists Collection")
 160         self.assertDirContents(self.testcollection, ['thing1.txt'])
 161         self.assertDirContents(os.path.join('by_id', self.testcollection),
 162                                ['thing1.txt'])
 163         mount_ls = llfuse.listdir(self.mounttmp)
 164         self.assertIn('README', mount_ls)
 165         self.assertIn(self.testcollection, mount_ls)
 166         self.assertIn(self.testcollection,
 167                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 168
 169         files = {}
 170         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 171
 172         for k, v in files.items():
 173             with open(os.path.join(self.mounttmp, k)) as f:
 174                 self.assertEqual(v, f.read())
 175
 176
 177 class FuseTagsTest(MountTestBase):
 178     def runTest(self):
 179         self.make_mount(fuse.TagsDirectory)
 180
 181         d1 = llfuse.listdir(self.mounttmp)
 182         d1.sort()
 183         self.assertEqual(['foo_tag'], d1)
 184
 185         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 186         d2.sort()
 187         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 188
 189         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 190         d3.sort()
 191         self.assertEqual(['foo'], d3)
 192
 193
 194 class FuseTagsUpdateTest(MountTestBase):
 195     def tag_collection(self, coll_uuid, tag_name):
 196         return self.api.links().create(
 197             body={'link': {'head_uuid': coll_uuid,
 198                            'link_class': 'tag',
 199                            'name': tag_name,
 200         }}).execute()
 201
 202     def runTest(self):
 203         self.make_mount(fuse.TagsDirectory, poll_time=1)
 204
 205         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 206
 207         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 208         self.tag_collection(bar_uuid, 'fuse_test_tag')
 209         for attempt in AssertWithTimeout(10):
 210             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 211         self.assertDirContents('fuse_test_tag', [bar_uuid])
 212
 213         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 214         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 215         for attempt in AssertWithTimeout(10):
 216             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 217
 218         self.api.links().delete(uuid=l['uuid']).execute()
 219         for attempt in AssertWithTimeout(10):
 220             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 221
 222
 223 class FuseSharedTest(MountTestBase):
 224     def runTest(self):
 225         self.make_mount(fuse.SharedDirectory,
 226                         exclude=self.api.users().current().execute()['uuid'])
 227
 228         # shared_dirs is a list of the directories exposed
 229         # by fuse.SharedDirectory (i.e. any object visible
 230         # to the current user)
 231         shared_dirs = llfuse.listdir(self.mounttmp)
 232         shared_dirs.sort()
 233         self.assertIn('FUSE User', shared_dirs)
 234
 235         # fuse_user_objs is a list of the objects owned by the FUSE
 236         # test user (which present as files in the 'FUSE User'
 237         # directory)
 238         fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
 239         fuse_user_objs.sort()
 240         self.assertEqual(['FUSE Test Project',                    # project owned by user
 241                           'collection #1 owned by FUSE',          # collection owned by user
 242                           'collection #2 owned by FUSE',          # collection owned by user
 243                           'pipeline instance owned by FUSE.pipelineInstance',  # pipeline instance owned by user
 244                       ], fuse_user_objs)
 245
 246         # test_proj_files is a list of the files in the FUSE Test Project.
 247         test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
 248         test_proj_files.sort()
 249         self.assertEqual(['collection in FUSE project',
 250                           'pipeline instance in FUSE project.pipelineInstance',
 251                           'pipeline template in FUSE project.pipelineTemplate'
 252                       ], test_proj_files)
 253
 254         # Double check that we can open and read objects in this folder as a file,
 255         # and that its contents are what we expect.
 256         pipeline_template_path = os.path.join(
 257                 self.mounttmp,
 258                 'FUSE User',
 259                 'FUSE Test Project',
 260                 'pipeline template in FUSE project.pipelineTemplate')
 261         with open(pipeline_template_path) as f:
 262             j = json.load(f)
 263             self.assertEqual("pipeline template in FUSE project", j['name'])
 264
 265         # check mtime on template
 266         st = os.stat(pipeline_template_path)
 267         try:
 268             mtime = st.st_mtime_ns / 1000000000
 269         except AttributeError:
 270             mtime = st.st_mtime
 271         self.assertEqual(mtime, 1397493304)
 272
 273         # check mtime on collection
 274         st = os.stat(os.path.join(
 275                 self.mounttmp,
 276                 'FUSE User',
 277                 'collection #1 owned by FUSE'))
 278         try:
 279             mtime = st.st_mtime_ns / 1000000000
 280         except AttributeError:
 281             mtime = st.st_mtime
 282         self.assertEqual(mtime, 1391448174)
 283
 284
 285 class FuseHomeTest(MountTestBase):
 286     def runTest(self):
 287         self.make_mount(fuse.ProjectDirectory,
 288                         project_object=self.api.users().current().execute())
 289
 290         d1 = llfuse.listdir(self.mounttmp)
 291         self.assertIn('Unrestricted public data', d1)
 292
 293         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 294         public_project = run_test_server.fixture('groups')[
 295             'anonymously_accessible_project']
 296         found_in = 0
 297         found_not_in = 0
 298         for name, item in run_test_server.fixture('collections').iteritems():
 299             if 'name' not in item:
 300                 pass
 301             elif item['owner_uuid'] == public_project['uuid']:
 302                 self.assertIn(item['name'], d2)
 303                 found_in += 1
 304             else:
 305                 # Artificial assumption here: there is no public
 306                 # collection fixture with the same name as a
 307                 # non-public collection.
 308                 self.assertNotIn(item['name'], d2)
 309                 found_not_in += 1
 310         self.assertNotEqual(0, found_in)
 311         self.assertNotEqual(0, found_not_in)
 312
 313         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 314         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 315
 316
 317 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 318     class Test(unittest.TestCase):
 319         def runTest(self):
 320             d1 = llfuse.listdir(mounttmp)
 321             self.assertEqual(["file1.txt"], d1)
 322             with open(os.path.join(mounttmp, "file1.txt")) as f:
 323                 self.assertEqual("blub", f.read())
 324     Test().runTest()
 325
 326 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 327     class Test(unittest.TestCase):
 328         def runTest(self):
 329             d1 = llfuse.listdir(mounttmp)
 330             self.assertEqual(["file1.txt"], d1)
 331             with open(os.path.join(mounttmp, "file1.txt")) as f:
 332                 self.assertEqual("plnp", f.read())
 333     Test().runTest()
 334
 335 class FuseModifyFileTest(MountTestBase):
 336     def runTest(self):
 337         collection = arvados.collection.Collection(api_client=self.api)
 338         with collection.open("file1.txt", "w") as f:
 339             f.write("blub")
 340
 341         collection.save_new()
 342
 343         m = self.make_mount(fuse.CollectionDirectory)
 344         with llfuse.lock:
 345             m.new_collection(collection.api_response(), collection)
 346
 347         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 348
 349         with collection.open("file1.txt", "w") as f:
 350             f.write("plnp")
 351
 352         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 353
 354
 355 class FuseAddFileToCollectionTest(MountTestBase):
 356     def runTest(self):
 357         collection = arvados.collection.Collection(api_client=self.api)
 358         with collection.open("file1.txt", "w") as f:
 359             f.write("blub")
 360
 361         collection.save_new()
 362
 363         m = self.make_mount(fuse.CollectionDirectory)
 364         with llfuse.lock:
 365             m.new_collection(collection.api_response(), collection)
 366
 367         d1 = llfuse.listdir(self.mounttmp)
 368         self.assertEqual(["file1.txt"], d1)
 369
 370         with collection.open("file2.txt", "w") as f:
 371             f.write("plnp")
 372
 373         d1 = llfuse.listdir(self.mounttmp)
 374         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 375
 376
 377 class FuseRemoveFileFromCollectionTest(MountTestBase):
 378     def runTest(self):
 379         collection = arvados.collection.Collection(api_client=self.api)
 380         with collection.open("file1.txt", "w") as f:
 381             f.write("blub")
 382
 383         with collection.open("file2.txt", "w") as f:
 384             f.write("plnp")
 385
 386         collection.save_new()
 387
 388         m = self.make_mount(fuse.CollectionDirectory)
 389         with llfuse.lock:
 390             m.new_collection(collection.api_response(), collection)
 391
 392         d1 = llfuse.listdir(self.mounttmp)
 393         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 394
 395         collection.remove("file2.txt")
 396
 397         d1 = llfuse.listdir(self.mounttmp)
 398         self.assertEqual(["file1.txt"], d1)
 399
 400
 401 def fuseCreateFileTestHelper(mounttmp):
 402     class Test(unittest.TestCase):
 403         def runTest(self):
 404             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 405                 pass
 406     Test().runTest()
 407
 408 class FuseCreateFileTest(MountTestBase):
 409     def runTest(self):
 410         collection = arvados.collection.Collection(api_client=self.api)
 411         collection.save_new()
 412
 413         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 414         self.assertEqual(collection2["manifest_text"], "")
 415
 416         collection.save_new()
 417
 418         m = self.make_mount(fuse.CollectionDirectory)
 419         with llfuse.lock:
 420             m.new_collection(collection.api_response(), collection)
 421         self.assertTrue(m.writable())
 422
 423         self.assertNotIn("file1.txt", collection)
 424
 425         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 426
 427         self.assertIn("file1.txt", collection)
 428
 429         d1 = llfuse.listdir(self.mounttmp)
 430         self.assertEqual(["file1.txt"], d1)
 431
 432         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 433         self.assertRegexpMatches(collection2["manifest_text"],
 434             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 435
 436
 437 def fuseWriteFileTestHelperWriteFile(mounttmp):
 438     class Test(unittest.TestCase):
 439         def runTest(self):
 440             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 441                 f.write("Hello world!")
 442     Test().runTest()
 443
 444 def fuseWriteFileTestHelperReadFile(mounttmp):
 445     class Test(unittest.TestCase):
 446         def runTest(self):
 447             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 448                 self.assertEqual(f.read(), "Hello world!")
 449     Test().runTest()
 450
 451 class FuseWriteFileTest(MountTestBase):
 452     def runTest(self):
 453         collection = arvados.collection.Collection(api_client=self.api)
 454         collection.save_new()
 455
 456         m = self.make_mount(fuse.CollectionDirectory)
 457         with llfuse.lock:
 458             m.new_collection(collection.api_response(), collection)
 459         self.assertTrue(m.writable())
 460
 461         self.assertNotIn("file1.txt", collection)
 462
 463         self.assertEqual(0, self.operations.write_counter.get())
 464         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 465         self.assertEqual(12, self.operations.write_counter.get())
 466
 467         with collection.open("file1.txt") as f:
 468             self.assertEqual(f.read(), "Hello world!")
 469
 470         self.assertEqual(0, self.operations.read_counter.get())
 471         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 472         self.assertEqual(12, self.operations.read_counter.get())
 473
 474         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 475         self.assertRegexpMatches(collection2["manifest_text"],
 476             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 477
 478
 479 def fuseUpdateFileTestHelper(mounttmp):
 480     class Test(unittest.TestCase):
 481         def runTest(self):
 482             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 483                 f.write("Hello world!")
 484
 485             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 486                 fr = f.read()
 487                 self.assertEqual(fr, "Hello world!")
 488                 f.seek(0)
 489                 f.write("Hola mundo!")
 490                 f.seek(0)
 491                 fr = f.read()
 492                 self.assertEqual(fr, "Hola mundo!!")
 493
 494             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 495                 self.assertEqual(f.read(), "Hola mundo!!")
 496
 497     Test().runTest()
 498
 499 class FuseUpdateFileTest(MountTestBase):
 500     def runTest(self):
 501         collection = arvados.collection.Collection(api_client=self.api)
 502         collection.save_new()
 503
 504         m = self.make_mount(fuse.CollectionDirectory)
 505         with llfuse.lock:
 506             m.new_collection(collection.api_response(), collection)
 507         self.assertTrue(m.writable())
 508
 509         # See note in MountTestBase.setUp
 510         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 511
 512         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 513         self.assertRegexpMatches(collection2["manifest_text"],
 514             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 515
 516
 517 def fuseMkdirTestHelper(mounttmp):
 518     class Test(unittest.TestCase):
 519         def runTest(self):
 520             with self.assertRaises(IOError):
 521                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 522                     f.write("Hello world!")
 523
 524             os.mkdir(os.path.join(mounttmp, "testdir"))
 525
 526             with self.assertRaises(OSError):
 527                 os.mkdir(os.path.join(mounttmp, "testdir"))
 528
 529             d1 = llfuse.listdir(mounttmp)
 530             self.assertEqual(["testdir"], d1)
 531
 532             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 533                 f.write("Hello world!")
 534
 535             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 536             self.assertEqual(["file1.txt"], d1)
 537
 538     Test().runTest()
 539
 540 class FuseMkdirTest(MountTestBase):
 541     def runTest(self):
 542         collection = arvados.collection.Collection(api_client=self.api)
 543         collection.save_new()
 544
 545         m = self.make_mount(fuse.CollectionDirectory)
 546         with llfuse.lock:
 547             m.new_collection(collection.api_response(), collection)
 548         self.assertTrue(m.writable())
 549
 550         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 551
 552         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 553         self.assertRegexpMatches(collection2["manifest_text"],
 554             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 555
 556
 557 def fuseRmTestHelperWriteFile(mounttmp):
 558     class Test(unittest.TestCase):
 559         def runTest(self):
 560             os.mkdir(os.path.join(mounttmp, "testdir"))
 561
 562             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 563                 f.write("Hello world!")
 564
 565     Test().runTest()
 566
 567 def fuseRmTestHelperDeleteFile(mounttmp):
 568     class Test(unittest.TestCase):
 569         def runTest(self):
 570             # Can't delete because it's not empty
 571             with self.assertRaises(OSError):
 572                 os.rmdir(os.path.join(mounttmp, "testdir"))
 573
 574             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 575             self.assertEqual(["file1.txt"], d1)
 576
 577             # Delete file
 578             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 579
 580             # Make sure it's empty
 581             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 582             self.assertEqual([], d1)
 583
 584             # Try to delete it again
 585             with self.assertRaises(OSError):
 586                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 587
 588     Test().runTest()
 589
 590 def fuseRmTestHelperRmdir(mounttmp):
 591     class Test(unittest.TestCase):
 592         def runTest(self):
 593             # Should be able to delete now that it is empty
 594             os.rmdir(os.path.join(mounttmp, "testdir"))
 595
 596             # Make sure it's empty
 597             d1 = llfuse.listdir(os.path.join(mounttmp))
 598             self.assertEqual([], d1)
 599
 600             # Try to delete it again
 601             with self.assertRaises(OSError):
 602                 os.rmdir(os.path.join(mounttmp, "testdir"))
 603
 604     Test().runTest()
 605
 606 class FuseRmTest(MountTestBase):
 607     def runTest(self):
 608         collection = arvados.collection.Collection(api_client=self.api)
 609         collection.save_new()
 610
 611         m = self.make_mount(fuse.CollectionDirectory)
 612         with llfuse.lock:
 613             m.new_collection(collection.api_response(), collection)
 614         self.assertTrue(m.writable())
 615
 616         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 617
 618         # Starting manifest
 619         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 620         self.assertRegexpMatches(collection2["manifest_text"],
 621             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 622         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 623
 624         # Can't have empty directories :-( so manifest will be empty.
 625         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 626         self.assertEqual(collection2["manifest_text"], "")
 627
 628         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 629
 630         # manifest should be empty now.
 631         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 632         self.assertEqual(collection2["manifest_text"], "")
 633
 634
 635 def fuseMvFileTestHelperWriteFile(mounttmp):
 636     class Test(unittest.TestCase):
 637         def runTest(self):
 638             os.mkdir(os.path.join(mounttmp, "testdir"))
 639
 640             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 641                 f.write("Hello world!")
 642
 643     Test().runTest()
 644
 645 def fuseMvFileTestHelperMoveFile(mounttmp):
 646     class Test(unittest.TestCase):
 647         def runTest(self):
 648             d1 = llfuse.listdir(os.path.join(mounttmp))
 649             self.assertEqual(["testdir"], d1)
 650             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 651             self.assertEqual(["file1.txt"], d1)
 652
 653             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 654
 655             d1 = llfuse.listdir(os.path.join(mounttmp))
 656             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 657             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 658             self.assertEqual([], d1)
 659
 660     Test().runTest()
 661
 662 class FuseMvFileTest(MountTestBase):
 663     def runTest(self):
 664         collection = arvados.collection.Collection(api_client=self.api)
 665         collection.save_new()
 666
 667         m = self.make_mount(fuse.CollectionDirectory)
 668         with llfuse.lock:
 669             m.new_collection(collection.api_response(), collection)
 670         self.assertTrue(m.writable())
 671
 672         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 673
 674         # Starting manifest
 675         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 676         self.assertRegexpMatches(collection2["manifest_text"],
 677             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 678
 679         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 680
 681         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 682         self.assertRegexpMatches(collection2["manifest_text"],
 683             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 684
 685
 686 def fuseRenameTestHelper(mounttmp):
 687     class Test(unittest.TestCase):
 688         def runTest(self):
 689             os.mkdir(os.path.join(mounttmp, "testdir"))
 690
 691             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 692                 f.write("Hello world!")
 693
 694     Test().runTest()
 695
 696 class FuseRenameTest(MountTestBase):
 697     def runTest(self):
 698         collection = arvados.collection.Collection(api_client=self.api)
 699         collection.save_new()
 700
 701         m = self.make_mount(fuse.CollectionDirectory)
 702         with llfuse.lock:
 703             m.new_collection(collection.api_response(), collection)
 704         self.assertTrue(m.writable())
 705
 706         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 707
 708         # Starting manifest
 709         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 710         self.assertRegexpMatches(collection2["manifest_text"],
 711             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 712
 713         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 714         self.assertEqual(["testdir"], d1)
 715         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 716         self.assertEqual(["file1.txt"], d1)
 717
 718         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 719
 720         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 721         self.assertEqual(["testdir2"], sorted(d1))
 722         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 723         self.assertEqual(["file1.txt"], d1)
 724
 725         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 726         self.assertRegexpMatches(collection2["manifest_text"],
 727             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 728
 729
 730 class FuseUpdateFromEventTest(MountTestBase):
 731     def runTest(self):
 732         collection = arvados.collection.Collection(api_client=self.api)
 733         collection.save_new()
 734
 735         m = self.make_mount(fuse.CollectionDirectory)
 736         with llfuse.lock:
 737             m.new_collection(collection.api_response(), collection)
 738
 739         self.operations.listen_for_events()
 740
 741         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 742         self.assertEqual([], sorted(d1))
 743
 744         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 745             with collection2.open("file1.txt", "w") as f:
 746                 f.write("foo")
 747
 748         for attempt in AssertWithTimeout(10):
 749             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 750
 751
 752 def fuseFileConflictTestHelper(mounttmp):
 753     class Test(unittest.TestCase):
 754         def runTest(self):
 755             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 756                 f.write("bar")
 757
 758             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 759             self.assertEqual(len(d1), 2)
 760
 761             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 762                 self.assertEqual(f.read(), "bar")
 763
 764             self.assertRegexpMatches(d1[1],
 765                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 766
 767             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 768                 self.assertEqual(f.read(), "foo")
 769
 770     Test().runTest()
 771
 772 class FuseFileConflictTest(MountTestBase):
 773     def runTest(self):
 774         collection = arvados.collection.Collection(api_client=self.api)
 775         collection.save_new()
 776
 777         m = self.make_mount(fuse.CollectionDirectory)
 778         with llfuse.lock:
 779             m.new_collection(collection.api_response(), collection)
 780
 781         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 782         self.assertEqual([], sorted(d1))
 783
 784         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 785             with collection2.open("file1.txt", "w") as f:
 786                 f.write("foo")
 787
 788         # See note in MountTestBase.setUp
 789         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 790
 791
 792 def fuseUnlinkOpenFileTest(mounttmp):
 793     class Test(unittest.TestCase):
 794         def runTest(self):
 795             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 796                 f.write("foo")
 797
 798                 d1 = llfuse.listdir(os.path.join(mounttmp))
 799                 self.assertEqual(["file1.txt"], sorted(d1))
 800
 801                 os.remove(os.path.join(mounttmp, "file1.txt"))
 802
 803                 d1 = llfuse.listdir(os.path.join(mounttmp))
 804                 self.assertEqual([], sorted(d1))
 805
 806                 f.seek(0)
 807                 self.assertEqual(f.read(), "foo")
 808                 f.write("bar")
 809
 810                 f.seek(0)
 811                 self.assertEqual(f.read(), "foobar")
 812
 813     Test().runTest()
 814
 815 class FuseUnlinkOpenFileTest(MountTestBase):
 816     def runTest(self):
 817         collection = arvados.collection.Collection(api_client=self.api)
 818         collection.save_new()
 819
 820         m = self.make_mount(fuse.CollectionDirectory)
 821         with llfuse.lock:
 822             m.new_collection(collection.api_response(), collection)
 823
 824         # See note in MountTestBase.setUp
 825         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 826
 827         self.assertEqual(collection.manifest_text(), "")
 828
 829
 830 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 831     class Test(unittest.TestCase):
 832         def runTest(self):
 833             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 834                 f.write("Hello world!")
 835
 836             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 837             self.assertEqual(["file1.txt"], sorted(d1))
 838             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 839             self.assertEqual([], sorted(d1))
 840
 841     Test().runTest()
 842
 843 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 844     class Test(unittest.TestCase):
 845         def runTest(self):
 846             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 847
 848             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 849             self.assertEqual([], sorted(d1))
 850             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 851             self.assertEqual(["file2.txt"], sorted(d1))
 852
 853     Test().runTest()
 854
 855 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 856     def runTest(self):
 857         collection1 = arvados.collection.Collection(api_client=self.api)
 858         collection1.save_new()
 859
 860         collection2 = arvados.collection.Collection(api_client=self.api)
 861         collection2.save_new()
 862
 863         m = self.make_mount(fuse.MagicDirectory)
 864
 865         # See note in MountTestBase.setUp
 866         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 867                                                   collection1.manifest_locator(),
 868                                                   collection2.manifest_locator()))
 869
 870         collection1.update()
 871         collection2.update()
 872
 873         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 874         self.assertEqual(collection2.manifest_text(), "")
 875
 876         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 877                                                   collection1.manifest_locator(),
 878                                                   collection2.manifest_locator()))
 879
 880         collection1.update()
 881         collection2.update()
 882
 883         self.assertEqual(collection1.manifest_text(), "")
 884         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 885
 886         collection1.stop_threads()
 887         collection2.stop_threads()
 888
 889
 890 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 891     class Test(unittest.TestCase):
 892         def runTest(self):
 893             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 894             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 895                 f.write("Hello world!")
 896
 897             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 898             self.assertEqual(["testdir"], sorted(d1))
 899             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 900             self.assertEqual(["file1.txt"], sorted(d1))
 901
 902             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 903             self.assertEqual([], sorted(d1))
 904
 905     Test().runTest()
 906
 907
 908 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 909     class Test(unittest.TestCase):
 910         def runTest(self):
 911             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 912
 913             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 914             self.assertEqual([], sorted(d1))
 915
 916             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 917             self.assertEqual(["testdir2"], sorted(d1))
 918             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 919             self.assertEqual(["file1.txt"], sorted(d1))
 920
 921             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 922                 self.assertEqual(f.read(), "Hello world!")
 923
 924     Test().runTest()
 925
 926 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 927     def runTest(self):
 928         collection1 = arvados.collection.Collection(api_client=self.api)
 929         collection1.save_new()
 930
 931         collection2 = arvados.collection.Collection(api_client=self.api)
 932         collection2.save_new()
 933
 934         m = self.make_mount(fuse.MagicDirectory)
 935
 936         # See note in MountTestBase.setUp
 937         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 938                                                   collection1.manifest_locator(),
 939                                                   collection2.manifest_locator()))
 940
 941         collection1.update()
 942         collection2.update()
 943
 944         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 945         self.assertEqual(collection2.manifest_text(), "")
 946
 947         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 948                                                   collection1.manifest_locator(),
 949                                                   collection2.manifest_locator()))
 950
 951         collection1.update()
 952         collection2.update()
 953
 954         self.assertEqual(collection1.manifest_text(), "")
 955         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 956
 957         collection1.stop_threads()
 958         collection2.stop_threads()
 959
 960 def fuseProjectMkdirTestHelper1(mounttmp):
 961     class Test(unittest.TestCase):
 962         def runTest(self):
 963             os.mkdir(os.path.join(mounttmp, "testcollection"))
 964             with self.assertRaises(OSError):
 965                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 966     Test().runTest()
 967
 968 def fuseProjectMkdirTestHelper2(mounttmp):
 969     class Test(unittest.TestCase):
 970         def runTest(self):
 971             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 972                 f.write("Hello world!")
 973             with self.assertRaises(OSError):
 974                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 975             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 976             with self.assertRaises(OSError):
 977                 os.remove(os.path.join(mounttmp, "testcollection"))
 978             os.rmdir(os.path.join(mounttmp, "testcollection"))
 979     Test().runTest()
 980
 981 class FuseProjectMkdirRmdirTest(MountTestBase):
 982     def runTest(self):
 983         self.make_mount(fuse.ProjectDirectory,
 984                         project_object=self.api.users().current().execute())
 985
 986         d1 = llfuse.listdir(self.mounttmp)
 987         self.assertNotIn('testcollection', d1)
 988
 989         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
 990
 991         d1 = llfuse.listdir(self.mounttmp)
 992         self.assertIn('testcollection', d1)
 993
 994         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
 995
 996         d1 = llfuse.listdir(self.mounttmp)
 997         self.assertNotIn('testcollection', d1)
 998
 999
1000 def fuseProjectMvTestHelper1(mounttmp):
1001     class Test(unittest.TestCase):
1002         def runTest(self):
1003             d1 = llfuse.listdir(mounttmp)
1004             self.assertNotIn('testcollection', d1)
1005
1006             os.mkdir(os.path.join(mounttmp, "testcollection"))
1007
1008             d1 = llfuse.listdir(mounttmp)
1009             self.assertIn('testcollection', d1)
1010
1011             with self.assertRaises(OSError):
1012                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1013
1014             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1015
1016             d1 = llfuse.listdir(mounttmp)
1017             self.assertNotIn('testcollection', d1)
1018
1019             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1020             self.assertIn('testcollection', d1)
1021
1022     Test().runTest()
1023
1024 class FuseProjectMvTest(MountTestBase):
1025     def runTest(self):
1026         self.make_mount(fuse.ProjectDirectory,
1027                         project_object=self.api.users().current().execute())
1028
1029         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1030
1031
1032 def fuseFsyncTestHelper(mounttmp, k):
1033     class Test(unittest.TestCase):
1034         def runTest(self):
1035             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1036             os.fsync(fd)
1037             os.close(fd)
1038
1039     Test().runTest()
1040
1041 class FuseFsyncTest(FuseMagicTest):
1042     def runTest(self):
1043         self.make_mount(fuse.MagicDirectory)
1044         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1045
1046
1047 class MagicDirApiError(FuseMagicTest):
1048     def setUp(self):
1049         api = mock.MagicMock()
1050         super(MagicDirApiError, self).setUp(api=api)
1051         api.collections().get().execute.side_effect = iter([Exception('API fail'), {"manifest_text": self.test_manifest}])
1052         api.keep.get.side_effect = Exception('Keep fail')
1053
1054     def runTest(self):
1055         self.make_mount(fuse.MagicDirectory)
1056
1057         self.operations.inodes.inode_cache.cap = 1
1058         self.operations.inodes.inode_cache.min_entries = 2
1059
1060         with self.assertRaises(OSError):
1061             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1062
1063         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1064
1065
1066 class FuseUnitTest(unittest.TestCase):
1067     def test_sanitize_filename(self):
1068         acceptable = [
1069             "foo.txt",
1070             ".foo",
1071             "..foo",
1072             "...",
1073             "foo...",
1074             "foo..",
1075             "foo.",
1076             "-",
1077             "\x01\x02\x03",
1078             ]
1079         unacceptable = [
1080             "f\00",
1081             "\00\00",
1082             "/foo",
1083             "foo/",
1084             "//",
1085             ]
1086         for f in acceptable:
1087             self.assertEqual(f, fuse.sanitize_filename(f))
1088         for f in unacceptable:
1089             self.assertNotEqual(f, fuse.sanitize_filename(f))
1090             # The sanitized filename should be the same length, though.
1091             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1092         # Special cases
1093         self.assertEqual("_", fuse.sanitize_filename(""))
1094         self.assertEqual("_", fuse.sanitize_filename("."))
1095         self.assertEqual("__", fuse.sanitize_filename(".."))
1096
1097
1098 class FuseMagicTestPDHOnly(MountTestBase):
1099     def setUp(self, api=None):
1100         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1101
1102         cw = arvados.CollectionWriter()
1103
1104         cw.start_new_file('thing1.txt')
1105         cw.write("data 1")
1106
1107         self.testcollection = cw.finish()
1108         self.test_manifest = cw.manifest_text()
1109         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1110         self.testcollectionuuid = str(created['uuid'])
1111
1112     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1113         if skip_pdh_only is True:
1114             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1115         else:
1116             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1117
1118         mount_ls = llfuse.listdir(self.mounttmp)
1119         self.assertIn('README', mount_ls)
1120         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1121                              arvados.util.uuid_pattern.match(fn)
1122                              for fn in mount_ls),
1123                          "new FUSE MagicDirectory lists Collection")
1124
1125         # look up using pdh should succeed in all cases
1126         self.assertDirContents(self.testcollection, ['thing1.txt'])
1127         self.assertDirContents(os.path.join('by_id', self.testcollection),
1128                                ['thing1.txt'])
1129         mount_ls = llfuse.listdir(self.mounttmp)
1130         self.assertIn('README', mount_ls)
1131         self.assertIn(self.testcollection, mount_ls)
1132         self.assertIn(self.testcollection,
1133                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1134
1135         files = {}
1136         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1137
1138         for k, v in files.items():
1139             with open(os.path.join(self.mounttmp, k)) as f:
1140                 self.assertEqual(v, f.read())
1141
1142         # look up using uuid should fail when pdh_only is set
1143         if pdh_only is True:
1144             with self.assertRaises(OSError):
1145                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1146                                ['thing1.txt'])
1147         else:
1148             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1149                                ['thing1.txt'])
1150
1151     def test_with_pdh_only_true(self):
1152         self.verify_pdh_only(pdh_only=True)
1153
1154     def test_with_pdh_only_false(self):
1155         self.verify_pdh_only(pdh_only=False)
1156
1157     def test_with_default_by_id(self):
1158         self.verify_pdh_only(skip_pdh_only=True)