services/fuse/tests/test_mount.py

   1 import arvados
   2 import arvados.safeapi
   3 import arvados_fuse as fuse
   4 import glob
   5 import json
   6 import llfuse
   7 import os
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tempfile
  12 import threading
  13 import time
  14 import unittest
  15 import logging
  16 import multiprocessing
  17 import run_test_server
  18 import mock
  19 import re
  20
  21 from mount_test_base import MountTestBase
  22
  23 logger = logging.getLogger('arvados.arv-mount')
  24
  25
  26 class AssertWithTimeout(object):
  27     """Allow some time for an assertion to pass."""
  28
  29     def __init__(self, timeout=0):
  30         self.timeout = timeout
  31
  32     def __iter__(self):
  33         self.deadline = time.time() + self.timeout
  34         self.done = False
  35         return self
  36
  37     def next(self):
  38         if self.done:
  39             raise StopIteration
  40         return self.attempt
  41
  42     def attempt(self, fn, *args, **kwargs):
  43         try:
  44             fn(*args, **kwargs)
  45         except AssertionError:
  46             if time.time() > self.deadline:
  47                 raise
  48             time.sleep(0.1)
  49         else:
  50             self.done = True
  51
  52
  53 class FuseMountTest(MountTestBase):
  54     def setUp(self):
  55         super(FuseMountTest, self).setUp()
  56
  57         cw = arvados.CollectionWriter()
  58
  59         cw.start_new_file('thing1.txt')
  60         cw.write("data 1")
  61         cw.start_new_file('thing2.txt')
  62         cw.write("data 2")
  63
  64         cw.start_new_stream('dir1')
  65         cw.start_new_file('thing3.txt')
  66         cw.write("data 3")
  67         cw.start_new_file('thing4.txt')
  68         cw.write("data 4")
  69
  70         cw.start_new_stream('dir2')
  71         cw.start_new_file('thing5.txt')
  72         cw.write("data 5")
  73         cw.start_new_file('thing6.txt')
  74         cw.write("data 6")
  75
  76         cw.start_new_stream('dir2/dir3')
  77         cw.start_new_file('thing7.txt')
  78         cw.write("data 7")
  79
  80         cw.start_new_file('thing8.txt')
  81         cw.write("data 8")
  82
  83         cw.start_new_stream('edgecases')
  84         for f in ":/.../-/*/\x01\\/ ".split("/"):
  85             cw.start_new_file(f)
  86             cw.write('x')
  87
  88         for f in ":/.../-/*/\x01\\/ ".split("/"):
  89             cw.start_new_stream('edgecases/dirs/' + f)
  90             cw.start_new_file('x/x')
  91             cw.write('x')
  92
  93         self.testcollection = cw.finish()
  94         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  95
  96     def runTest(self):
  97         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  98
  99         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
 100                                       'edgecases', 'dir1', 'dir2'])
 101         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
 102         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
 103         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
 104         self.assertDirContents('edgecases',
 105                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
 106         self.assertDirContents('edgecases/dirs',
 107                                ":/.../-/*/\x01\\/ ".split("/"))
 108
 109         files = {'thing1.txt': 'data 1',
 110                  'thing2.txt': 'data 2',
 111                  'dir1/thing3.txt': 'data 3',
 112                  'dir1/thing4.txt': 'data 4',
 113                  'dir2/thing5.txt': 'data 5',
 114                  'dir2/thing6.txt': 'data 6',
 115                  'dir2/dir3/thing7.txt': 'data 7',
 116                  'dir2/dir3/thing8.txt': 'data 8'}
 117
 118         for k, v in files.items():
 119             with open(os.path.join(self.mounttmp, k)) as f:
 120                 self.assertEqual(v, f.read())
 121
 122
 123 class FuseNoAPITest(MountTestBase):
 124     def setUp(self):
 125         super(FuseNoAPITest, self).setUp()
 126         keep = arvados.keep.KeepClient(local_store=self.keeptmp)
 127         self.file_data = "API-free text\n"
 128         self.file_loc = keep.put(self.file_data)
 129         self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
 130                 self.file_loc, len(self.file_data)))
 131
 132     def runTest(self):
 133         self.make_mount(fuse.MagicDirectory)
 134         self.assertDirContents(self.coll_loc, ['api-free.txt'])
 135         with open(os.path.join(
 136                 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
 137             actual = keep_file.read(-1)
 138         self.assertEqual(self.file_data, actual)
 139
 140
 141 class FuseMagicTest(MountTestBase):
 142     def setUp(self, api=None):
 143         super(FuseMagicTest, self).setUp(api=api)
 144
 145         cw = arvados.CollectionWriter()
 146
 147         cw.start_new_file('thing1.txt')
 148         cw.write("data 1")
 149
 150         self.testcollection = cw.finish()
 151         self.test_manifest = cw.manifest_text()
 152         self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 153
 154     def runTest(self):
 155         self.make_mount(fuse.MagicDirectory)
 156
 157         mount_ls = llfuse.listdir(self.mounttmp)
 158         self.assertIn('README', mount_ls)
 159         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 160                              arvados.util.uuid_pattern.match(fn)
 161                              for fn in mount_ls),
 162                          "new FUSE MagicDirectory lists Collection")
 163         self.assertDirContents(self.testcollection, ['thing1.txt'])
 164         self.assertDirContents(os.path.join('by_id', self.testcollection),
 165                                ['thing1.txt'])
 166         mount_ls = llfuse.listdir(self.mounttmp)
 167         self.assertIn('README', mount_ls)
 168         self.assertIn(self.testcollection, mount_ls)
 169         self.assertIn(self.testcollection,
 170                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 171
 172         files = {}
 173         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 174
 175         for k, v in files.items():
 176             with open(os.path.join(self.mounttmp, k)) as f:
 177                 self.assertEqual(v, f.read())
 178
 179
 180 class FuseTagsTest(MountTestBase):
 181     def runTest(self):
 182         self.make_mount(fuse.TagsDirectory)
 183
 184         d1 = llfuse.listdir(self.mounttmp)
 185         d1.sort()
 186         self.assertEqual(['foo_tag'], d1)
 187
 188         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 189         d2.sort()
 190         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 191
 192         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 193         d3.sort()
 194         self.assertEqual(['foo'], d3)
 195
 196
 197 class FuseTagsUpdateTest(MountTestBase):
 198     def tag_collection(self, coll_uuid, tag_name):
 199         return self.api.links().create(
 200             body={'link': {'head_uuid': coll_uuid,
 201                            'link_class': 'tag',
 202                            'name': tag_name,
 203         }}).execute()
 204
 205     def runTest(self):
 206         self.make_mount(fuse.TagsDirectory, poll_time=1)
 207
 208         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 209
 210         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 211         self.tag_collection(bar_uuid, 'fuse_test_tag')
 212         for attempt in AssertWithTimeout(10):
 213             attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
 214         self.assertDirContents('fuse_test_tag', [bar_uuid])
 215
 216         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 217         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 218         for attempt in AssertWithTimeout(10):
 219             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
 220
 221         self.api.links().delete(uuid=l['uuid']).execute()
 222         for attempt in AssertWithTimeout(10):
 223             attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
 224
 225
 226 class FuseSharedTest(MountTestBase):
 227     def runTest(self):
 228         self.make_mount(fuse.SharedDirectory,
 229                         exclude=self.api.users().current().execute()['uuid'])
 230
 231         # shared_dirs is a list of the directories exposed
 232         # by fuse.SharedDirectory (i.e. any object visible
 233         # to the current user)
 234         shared_dirs = llfuse.listdir(self.mounttmp)
 235         shared_dirs.sort()
 236         self.assertIn('FUSE User', shared_dirs)
 237
 238         # fuse_user_objs is a list of the objects owned by the FUSE
 239         # test user (which present as files in the 'FUSE User'
 240         # directory)
 241         fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
 242         fuse_user_objs.sort()
 243         self.assertEqual(['FUSE Test Project',                    # project owned by user
 244                           'collection #1 owned by FUSE',          # collection owned by user
 245                           'collection #2 owned by FUSE',          # collection owned by user
 246                           'pipeline instance owned by FUSE.pipelineInstance',  # pipeline instance owned by user
 247                       ], fuse_user_objs)
 248
 249         # test_proj_files is a list of the files in the FUSE Test Project.
 250         test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
 251         test_proj_files.sort()
 252         self.assertEqual(['collection in FUSE project',
 253                           'pipeline instance in FUSE project.pipelineInstance',
 254                           'pipeline template in FUSE project.pipelineTemplate'
 255                       ], test_proj_files)
 256
 257         # Double check that we can open and read objects in this folder as a file,
 258         # and that its contents are what we expect.
 259         pipeline_template_path = os.path.join(
 260                 self.mounttmp,
 261                 'FUSE User',
 262                 'FUSE Test Project',
 263                 'pipeline template in FUSE project.pipelineTemplate')
 264         with open(pipeline_template_path) as f:
 265             j = json.load(f)
 266             self.assertEqual("pipeline template in FUSE project", j['name'])
 267
 268         # check mtime on template
 269         st = os.stat(pipeline_template_path)
 270         try:
 271             mtime = st.st_mtime_ns / 1000000000
 272         except AttributeError:
 273             mtime = st.st_mtime
 274         self.assertEqual(mtime, 1397493304)
 275
 276         # check mtime on collection
 277         st = os.stat(os.path.join(
 278                 self.mounttmp,
 279                 'FUSE User',
 280                 'collection #1 owned by FUSE'))
 281         try:
 282             mtime = st.st_mtime_ns / 1000000000
 283         except AttributeError:
 284             mtime = st.st_mtime
 285         self.assertEqual(mtime, 1391448174)
 286
 287
 288 class FuseHomeTest(MountTestBase):
 289     def runTest(self):
 290         self.make_mount(fuse.ProjectDirectory,
 291                         project_object=self.api.users().current().execute())
 292
 293         d1 = llfuse.listdir(self.mounttmp)
 294         self.assertIn('Unrestricted public data', d1)
 295
 296         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 297         public_project = run_test_server.fixture('groups')[
 298             'anonymously_accessible_project']
 299         found_in = 0
 300         found_not_in = 0
 301         for name, item in run_test_server.fixture('collections').iteritems():
 302             if 'name' not in item:
 303                 pass
 304             elif item['owner_uuid'] == public_project['uuid']:
 305                 self.assertIn(item['name'], d2)
 306                 found_in += 1
 307             else:
 308                 # Artificial assumption here: there is no public
 309                 # collection fixture with the same name as a
 310                 # non-public collection.
 311                 self.assertNotIn(item['name'], d2)
 312                 found_not_in += 1
 313         self.assertNotEqual(0, found_in)
 314         self.assertNotEqual(0, found_not_in)
 315
 316         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 317         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 318
 319
 320 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 321     class Test(unittest.TestCase):
 322         def runTest(self):
 323             d1 = llfuse.listdir(mounttmp)
 324             self.assertEqual(["file1.txt"], d1)
 325             with open(os.path.join(mounttmp, "file1.txt")) as f:
 326                 self.assertEqual("blub", f.read())
 327     Test().runTest()
 328
 329 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 330     class Test(unittest.TestCase):
 331         def runTest(self):
 332             d1 = llfuse.listdir(mounttmp)
 333             self.assertEqual(["file1.txt"], d1)
 334             with open(os.path.join(mounttmp, "file1.txt")) as f:
 335                 self.assertEqual("plnp", f.read())
 336     Test().runTest()
 337
 338 class FuseModifyFileTest(MountTestBase):
 339     def runTest(self):
 340         collection = arvados.collection.Collection(api_client=self.api)
 341         with collection.open("file1.txt", "w") as f:
 342             f.write("blub")
 343
 344         collection.save_new()
 345
 346         m = self.make_mount(fuse.CollectionDirectory)
 347         with llfuse.lock:
 348             m.new_collection(collection.api_response(), collection)
 349
 350         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 351
 352         with collection.open("file1.txt", "w") as f:
 353             f.write("plnp")
 354
 355         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 356
 357
 358 class FuseAddFileToCollectionTest(MountTestBase):
 359     def runTest(self):
 360         collection = arvados.collection.Collection(api_client=self.api)
 361         with collection.open("file1.txt", "w") as f:
 362             f.write("blub")
 363
 364         collection.save_new()
 365
 366         m = self.make_mount(fuse.CollectionDirectory)
 367         with llfuse.lock:
 368             m.new_collection(collection.api_response(), collection)
 369
 370         d1 = llfuse.listdir(self.mounttmp)
 371         self.assertEqual(["file1.txt"], d1)
 372
 373         with collection.open("file2.txt", "w") as f:
 374             f.write("plnp")
 375
 376         d1 = llfuse.listdir(self.mounttmp)
 377         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 378
 379
 380 class FuseRemoveFileFromCollectionTest(MountTestBase):
 381     def runTest(self):
 382         collection = arvados.collection.Collection(api_client=self.api)
 383         with collection.open("file1.txt", "w") as f:
 384             f.write("blub")
 385
 386         with collection.open("file2.txt", "w") as f:
 387             f.write("plnp")
 388
 389         collection.save_new()
 390
 391         m = self.make_mount(fuse.CollectionDirectory)
 392         with llfuse.lock:
 393             m.new_collection(collection.api_response(), collection)
 394
 395         d1 = llfuse.listdir(self.mounttmp)
 396         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 397
 398         collection.remove("file2.txt")
 399
 400         d1 = llfuse.listdir(self.mounttmp)
 401         self.assertEqual(["file1.txt"], d1)
 402
 403
 404 def fuseCreateFileTestHelper(mounttmp):
 405     class Test(unittest.TestCase):
 406         def runTest(self):
 407             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 408                 pass
 409     Test().runTest()
 410
 411 class FuseCreateFileTest(MountTestBase):
 412     def runTest(self):
 413         collection = arvados.collection.Collection(api_client=self.api)
 414         collection.save_new()
 415
 416         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 417         self.assertEqual(collection2["manifest_text"], "")
 418
 419         collection.save_new()
 420
 421         m = self.make_mount(fuse.CollectionDirectory)
 422         with llfuse.lock:
 423             m.new_collection(collection.api_response(), collection)
 424         self.assertTrue(m.writable())
 425
 426         self.assertNotIn("file1.txt", collection)
 427
 428         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 429
 430         self.assertIn("file1.txt", collection)
 431
 432         d1 = llfuse.listdir(self.mounttmp)
 433         self.assertEqual(["file1.txt"], d1)
 434
 435         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 436         self.assertRegexpMatches(collection2["manifest_text"],
 437             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 438
 439
 440 def fuseWriteFileTestHelperWriteFile(mounttmp):
 441     class Test(unittest.TestCase):
 442         def runTest(self):
 443             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 444                 f.write("Hello world!")
 445     Test().runTest()
 446
 447 def fuseWriteFileTestHelperReadFile(mounttmp):
 448     class Test(unittest.TestCase):
 449         def runTest(self):
 450             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 451                 self.assertEqual(f.read(), "Hello world!")
 452     Test().runTest()
 453
 454 class FuseWriteFileTest(MountTestBase):
 455     def runTest(self):
 456         collection = arvados.collection.Collection(api_client=self.api)
 457         collection.save_new()
 458
 459         m = self.make_mount(fuse.CollectionDirectory)
 460         with llfuse.lock:
 461             m.new_collection(collection.api_response(), collection)
 462         self.assertTrue(m.writable())
 463
 464         self.assertNotIn("file1.txt", collection)
 465
 466         self.assertEqual(0, self.operations.write_counter.get())
 467         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 468         self.assertEqual(12, self.operations.write_counter.get())
 469
 470         with collection.open("file1.txt") as f:
 471             self.assertEqual(f.read(), "Hello world!")
 472
 473         self.assertEqual(0, self.operations.read_counter.get())
 474         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 475         self.assertEqual(12, self.operations.read_counter.get())
 476
 477         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 478         self.assertRegexpMatches(collection2["manifest_text"],
 479             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 480
 481
 482 def fuseUpdateFileTestHelper(mounttmp):
 483     class Test(unittest.TestCase):
 484         def runTest(self):
 485             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 486                 f.write("Hello world!")
 487
 488             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 489                 fr = f.read()
 490                 self.assertEqual(fr, "Hello world!")
 491                 f.seek(0)
 492                 f.write("Hola mundo!")
 493                 f.seek(0)
 494                 fr = f.read()
 495                 self.assertEqual(fr, "Hola mundo!!")
 496
 497             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 498                 self.assertEqual(f.read(), "Hola mundo!!")
 499
 500     Test().runTest()
 501
 502 class FuseUpdateFileTest(MountTestBase):
 503     def runTest(self):
 504         collection = arvados.collection.Collection(api_client=self.api)
 505         collection.save_new()
 506
 507         m = self.make_mount(fuse.CollectionDirectory)
 508         with llfuse.lock:
 509             m.new_collection(collection.api_response(), collection)
 510         self.assertTrue(m.writable())
 511
 512         # See note in MountTestBase.setUp
 513         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 514
 515         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 516         self.assertRegexpMatches(collection2["manifest_text"],
 517             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 518
 519
 520 def fuseMkdirTestHelper(mounttmp):
 521     class Test(unittest.TestCase):
 522         def runTest(self):
 523             with self.assertRaises(IOError):
 524                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 525                     f.write("Hello world!")
 526
 527             os.mkdir(os.path.join(mounttmp, "testdir"))
 528
 529             with self.assertRaises(OSError):
 530                 os.mkdir(os.path.join(mounttmp, "testdir"))
 531
 532             d1 = llfuse.listdir(mounttmp)
 533             self.assertEqual(["testdir"], d1)
 534
 535             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 536                 f.write("Hello world!")
 537
 538             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 539             self.assertEqual(["file1.txt"], d1)
 540
 541     Test().runTest()
 542
 543 class FuseMkdirTest(MountTestBase):
 544     def runTest(self):
 545         collection = arvados.collection.Collection(api_client=self.api)
 546         collection.save_new()
 547
 548         m = self.make_mount(fuse.CollectionDirectory)
 549         with llfuse.lock:
 550             m.new_collection(collection.api_response(), collection)
 551         self.assertTrue(m.writable())
 552
 553         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 554
 555         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 556         self.assertRegexpMatches(collection2["manifest_text"],
 557             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 558
 559
 560 def fuseRmTestHelperWriteFile(mounttmp):
 561     class Test(unittest.TestCase):
 562         def runTest(self):
 563             os.mkdir(os.path.join(mounttmp, "testdir"))
 564
 565             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 566                 f.write("Hello world!")
 567
 568     Test().runTest()
 569
 570 def fuseRmTestHelperDeleteFile(mounttmp):
 571     class Test(unittest.TestCase):
 572         def runTest(self):
 573             # Can't delete because it's not empty
 574             with self.assertRaises(OSError):
 575                 os.rmdir(os.path.join(mounttmp, "testdir"))
 576
 577             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 578             self.assertEqual(["file1.txt"], d1)
 579
 580             # Delete file
 581             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 582
 583             # Make sure it's empty
 584             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 585             self.assertEqual([], d1)
 586
 587             # Try to delete it again
 588             with self.assertRaises(OSError):
 589                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 590
 591     Test().runTest()
 592
 593 def fuseRmTestHelperRmdir(mounttmp):
 594     class Test(unittest.TestCase):
 595         def runTest(self):
 596             # Should be able to delete now that it is empty
 597             os.rmdir(os.path.join(mounttmp, "testdir"))
 598
 599             # Make sure it's empty
 600             d1 = llfuse.listdir(os.path.join(mounttmp))
 601             self.assertEqual([], d1)
 602
 603             # Try to delete it again
 604             with self.assertRaises(OSError):
 605                 os.rmdir(os.path.join(mounttmp, "testdir"))
 606
 607     Test().runTest()
 608
 609 class FuseRmTest(MountTestBase):
 610     def runTest(self):
 611         collection = arvados.collection.Collection(api_client=self.api)
 612         collection.save_new()
 613
 614         m = self.make_mount(fuse.CollectionDirectory)
 615         with llfuse.lock:
 616             m.new_collection(collection.api_response(), collection)
 617         self.assertTrue(m.writable())
 618
 619         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 620
 621         # Starting manifest
 622         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 623         self.assertRegexpMatches(collection2["manifest_text"],
 624             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 625         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 626
 627         # Can't have empty directories :-( so manifest will be empty.
 628         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 629         self.assertEqual(collection2["manifest_text"], "")
 630
 631         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 632
 633         # manifest should be empty now.
 634         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 635         self.assertEqual(collection2["manifest_text"], "")
 636
 637
 638 def fuseMvFileTestHelperWriteFile(mounttmp):
 639     class Test(unittest.TestCase):
 640         def runTest(self):
 641             os.mkdir(os.path.join(mounttmp, "testdir"))
 642
 643             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 644                 f.write("Hello world!")
 645
 646     Test().runTest()
 647
 648 def fuseMvFileTestHelperMoveFile(mounttmp):
 649     class Test(unittest.TestCase):
 650         def runTest(self):
 651             d1 = llfuse.listdir(os.path.join(mounttmp))
 652             self.assertEqual(["testdir"], d1)
 653             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 654             self.assertEqual(["file1.txt"], d1)
 655
 656             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 657
 658             d1 = llfuse.listdir(os.path.join(mounttmp))
 659             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 660             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 661             self.assertEqual([], d1)
 662
 663     Test().runTest()
 664
 665 class FuseMvFileTest(MountTestBase):
 666     def runTest(self):
 667         collection = arvados.collection.Collection(api_client=self.api)
 668         collection.save_new()
 669
 670         m = self.make_mount(fuse.CollectionDirectory)
 671         with llfuse.lock:
 672             m.new_collection(collection.api_response(), collection)
 673         self.assertTrue(m.writable())
 674
 675         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 676
 677         # Starting manifest
 678         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 679         self.assertRegexpMatches(collection2["manifest_text"],
 680             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 681
 682         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 683
 684         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 685         self.assertRegexpMatches(collection2["manifest_text"],
 686             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 687
 688
 689 def fuseRenameTestHelper(mounttmp):
 690     class Test(unittest.TestCase):
 691         def runTest(self):
 692             os.mkdir(os.path.join(mounttmp, "testdir"))
 693
 694             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 695                 f.write("Hello world!")
 696
 697     Test().runTest()
 698
 699 class FuseRenameTest(MountTestBase):
 700     def runTest(self):
 701         collection = arvados.collection.Collection(api_client=self.api)
 702         collection.save_new()
 703
 704         m = self.make_mount(fuse.CollectionDirectory)
 705         with llfuse.lock:
 706             m.new_collection(collection.api_response(), collection)
 707         self.assertTrue(m.writable())
 708
 709         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 710
 711         # Starting manifest
 712         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 713         self.assertRegexpMatches(collection2["manifest_text"],
 714             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 715
 716         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 717         self.assertEqual(["testdir"], d1)
 718         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 719         self.assertEqual(["file1.txt"], d1)
 720
 721         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 722
 723         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 724         self.assertEqual(["testdir2"], sorted(d1))
 725         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 726         self.assertEqual(["file1.txt"], d1)
 727
 728         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 729         self.assertRegexpMatches(collection2["manifest_text"],
 730             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 731
 732
 733 class FuseUpdateFromEventTest(MountTestBase):
 734     def runTest(self):
 735         collection = arvados.collection.Collection(api_client=self.api)
 736         collection.save_new()
 737
 738         m = self.make_mount(fuse.CollectionDirectory)
 739         with llfuse.lock:
 740             m.new_collection(collection.api_response(), collection)
 741
 742         self.operations.listen_for_events()
 743
 744         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 745         self.assertEqual([], sorted(d1))
 746
 747         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 748             with collection2.open("file1.txt", "w") as f:
 749                 f.write("foo")
 750
 751         for attempt in AssertWithTimeout(10):
 752             attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
 753
 754
 755 def fuseFileConflictTestHelper(mounttmp):
 756     class Test(unittest.TestCase):
 757         def runTest(self):
 758             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 759                 f.write("bar")
 760
 761             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 762             self.assertEqual(len(d1), 2)
 763
 764             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 765                 self.assertEqual(f.read(), "bar")
 766
 767             self.assertRegexpMatches(d1[1],
 768                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 769
 770             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 771                 self.assertEqual(f.read(), "foo")
 772
 773     Test().runTest()
 774
 775 class FuseFileConflictTest(MountTestBase):
 776     def runTest(self):
 777         collection = arvados.collection.Collection(api_client=self.api)
 778         collection.save_new()
 779
 780         m = self.make_mount(fuse.CollectionDirectory)
 781         with llfuse.lock:
 782             m.new_collection(collection.api_response(), collection)
 783
 784         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 785         self.assertEqual([], sorted(d1))
 786
 787         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 788             with collection2.open("file1.txt", "w") as f:
 789                 f.write("foo")
 790
 791         # See note in MountTestBase.setUp
 792         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 793
 794
 795 def fuseUnlinkOpenFileTest(mounttmp):
 796     class Test(unittest.TestCase):
 797         def runTest(self):
 798             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 799                 f.write("foo")
 800
 801                 d1 = llfuse.listdir(os.path.join(mounttmp))
 802                 self.assertEqual(["file1.txt"], sorted(d1))
 803
 804                 os.remove(os.path.join(mounttmp, "file1.txt"))
 805
 806                 d1 = llfuse.listdir(os.path.join(mounttmp))
 807                 self.assertEqual([], sorted(d1))
 808
 809                 f.seek(0)
 810                 self.assertEqual(f.read(), "foo")
 811                 f.write("bar")
 812
 813                 f.seek(0)
 814                 self.assertEqual(f.read(), "foobar")
 815
 816     Test().runTest()
 817
 818 class FuseUnlinkOpenFileTest(MountTestBase):
 819     def runTest(self):
 820         collection = arvados.collection.Collection(api_client=self.api)
 821         collection.save_new()
 822
 823         m = self.make_mount(fuse.CollectionDirectory)
 824         with llfuse.lock:
 825             m.new_collection(collection.api_response(), collection)
 826
 827         # See note in MountTestBase.setUp
 828         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 829
 830         self.assertEqual(collection.manifest_text(), "")
 831
 832
 833 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 834     class Test(unittest.TestCase):
 835         def runTest(self):
 836             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 837                 f.write("Hello world!")
 838
 839             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 840             self.assertEqual(["file1.txt"], sorted(d1))
 841             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 842             self.assertEqual([], sorted(d1))
 843
 844     Test().runTest()
 845
 846 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 847     class Test(unittest.TestCase):
 848         def runTest(self):
 849             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 850
 851             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 852             self.assertEqual([], sorted(d1))
 853             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 854             self.assertEqual(["file2.txt"], sorted(d1))
 855
 856     Test().runTest()
 857
 858 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 859     def runTest(self):
 860         collection1 = arvados.collection.Collection(api_client=self.api)
 861         collection1.save_new()
 862
 863         collection2 = arvados.collection.Collection(api_client=self.api)
 864         collection2.save_new()
 865
 866         m = self.make_mount(fuse.MagicDirectory)
 867
 868         # See note in MountTestBase.setUp
 869         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 870                                                   collection1.manifest_locator(),
 871                                                   collection2.manifest_locator()))
 872
 873         collection1.update()
 874         collection2.update()
 875
 876         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 877         self.assertEqual(collection2.manifest_text(), "")
 878
 879         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 880                                                   collection1.manifest_locator(),
 881                                                   collection2.manifest_locator()))
 882
 883         collection1.update()
 884         collection2.update()
 885
 886         self.assertEqual(collection1.manifest_text(), "")
 887         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 888
 889         collection1.stop_threads()
 890         collection2.stop_threads()
 891
 892
 893 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 894     class Test(unittest.TestCase):
 895         def runTest(self):
 896             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 897             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 898                 f.write("Hello world!")
 899
 900             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 901             self.assertEqual(["testdir"], sorted(d1))
 902             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 903             self.assertEqual(["file1.txt"], sorted(d1))
 904
 905             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 906             self.assertEqual([], sorted(d1))
 907
 908     Test().runTest()
 909
 910
 911 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 912     class Test(unittest.TestCase):
 913         def runTest(self):
 914             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 915
 916             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 917             self.assertEqual([], sorted(d1))
 918
 919             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 920             self.assertEqual(["testdir2"], sorted(d1))
 921             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 922             self.assertEqual(["file1.txt"], sorted(d1))
 923
 924             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 925                 self.assertEqual(f.read(), "Hello world!")
 926
 927     Test().runTest()
 928
 929 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 930     def runTest(self):
 931         collection1 = arvados.collection.Collection(api_client=self.api)
 932         collection1.save_new()
 933
 934         collection2 = arvados.collection.Collection(api_client=self.api)
 935         collection2.save_new()
 936
 937         m = self.make_mount(fuse.MagicDirectory)
 938
 939         # See note in MountTestBase.setUp
 940         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 941                                                   collection1.manifest_locator(),
 942                                                   collection2.manifest_locator()))
 943
 944         collection1.update()
 945         collection2.update()
 946
 947         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 948         self.assertEqual(collection2.manifest_text(), "")
 949
 950         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 951                                                   collection1.manifest_locator(),
 952                                                   collection2.manifest_locator()))
 953
 954         collection1.update()
 955         collection2.update()
 956
 957         self.assertEqual(collection1.manifest_text(), "")
 958         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 959
 960         collection1.stop_threads()
 961         collection2.stop_threads()
 962
 963 def fuseProjectMkdirTestHelper1(mounttmp):
 964     class Test(unittest.TestCase):
 965         def runTest(self):
 966             os.mkdir(os.path.join(mounttmp, "testcollection"))
 967             with self.assertRaises(OSError):
 968                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 969     Test().runTest()
 970
 971 def fuseProjectMkdirTestHelper2(mounttmp):
 972     class Test(unittest.TestCase):
 973         def runTest(self):
 974             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 975                 f.write("Hello world!")
 976             with self.assertRaises(OSError):
 977                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 978             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 979             with self.assertRaises(OSError):
 980                 os.remove(os.path.join(mounttmp, "testcollection"))
 981             os.rmdir(os.path.join(mounttmp, "testcollection"))
 982     Test().runTest()
 983
 984 class FuseProjectMkdirRmdirTest(MountTestBase):
 985     def runTest(self):
 986         self.make_mount(fuse.ProjectDirectory,
 987                         project_object=self.api.users().current().execute())
 988
 989         d1 = llfuse.listdir(self.mounttmp)
 990         self.assertNotIn('testcollection', d1)
 991
 992         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
 993
 994         d1 = llfuse.listdir(self.mounttmp)
 995         self.assertIn('testcollection', d1)
 996
 997         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
 998
 999         d1 = llfuse.listdir(self.mounttmp)
1000         self.assertNotIn('testcollection', d1)
1001
1002
1003 def fuseProjectMvTestHelper1(mounttmp):
1004     class Test(unittest.TestCase):
1005         def runTest(self):
1006             d1 = llfuse.listdir(mounttmp)
1007             self.assertNotIn('testcollection', d1)
1008
1009             os.mkdir(os.path.join(mounttmp, "testcollection"))
1010
1011             d1 = llfuse.listdir(mounttmp)
1012             self.assertIn('testcollection', d1)
1013
1014             with self.assertRaises(OSError):
1015                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
1016
1017             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
1018
1019             d1 = llfuse.listdir(mounttmp)
1020             self.assertNotIn('testcollection', d1)
1021
1022             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
1023             self.assertIn('testcollection', d1)
1024
1025     Test().runTest()
1026
1027 class FuseProjectMvTest(MountTestBase):
1028     def runTest(self):
1029         self.make_mount(fuse.ProjectDirectory,
1030                         project_object=self.api.users().current().execute())
1031
1032         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1033
1034
1035 def fuseFsyncTestHelper(mounttmp, k):
1036     class Test(unittest.TestCase):
1037         def runTest(self):
1038             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1039             os.fsync(fd)
1040             os.close(fd)
1041
1042     Test().runTest()
1043
1044 class FuseFsyncTest(FuseMagicTest):
1045     def runTest(self):
1046         self.make_mount(fuse.MagicDirectory)
1047         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1048
1049
1050 class MagicDirApiError(FuseMagicTest):
1051     def setUp(self):
1052         api = mock.MagicMock()
1053         super(MagicDirApiError, self).setUp(api=api)
1054         api.collections().get().execute.side_effect = iter([Exception('API fail'), {"manifest_text": self.test_manifest}])
1055         api.keep.get.side_effect = Exception('Keep fail')
1056
1057     def runTest(self):
1058         self.make_mount(fuse.MagicDirectory)
1059
1060         self.operations.inodes.inode_cache.cap = 1
1061         self.operations.inodes.inode_cache.min_entries = 2
1062
1063         with self.assertRaises(OSError):
1064             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1065
1066         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1067
1068
1069 class FuseUnitTest(unittest.TestCase):
1070     def test_sanitize_filename(self):
1071         acceptable = [
1072             "foo.txt",
1073             ".foo",
1074             "..foo",
1075             "...",
1076             "foo...",
1077             "foo..",
1078             "foo.",
1079             "-",
1080             "\x01\x02\x03",
1081             ]
1082         unacceptable = [
1083             "f\00",
1084             "\00\00",
1085             "/foo",
1086             "foo/",
1087             "//",
1088             ]
1089         for f in acceptable:
1090             self.assertEqual(f, fuse.sanitize_filename(f))
1091         for f in unacceptable:
1092             self.assertNotEqual(f, fuse.sanitize_filename(f))
1093             # The sanitized filename should be the same length, though.
1094             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1095         # Special cases
1096         self.assertEqual("_", fuse.sanitize_filename(""))
1097         self.assertEqual("_", fuse.sanitize_filename("."))
1098         self.assertEqual("__", fuse.sanitize_filename(".."))
1099
1100
1101 class FuseMagicTestPDHOnly(MountTestBase):
1102     def setUp(self, api=None):
1103         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1104
1105         cw = arvados.CollectionWriter()
1106
1107         cw.start_new_file('thing1.txt')
1108         cw.write("data 1")
1109
1110         self.testcollection = cw.finish()
1111         self.test_manifest = cw.manifest_text()
1112         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1113         self.testcollectionuuid = str(created['uuid'])
1114
1115     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1116         if skip_pdh_only is True:
1117             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1118         else:
1119             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1120
1121         mount_ls = llfuse.listdir(self.mounttmp)
1122         self.assertIn('README', mount_ls)
1123         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1124                              arvados.util.uuid_pattern.match(fn)
1125                              for fn in mount_ls),
1126                          "new FUSE MagicDirectory lists Collection")
1127
1128         # look up using pdh should succeed in all cases
1129         self.assertDirContents(self.testcollection, ['thing1.txt'])
1130         self.assertDirContents(os.path.join('by_id', self.testcollection),
1131                                ['thing1.txt'])
1132         mount_ls = llfuse.listdir(self.mounttmp)
1133         self.assertIn('README', mount_ls)
1134         self.assertIn(self.testcollection, mount_ls)
1135         self.assertIn(self.testcollection,
1136                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1137
1138         files = {}
1139         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1140
1141         for k, v in files.items():
1142             with open(os.path.join(self.mounttmp, k)) as f:
1143                 self.assertEqual(v, f.read())
1144
1145         # look up using uuid should fail when pdh_only is set
1146         if pdh_only is True:
1147             with self.assertRaises(OSError):
1148                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1149                                ['thing1.txt'])
1150         else:
1151             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1152                                ['thing1.txt'])
1153
1154     def test_with_pdh_only_true(self):
1155         self.verify_pdh_only(pdh_only=True)
1156
1157     def test_with_pdh_only_false(self):
1158         self.verify_pdh_only(pdh_only=False)
1159
1160     def test_with_default_by_id(self):
1161         self.verify_pdh_only(skip_pdh_only=True)