services/fuse/tests/test_mount.py

   1 import arvados
   2 import arvados.safeapi
   3 import arvados_fuse as fuse
   4 import glob
   5 import json
   6 import llfuse
   7 import os
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tempfile
  12 import threading
  13 import time
  14 import unittest
  15 import logging
  16 import multiprocessing
  17 import run_test_server
  18 import mock
  19 import re
  20
  21 from mount_test_base import MountTestBase
  22
  23 logger = logging.getLogger('arvados.arv-mount')
  24
  25
  26 class FuseMountTest(MountTestBase):
  27     def setUp(self):
  28         super(FuseMountTest, self).setUp()
  29
  30         cw = arvados.CollectionWriter()
  31
  32         cw.start_new_file('thing1.txt')
  33         cw.write("data 1")
  34         cw.start_new_file('thing2.txt')
  35         cw.write("data 2")
  36
  37         cw.start_new_stream('dir1')
  38         cw.start_new_file('thing3.txt')
  39         cw.write("data 3")
  40         cw.start_new_file('thing4.txt')
  41         cw.write("data 4")
  42
  43         cw.start_new_stream('dir2')
  44         cw.start_new_file('thing5.txt')
  45         cw.write("data 5")
  46         cw.start_new_file('thing6.txt')
  47         cw.write("data 6")
  48
  49         cw.start_new_stream('dir2/dir3')
  50         cw.start_new_file('thing7.txt')
  51         cw.write("data 7")
  52
  53         cw.start_new_file('thing8.txt')
  54         cw.write("data 8")
  55
  56         cw.start_new_stream('edgecases')
  57         for f in ":/.../-/*/\x01\\/ ".split("/"):
  58             cw.start_new_file(f)
  59             cw.write('x')
  60
  61         for f in ":/.../-/*/\x01\\/ ".split("/"):
  62             cw.start_new_stream('edgecases/dirs/' + f)
  63             cw.start_new_file('x/x')
  64             cw.write('x')
  65
  66         self.testcollection = cw.finish()
  67         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  68
  69     def runTest(self):
  70         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  71
  72         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
  73                                       'edgecases', 'dir1', 'dir2'])
  74         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
  75         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
  76         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
  77         self.assertDirContents('edgecases',
  78                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
  79         self.assertDirContents('edgecases/dirs',
  80                                ":/.../-/*/\x01\\/ ".split("/"))
  81
  82         files = {'thing1.txt': 'data 1',
  83                  'thing2.txt': 'data 2',
  84                  'dir1/thing3.txt': 'data 3',
  85                  'dir1/thing4.txt': 'data 4',
  86                  'dir2/thing5.txt': 'data 5',
  87                  'dir2/thing6.txt': 'data 6',
  88                  'dir2/dir3/thing7.txt': 'data 7',
  89                  'dir2/dir3/thing8.txt': 'data 8'}
  90
  91         for k, v in files.items():
  92             with open(os.path.join(self.mounttmp, k)) as f:
  93                 self.assertEqual(v, f.read())
  94
  95
  96 class FuseNoAPITest(MountTestBase):
  97     def setUp(self):
  98         super(FuseNoAPITest, self).setUp()
  99         keep = arvados.keep.KeepClient(local_store=self.keeptmp)
 100         self.file_data = "API-free text\n"
 101         self.file_loc = keep.put(self.file_data)
 102         self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
 103                 self.file_loc, len(self.file_data)))
 104
 105     def runTest(self):
 106         self.make_mount(fuse.MagicDirectory)
 107         self.assertDirContents(self.coll_loc, ['api-free.txt'])
 108         with open(os.path.join(
 109                 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
 110             actual = keep_file.read(-1)
 111         self.assertEqual(self.file_data, actual)
 112
 113
 114 class FuseMagicTest(MountTestBase):
 115     def setUp(self, api=None):
 116         super(FuseMagicTest, self).setUp(api=api)
 117
 118         cw = arvados.CollectionWriter()
 119
 120         cw.start_new_file('thing1.txt')
 121         cw.write("data 1")
 122
 123         self.testcollection = cw.finish()
 124         self.test_manifest = cw.manifest_text()
 125         self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 126
 127     def runTest(self):
 128         self.make_mount(fuse.MagicDirectory)
 129
 130         mount_ls = llfuse.listdir(self.mounttmp)
 131         self.assertIn('README', mount_ls)
 132         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 133                              arvados.util.uuid_pattern.match(fn)
 134                              for fn in mount_ls),
 135                          "new FUSE MagicDirectory lists Collection")
 136         self.assertDirContents(self.testcollection, ['thing1.txt'])
 137         self.assertDirContents(os.path.join('by_id', self.testcollection),
 138                                ['thing1.txt'])
 139         mount_ls = llfuse.listdir(self.mounttmp)
 140         self.assertIn('README', mount_ls)
 141         self.assertIn(self.testcollection, mount_ls)
 142         self.assertIn(self.testcollection,
 143                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 144
 145         files = {}
 146         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 147
 148         for k, v in files.items():
 149             with open(os.path.join(self.mounttmp, k)) as f:
 150                 self.assertEqual(v, f.read())
 151
 152
 153 class FuseTagsTest(MountTestBase):
 154     def runTest(self):
 155         self.make_mount(fuse.TagsDirectory)
 156
 157         d1 = llfuse.listdir(self.mounttmp)
 158         d1.sort()
 159         self.assertEqual(['foo_tag'], d1)
 160
 161         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 162         d2.sort()
 163         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 164
 165         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 166         d3.sort()
 167         self.assertEqual(['foo'], d3)
 168
 169
 170 class FuseTagsUpdateTest(MountTestBase):
 171     def tag_collection(self, coll_uuid, tag_name):
 172         return self.api.links().create(
 173             body={'link': {'head_uuid': coll_uuid,
 174                            'link_class': 'tag',
 175                            'name': tag_name,
 176         }}).execute()
 177
 178     def runTest(self):
 179         self.make_mount(fuse.TagsDirectory, poll_time=1)
 180
 181         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 182
 183         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 184         self.tag_collection(bar_uuid, 'fuse_test_tag')
 185         time.sleep(1)
 186         self.assertIn('fuse_test_tag', llfuse.listdir(self.mounttmp))
 187         self.assertDirContents('fuse_test_tag', [bar_uuid])
 188
 189         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 190         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 191         time.sleep(1)
 192         self.assertDirContents('fuse_test_tag', [bar_uuid, baz_uuid])
 193
 194         self.api.links().delete(uuid=l['uuid']).execute()
 195         time.sleep(1)
 196         self.assertDirContents('fuse_test_tag', [bar_uuid])
 197
 198
 199 class FuseSharedTest(MountTestBase):
 200     def runTest(self):
 201         self.make_mount(fuse.SharedDirectory,
 202                         exclude=self.api.users().current().execute()['uuid'])
 203
 204         # shared_dirs is a list of the directories exposed
 205         # by fuse.SharedDirectory (i.e. any object visible
 206         # to the current user)
 207         shared_dirs = llfuse.listdir(self.mounttmp)
 208         shared_dirs.sort()
 209         self.assertIn('FUSE User', shared_dirs)
 210
 211         # fuse_user_objs is a list of the objects owned by the FUSE
 212         # test user (which present as files in the 'FUSE User'
 213         # directory)
 214         fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
 215         fuse_user_objs.sort()
 216         self.assertEqual(['FUSE Test Project',                    # project owned by user
 217                           'collection #1 owned by FUSE',          # collection owned by user
 218                           'collection #2 owned by FUSE',          # collection owned by user
 219                           'pipeline instance owned by FUSE.pipelineInstance',  # pipeline instance owned by user
 220                       ], fuse_user_objs)
 221
 222         # test_proj_files is a list of the files in the FUSE Test Project.
 223         test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
 224         test_proj_files.sort()
 225         self.assertEqual(['collection in FUSE project',
 226                           'pipeline instance in FUSE project.pipelineInstance',
 227                           'pipeline template in FUSE project.pipelineTemplate'
 228                       ], test_proj_files)
 229
 230         # Double check that we can open and read objects in this folder as a file,
 231         # and that its contents are what we expect.
 232         pipeline_template_path = os.path.join(
 233                 self.mounttmp,
 234                 'FUSE User',
 235                 'FUSE Test Project',
 236                 'pipeline template in FUSE project.pipelineTemplate')
 237         with open(pipeline_template_path) as f:
 238             j = json.load(f)
 239             self.assertEqual("pipeline template in FUSE project", j['name'])
 240
 241         # check mtime on template
 242         st = os.stat(pipeline_template_path)
 243         self.assertEqual(st.st_mtime, 1397493304)
 244
 245         # check mtime on collection
 246         st = os.stat(os.path.join(
 247                 self.mounttmp,
 248                 'FUSE User',
 249                 'collection #1 owned by FUSE'))
 250         self.assertEqual(st.st_mtime, 1391448174)
 251
 252
 253 class FuseHomeTest(MountTestBase):
 254     def runTest(self):
 255         self.make_mount(fuse.ProjectDirectory,
 256                         project_object=self.api.users().current().execute())
 257
 258         d1 = llfuse.listdir(self.mounttmp)
 259         self.assertIn('Unrestricted public data', d1)
 260
 261         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 262         public_project = run_test_server.fixture('groups')[
 263             'anonymously_accessible_project']
 264         found_in = 0
 265         found_not_in = 0
 266         for name, item in run_test_server.fixture('collections').iteritems():
 267             if 'name' not in item:
 268                 pass
 269             elif item['owner_uuid'] == public_project['uuid']:
 270                 self.assertIn(item['name'], d2)
 271                 found_in += 1
 272             else:
 273                 # Artificial assumption here: there is no public
 274                 # collection fixture with the same name as a
 275                 # non-public collection.
 276                 self.assertNotIn(item['name'], d2)
 277                 found_not_in += 1
 278         self.assertNotEqual(0, found_in)
 279         self.assertNotEqual(0, found_not_in)
 280
 281         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 282         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 283
 284
 285 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 286     class Test(unittest.TestCase):
 287         def runTest(self):
 288             d1 = llfuse.listdir(mounttmp)
 289             self.assertEqual(["file1.txt"], d1)
 290             with open(os.path.join(mounttmp, "file1.txt")) as f:
 291                 self.assertEqual("blub", f.read())
 292     Test().runTest()
 293
 294 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 295     class Test(unittest.TestCase):
 296         def runTest(self):
 297             d1 = llfuse.listdir(mounttmp)
 298             self.assertEqual(["file1.txt"], d1)
 299             with open(os.path.join(mounttmp, "file1.txt")) as f:
 300                 self.assertEqual("plnp", f.read())
 301     Test().runTest()
 302
 303 class FuseModifyFileTest(MountTestBase):
 304     def runTest(self):
 305         collection = arvados.collection.Collection(api_client=self.api)
 306         with collection.open("file1.txt", "w") as f:
 307             f.write("blub")
 308
 309         collection.save_new()
 310
 311         m = self.make_mount(fuse.CollectionDirectory)
 312         with llfuse.lock:
 313             m.new_collection(collection.api_response(), collection)
 314
 315         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 316
 317         with collection.open("file1.txt", "w") as f:
 318             f.write("plnp")
 319
 320         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 321
 322
 323 class FuseAddFileToCollectionTest(MountTestBase):
 324     def runTest(self):
 325         collection = arvados.collection.Collection(api_client=self.api)
 326         with collection.open("file1.txt", "w") as f:
 327             f.write("blub")
 328
 329         collection.save_new()
 330
 331         m = self.make_mount(fuse.CollectionDirectory)
 332         with llfuse.lock:
 333             m.new_collection(collection.api_response(), collection)
 334
 335         d1 = llfuse.listdir(self.mounttmp)
 336         self.assertEqual(["file1.txt"], d1)
 337
 338         with collection.open("file2.txt", "w") as f:
 339             f.write("plnp")
 340
 341         d1 = llfuse.listdir(self.mounttmp)
 342         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 343
 344
 345 class FuseRemoveFileFromCollectionTest(MountTestBase):
 346     def runTest(self):
 347         collection = arvados.collection.Collection(api_client=self.api)
 348         with collection.open("file1.txt", "w") as f:
 349             f.write("blub")
 350
 351         with collection.open("file2.txt", "w") as f:
 352             f.write("plnp")
 353
 354         collection.save_new()
 355
 356         m = self.make_mount(fuse.CollectionDirectory)
 357         with llfuse.lock:
 358             m.new_collection(collection.api_response(), collection)
 359
 360         d1 = llfuse.listdir(self.mounttmp)
 361         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 362
 363         collection.remove("file2.txt")
 364
 365         d1 = llfuse.listdir(self.mounttmp)
 366         self.assertEqual(["file1.txt"], d1)
 367
 368
 369 def fuseCreateFileTestHelper(mounttmp):
 370     class Test(unittest.TestCase):
 371         def runTest(self):
 372             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 373                 pass
 374     Test().runTest()
 375
 376 class FuseCreateFileTest(MountTestBase):
 377     def runTest(self):
 378         collection = arvados.collection.Collection(api_client=self.api)
 379         collection.save_new()
 380
 381         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 382         self.assertEqual(collection2["manifest_text"], "")
 383
 384         collection.save_new()
 385
 386         m = self.make_mount(fuse.CollectionDirectory)
 387         with llfuse.lock:
 388             m.new_collection(collection.api_response(), collection)
 389         self.assertTrue(m.writable())
 390
 391         self.assertNotIn("file1.txt", collection)
 392
 393         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 394
 395         self.assertIn("file1.txt", collection)
 396
 397         d1 = llfuse.listdir(self.mounttmp)
 398         self.assertEqual(["file1.txt"], d1)
 399
 400         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 401         self.assertRegexpMatches(collection2["manifest_text"],
 402             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 403
 404
 405 def fuseWriteFileTestHelperWriteFile(mounttmp):
 406     class Test(unittest.TestCase):
 407         def runTest(self):
 408             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 409                 f.write("Hello world!")
 410     Test().runTest()
 411
 412 def fuseWriteFileTestHelperReadFile(mounttmp):
 413     class Test(unittest.TestCase):
 414         def runTest(self):
 415             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 416                 self.assertEqual(f.read(), "Hello world!")
 417     Test().runTest()
 418
 419 class FuseWriteFileTest(MountTestBase):
 420     def runTest(self):
 421         collection = arvados.collection.Collection(api_client=self.api)
 422         collection.save_new()
 423
 424         m = self.make_mount(fuse.CollectionDirectory)
 425         with llfuse.lock:
 426             m.new_collection(collection.api_response(), collection)
 427         self.assertTrue(m.writable())
 428
 429         self.assertNotIn("file1.txt", collection)
 430
 431         self.assertEqual(0, self.operations.write_counter.get())
 432         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 433         self.assertEqual(12, self.operations.write_counter.get())
 434
 435         with collection.open("file1.txt") as f:
 436             self.assertEqual(f.read(), "Hello world!")
 437
 438         self.assertEqual(0, self.operations.read_counter.get())
 439         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 440         self.assertEqual(12, self.operations.read_counter.get())
 441
 442         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 443         self.assertRegexpMatches(collection2["manifest_text"],
 444             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 445
 446
 447 def fuseUpdateFileTestHelper(mounttmp):
 448     class Test(unittest.TestCase):
 449         def runTest(self):
 450             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 451                 f.write("Hello world!")
 452
 453             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 454                 fr = f.read()
 455                 self.assertEqual(fr, "Hello world!")
 456                 f.seek(0)
 457                 f.write("Hola mundo!")
 458                 f.seek(0)
 459                 fr = f.read()
 460                 self.assertEqual(fr, "Hola mundo!!")
 461
 462             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 463                 self.assertEqual(f.read(), "Hola mundo!!")
 464
 465     Test().runTest()
 466
 467 class FuseUpdateFileTest(MountTestBase):
 468     def runTest(self):
 469         collection = arvados.collection.Collection(api_client=self.api)
 470         collection.save_new()
 471
 472         m = self.make_mount(fuse.CollectionDirectory)
 473         with llfuse.lock:
 474             m.new_collection(collection.api_response(), collection)
 475         self.assertTrue(m.writable())
 476
 477         # See note in MountTestBase.setUp
 478         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 479
 480         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 481         self.assertRegexpMatches(collection2["manifest_text"],
 482             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 483
 484
 485 def fuseMkdirTestHelper(mounttmp):
 486     class Test(unittest.TestCase):
 487         def runTest(self):
 488             with self.assertRaises(IOError):
 489                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 490                     f.write("Hello world!")
 491
 492             os.mkdir(os.path.join(mounttmp, "testdir"))
 493
 494             with self.assertRaises(OSError):
 495                 os.mkdir(os.path.join(mounttmp, "testdir"))
 496
 497             d1 = llfuse.listdir(mounttmp)
 498             self.assertEqual(["testdir"], d1)
 499
 500             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 501                 f.write("Hello world!")
 502
 503             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 504             self.assertEqual(["file1.txt"], d1)
 505
 506     Test().runTest()
 507
 508 class FuseMkdirTest(MountTestBase):
 509     def runTest(self):
 510         collection = arvados.collection.Collection(api_client=self.api)
 511         collection.save_new()
 512
 513         m = self.make_mount(fuse.CollectionDirectory)
 514         with llfuse.lock:
 515             m.new_collection(collection.api_response(), collection)
 516         self.assertTrue(m.writable())
 517
 518         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 519
 520         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 521         self.assertRegexpMatches(collection2["manifest_text"],
 522             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 523
 524
 525 def fuseRmTestHelperWriteFile(mounttmp):
 526     class Test(unittest.TestCase):
 527         def runTest(self):
 528             os.mkdir(os.path.join(mounttmp, "testdir"))
 529
 530             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 531                 f.write("Hello world!")
 532
 533     Test().runTest()
 534
 535 def fuseRmTestHelperDeleteFile(mounttmp):
 536     class Test(unittest.TestCase):
 537         def runTest(self):
 538             # Can't delete because it's not empty
 539             with self.assertRaises(OSError):
 540                 os.rmdir(os.path.join(mounttmp, "testdir"))
 541
 542             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 543             self.assertEqual(["file1.txt"], d1)
 544
 545             # Delete file
 546             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 547
 548             # Make sure it's empty
 549             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 550             self.assertEqual([], d1)
 551
 552             # Try to delete it again
 553             with self.assertRaises(OSError):
 554                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 555
 556     Test().runTest()
 557
 558 def fuseRmTestHelperRmdir(mounttmp):
 559     class Test(unittest.TestCase):
 560         def runTest(self):
 561             # Should be able to delete now that it is empty
 562             os.rmdir(os.path.join(mounttmp, "testdir"))
 563
 564             # Make sure it's empty
 565             d1 = llfuse.listdir(os.path.join(mounttmp))
 566             self.assertEqual([], d1)
 567
 568             # Try to delete it again
 569             with self.assertRaises(OSError):
 570                 os.rmdir(os.path.join(mounttmp, "testdir"))
 571
 572     Test().runTest()
 573
 574 class FuseRmTest(MountTestBase):
 575     def runTest(self):
 576         collection = arvados.collection.Collection(api_client=self.api)
 577         collection.save_new()
 578
 579         m = self.make_mount(fuse.CollectionDirectory)
 580         with llfuse.lock:
 581             m.new_collection(collection.api_response(), collection)
 582         self.assertTrue(m.writable())
 583
 584         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 585
 586         # Starting manifest
 587         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 588         self.assertRegexpMatches(collection2["manifest_text"],
 589             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 590         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 591
 592         # Can't have empty directories :-( so manifest will be empty.
 593         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 594         self.assertEqual(collection2["manifest_text"], "")
 595
 596         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 597
 598         # manifest should be empty now.
 599         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 600         self.assertEqual(collection2["manifest_text"], "")
 601
 602
 603 def fuseMvFileTestHelperWriteFile(mounttmp):
 604     class Test(unittest.TestCase):
 605         def runTest(self):
 606             os.mkdir(os.path.join(mounttmp, "testdir"))
 607
 608             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 609                 f.write("Hello world!")
 610
 611     Test().runTest()
 612
 613 def fuseMvFileTestHelperMoveFile(mounttmp):
 614     class Test(unittest.TestCase):
 615         def runTest(self):
 616             d1 = llfuse.listdir(os.path.join(mounttmp))
 617             self.assertEqual(["testdir"], d1)
 618             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 619             self.assertEqual(["file1.txt"], d1)
 620
 621             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 622
 623             d1 = llfuse.listdir(os.path.join(mounttmp))
 624             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 625             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 626             self.assertEqual([], d1)
 627
 628     Test().runTest()
 629
 630 class FuseMvFileTest(MountTestBase):
 631     def runTest(self):
 632         collection = arvados.collection.Collection(api_client=self.api)
 633         collection.save_new()
 634
 635         m = self.make_mount(fuse.CollectionDirectory)
 636         with llfuse.lock:
 637             m.new_collection(collection.api_response(), collection)
 638         self.assertTrue(m.writable())
 639
 640         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 641
 642         # Starting manifest
 643         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 644         self.assertRegexpMatches(collection2["manifest_text"],
 645             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 646
 647         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 648
 649         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 650         self.assertRegexpMatches(collection2["manifest_text"],
 651             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 652
 653
 654 def fuseRenameTestHelper(mounttmp):
 655     class Test(unittest.TestCase):
 656         def runTest(self):
 657             os.mkdir(os.path.join(mounttmp, "testdir"))
 658
 659             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 660                 f.write("Hello world!")
 661
 662     Test().runTest()
 663
 664 class FuseRenameTest(MountTestBase):
 665     def runTest(self):
 666         collection = arvados.collection.Collection(api_client=self.api)
 667         collection.save_new()
 668
 669         m = self.make_mount(fuse.CollectionDirectory)
 670         with llfuse.lock:
 671             m.new_collection(collection.api_response(), collection)
 672         self.assertTrue(m.writable())
 673
 674         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 675
 676         # Starting manifest
 677         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 678         self.assertRegexpMatches(collection2["manifest_text"],
 679             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 680
 681         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 682         self.assertEqual(["testdir"], d1)
 683         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 684         self.assertEqual(["file1.txt"], d1)
 685
 686         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 687
 688         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 689         self.assertEqual(["testdir2"], sorted(d1))
 690         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 691         self.assertEqual(["file1.txt"], d1)
 692
 693         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 694         self.assertRegexpMatches(collection2["manifest_text"],
 695             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 696
 697
 698 class FuseUpdateFromEventTest(MountTestBase):
 699     def runTest(self):
 700         collection = arvados.collection.Collection(api_client=self.api)
 701         collection.save_new()
 702
 703         m = self.make_mount(fuse.CollectionDirectory)
 704         with llfuse.lock:
 705             m.new_collection(collection.api_response(), collection)
 706
 707         self.operations.listen_for_events()
 708
 709         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 710         self.assertEqual([], sorted(d1))
 711
 712         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 713             with collection2.open("file1.txt", "w") as f:
 714                 f.write("foo")
 715
 716         # should show up in <10s via event bus notify
 717         expect = ["file1.txt"]
 718         for _ in range(100):
 719             d1 = sorted(llfuse.listdir(os.path.join(self.mounttmp)))
 720             if d1 == expect:
 721                 break
 722             time.sleep(0.1)
 723
 724         self.assertEqual(expect, d1)
 725
 726
 727 def fuseFileConflictTestHelper(mounttmp):
 728     class Test(unittest.TestCase):
 729         def runTest(self):
 730             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 731                 f.write("bar")
 732
 733             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 734             self.assertEqual(len(d1), 2)
 735
 736             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 737                 self.assertEqual(f.read(), "bar")
 738
 739             self.assertRegexpMatches(d1[1],
 740                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 741
 742             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 743                 self.assertEqual(f.read(), "foo")
 744
 745     Test().runTest()
 746
 747 class FuseFileConflictTest(MountTestBase):
 748     def runTest(self):
 749         collection = arvados.collection.Collection(api_client=self.api)
 750         collection.save_new()
 751
 752         m = self.make_mount(fuse.CollectionDirectory)
 753         with llfuse.lock:
 754             m.new_collection(collection.api_response(), collection)
 755
 756         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 757         self.assertEqual([], sorted(d1))
 758
 759         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 760             with collection2.open("file1.txt", "w") as f:
 761                 f.write("foo")
 762
 763         # See note in MountTestBase.setUp
 764         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 765
 766
 767 def fuseUnlinkOpenFileTest(mounttmp):
 768     class Test(unittest.TestCase):
 769         def runTest(self):
 770             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 771                 f.write("foo")
 772
 773                 d1 = llfuse.listdir(os.path.join(mounttmp))
 774                 self.assertEqual(["file1.txt"], sorted(d1))
 775
 776                 os.remove(os.path.join(mounttmp, "file1.txt"))
 777
 778                 d1 = llfuse.listdir(os.path.join(mounttmp))
 779                 self.assertEqual([], sorted(d1))
 780
 781                 f.seek(0)
 782                 self.assertEqual(f.read(), "foo")
 783                 f.write("bar")
 784
 785                 f.seek(0)
 786                 self.assertEqual(f.read(), "foobar")
 787
 788     Test().runTest()
 789
 790 class FuseUnlinkOpenFileTest(MountTestBase):
 791     def runTest(self):
 792         collection = arvados.collection.Collection(api_client=self.api)
 793         collection.save_new()
 794
 795         m = self.make_mount(fuse.CollectionDirectory)
 796         with llfuse.lock:
 797             m.new_collection(collection.api_response(), collection)
 798
 799         # See note in MountTestBase.setUp
 800         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 801
 802         self.assertEqual(collection.manifest_text(), "")
 803
 804
 805 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 806     class Test(unittest.TestCase):
 807         def runTest(self):
 808             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 809                 f.write("Hello world!")
 810
 811             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 812             self.assertEqual(["file1.txt"], sorted(d1))
 813             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 814             self.assertEqual([], sorted(d1))
 815
 816     Test().runTest()
 817
 818 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 819     class Test(unittest.TestCase):
 820         def runTest(self):
 821             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 822
 823             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 824             self.assertEqual([], sorted(d1))
 825             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 826             self.assertEqual(["file2.txt"], sorted(d1))
 827
 828     Test().runTest()
 829
 830 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 831     def runTest(self):
 832         collection1 = arvados.collection.Collection(api_client=self.api)
 833         collection1.save_new()
 834
 835         collection2 = arvados.collection.Collection(api_client=self.api)
 836         collection2.save_new()
 837
 838         m = self.make_mount(fuse.MagicDirectory)
 839
 840         # See note in MountTestBase.setUp
 841         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 842                                                   collection1.manifest_locator(),
 843                                                   collection2.manifest_locator()))
 844
 845         collection1.update()
 846         collection2.update()
 847
 848         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 849         self.assertEqual(collection2.manifest_text(), "")
 850
 851         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 852                                                   collection1.manifest_locator(),
 853                                                   collection2.manifest_locator()))
 854
 855         collection1.update()
 856         collection2.update()
 857
 858         self.assertEqual(collection1.manifest_text(), "")
 859         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 860
 861         collection1.stop_threads()
 862         collection2.stop_threads()
 863
 864
 865 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 866     class Test(unittest.TestCase):
 867         def runTest(self):
 868             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 869             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 870                 f.write("Hello world!")
 871
 872             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 873             self.assertEqual(["testdir"], sorted(d1))
 874             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 875             self.assertEqual(["file1.txt"], sorted(d1))
 876
 877             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 878             self.assertEqual([], sorted(d1))
 879
 880     Test().runTest()
 881
 882
 883 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 884     class Test(unittest.TestCase):
 885         def runTest(self):
 886             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 887
 888             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 889             self.assertEqual([], sorted(d1))
 890
 891             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 892             self.assertEqual(["testdir2"], sorted(d1))
 893             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 894             self.assertEqual(["file1.txt"], sorted(d1))
 895
 896             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 897                 self.assertEqual(f.read(), "Hello world!")
 898
 899     Test().runTest()
 900
 901 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 902     def runTest(self):
 903         collection1 = arvados.collection.Collection(api_client=self.api)
 904         collection1.save_new()
 905
 906         collection2 = arvados.collection.Collection(api_client=self.api)
 907         collection2.save_new()
 908
 909         m = self.make_mount(fuse.MagicDirectory)
 910
 911         # See note in MountTestBase.setUp
 912         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 913                                                   collection1.manifest_locator(),
 914                                                   collection2.manifest_locator()))
 915
 916         collection1.update()
 917         collection2.update()
 918
 919         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 920         self.assertEqual(collection2.manifest_text(), "")
 921
 922         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 923                                                   collection1.manifest_locator(),
 924                                                   collection2.manifest_locator()))
 925
 926         collection1.update()
 927         collection2.update()
 928
 929         self.assertEqual(collection1.manifest_text(), "")
 930         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 931
 932         collection1.stop_threads()
 933         collection2.stop_threads()
 934
 935 def fuseProjectMkdirTestHelper1(mounttmp):
 936     class Test(unittest.TestCase):
 937         def runTest(self):
 938             os.mkdir(os.path.join(mounttmp, "testcollection"))
 939             with self.assertRaises(OSError):
 940                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 941     Test().runTest()
 942
 943 def fuseProjectMkdirTestHelper2(mounttmp):
 944     class Test(unittest.TestCase):
 945         def runTest(self):
 946             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 947                 f.write("Hello world!")
 948             with self.assertRaises(OSError):
 949                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 950             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 951             with self.assertRaises(OSError):
 952                 os.remove(os.path.join(mounttmp, "testcollection"))
 953             os.rmdir(os.path.join(mounttmp, "testcollection"))
 954     Test().runTest()
 955
 956 class FuseProjectMkdirRmdirTest(MountTestBase):
 957     def runTest(self):
 958         self.make_mount(fuse.ProjectDirectory,
 959                         project_object=self.api.users().current().execute())
 960
 961         d1 = llfuse.listdir(self.mounttmp)
 962         self.assertNotIn('testcollection', d1)
 963
 964         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
 965
 966         d1 = llfuse.listdir(self.mounttmp)
 967         self.assertIn('testcollection', d1)
 968
 969         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
 970
 971         d1 = llfuse.listdir(self.mounttmp)
 972         self.assertNotIn('testcollection', d1)
 973
 974
 975 def fuseProjectMvTestHelper1(mounttmp):
 976     class Test(unittest.TestCase):
 977         def runTest(self):
 978             d1 = llfuse.listdir(mounttmp)
 979             self.assertNotIn('testcollection', d1)
 980
 981             os.mkdir(os.path.join(mounttmp, "testcollection"))
 982
 983             d1 = llfuse.listdir(mounttmp)
 984             self.assertIn('testcollection', d1)
 985
 986             with self.assertRaises(OSError):
 987                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
 988
 989             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
 990
 991             d1 = llfuse.listdir(mounttmp)
 992             self.assertNotIn('testcollection', d1)
 993
 994             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
 995             self.assertIn('testcollection', d1)
 996
 997     Test().runTest()
 998
 999 class FuseProjectMvTest(MountTestBase):
1000     def runTest(self):
1001         self.make_mount(fuse.ProjectDirectory,
1002                         project_object=self.api.users().current().execute())
1003
1004         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1005
1006
1007 def fuseFsyncTestHelper(mounttmp, k):
1008     class Test(unittest.TestCase):
1009         def runTest(self):
1010             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1011             os.fsync(fd)
1012             os.close(fd)
1013
1014     Test().runTest()
1015
1016 class FuseFsyncTest(FuseMagicTest):
1017     def runTest(self):
1018         self.make_mount(fuse.MagicDirectory)
1019         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1020
1021
1022 class MagicDirApiError(FuseMagicTest):
1023     def setUp(self):
1024         api = mock.MagicMock()
1025         super(MagicDirApiError, self).setUp(api=api)
1026         api.collections().get().execute.side_effect = iter([Exception('API fail'), {"manifest_text": self.test_manifest}])
1027         api.keep.get.side_effect = Exception('Keep fail')
1028
1029     def runTest(self):
1030         self.make_mount(fuse.MagicDirectory)
1031
1032         self.operations.inodes.inode_cache.cap = 1
1033         self.operations.inodes.inode_cache.min_entries = 2
1034
1035         with self.assertRaises(OSError):
1036             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1037
1038         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1039
1040
1041 class FuseUnitTest(unittest.TestCase):
1042     def test_sanitize_filename(self):
1043         acceptable = [
1044             "foo.txt",
1045             ".foo",
1046             "..foo",
1047             "...",
1048             "foo...",
1049             "foo..",
1050             "foo.",
1051             "-",
1052             "\x01\x02\x03",
1053             ]
1054         unacceptable = [
1055             "f\00",
1056             "\00\00",
1057             "/foo",
1058             "foo/",
1059             "//",
1060             ]
1061         for f in acceptable:
1062             self.assertEqual(f, fuse.sanitize_filename(f))
1063         for f in unacceptable:
1064             self.assertNotEqual(f, fuse.sanitize_filename(f))
1065             # The sanitized filename should be the same length, though.
1066             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1067         # Special cases
1068         self.assertEqual("_", fuse.sanitize_filename(""))
1069         self.assertEqual("_", fuse.sanitize_filename("."))
1070         self.assertEqual("__", fuse.sanitize_filename(".."))
1071
1072
1073 class FuseMagicTestPDHOnly(MountTestBase):
1074     def setUp(self, api=None):
1075         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1076
1077         cw = arvados.CollectionWriter()
1078
1079         cw.start_new_file('thing1.txt')
1080         cw.write("data 1")
1081
1082         self.testcollection = cw.finish()
1083         self.test_manifest = cw.manifest_text()
1084         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1085         self.testcollectionuuid = str(created['uuid'])
1086
1087     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1088         if skip_pdh_only is True:
1089             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1090         else:
1091             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1092
1093         mount_ls = llfuse.listdir(self.mounttmp)
1094         self.assertIn('README', mount_ls)
1095         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1096                              arvados.util.uuid_pattern.match(fn)
1097                              for fn in mount_ls),
1098                          "new FUSE MagicDirectory lists Collection")
1099
1100         # look up using pdh should succeed in all cases
1101         self.assertDirContents(self.testcollection, ['thing1.txt'])
1102         self.assertDirContents(os.path.join('by_id', self.testcollection),
1103                                ['thing1.txt'])
1104         mount_ls = llfuse.listdir(self.mounttmp)
1105         self.assertIn('README', mount_ls)
1106         self.assertIn(self.testcollection, mount_ls)
1107         self.assertIn(self.testcollection,
1108                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1109
1110         files = {}
1111         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1112
1113         for k, v in files.items():
1114             with open(os.path.join(self.mounttmp, k)) as f:
1115                 self.assertEqual(v, f.read())
1116
1117         # look up using uuid should fail when pdh_only is set
1118         if pdh_only is True:
1119             with self.assertRaises(OSError):
1120                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1121                                ['thing1.txt'])
1122         else:
1123             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1124                                ['thing1.txt'])
1125
1126     def test_with_pdh_only_true(self):
1127         self.verify_pdh_only(pdh_only=True)
1128
1129     def test_with_pdh_only_false(self):
1130         self.verify_pdh_only(pdh_only=False)
1131
1132     def test_with_default_by_id(self):
1133         self.verify_pdh_only(skip_pdh_only=True)
1134
1135 def _test_refresh_old_manifest(zzz):
1136     fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt'
1137     os.listdir(os.path.join(zzz))
1138     time.sleep(3)
1139     with open(os.path.join(zzz, fnm)) as f:
1140         f.read()
1141
1142 class TokenExpiryTest(MountTestBase):
1143     def setUp(self):
1144         super(TokenExpiryTest, self).setUp(local_store=False)
1145
1146     @mock.patch('arvados.keep.KeepClient.get')
1147     def runTest(self, mocked_get):
1148         self.api._rootDesc = {"blobSignatureTtl": 2}
1149         mnt = self.make_mount(fuse.CollectionDirectory, collection_record='zzzzz-4zz18-op4e2lbej01tcvu')
1150         mocked_get.return_value = 'fake data'
1151
1152         old_exp = int(time.time()) + 86400*14
1153         self.pool.apply(_test_refresh_old_manifest, (self.mounttmp,))
1154         want_exp = int(time.time()) + 86400*14
1155
1156         got_loc = mocked_get.call_args[0][0]
1157         got_exp = int(
1158             re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1),
1159             16)
1160         self.assertGreaterEqual(
1161             got_exp, want_exp-2,
1162             msg='now+2w = {:x}, but fuse fetched locator {} (old_exp {:x})'.format(
1163                 want_exp, got_loc, old_exp))
1164         self.assertLessEqual(
1165             got_exp, want_exp,
1166             msg='server is not using the expected 2w TTL; test is ineffective')