services/fuse/tests/test_mount.py

   1 import arvados
   2 import arvados.safeapi
   3 import arvados_fuse as fuse
   4 import glob
   5 import json
   6 import llfuse
   7 import os
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tempfile
  12 import threading
  13 import time
  14 import unittest
  15 import logging
  16 import multiprocessing
  17 import run_test_server
  18 import mock
  19 import re
  20
  21 from mount_test_base import MountTestBase
  22
  23 logger = logging.getLogger('arvados.arv-mount')
  24
  25
  26 class FuseMountTest(MountTestBase):
  27     def setUp(self):
  28         super(FuseMountTest, self).setUp()
  29
  30         cw = arvados.CollectionWriter()
  31
  32         cw.start_new_file('thing1.txt')
  33         cw.write("data 1")
  34         cw.start_new_file('thing2.txt')
  35         cw.write("data 2")
  36
  37         cw.start_new_stream('dir1')
  38         cw.start_new_file('thing3.txt')
  39         cw.write("data 3")
  40         cw.start_new_file('thing4.txt')
  41         cw.write("data 4")
  42
  43         cw.start_new_stream('dir2')
  44         cw.start_new_file('thing5.txt')
  45         cw.write("data 5")
  46         cw.start_new_file('thing6.txt')
  47         cw.write("data 6")
  48
  49         cw.start_new_stream('dir2/dir3')
  50         cw.start_new_file('thing7.txt')
  51         cw.write("data 7")
  52
  53         cw.start_new_file('thing8.txt')
  54         cw.write("data 8")
  55
  56         cw.start_new_stream('edgecases')
  57         for f in ":/.../-/*/\x01\\/ ".split("/"):
  58             cw.start_new_file(f)
  59             cw.write('x')
  60
  61         for f in ":/.../-/*/\x01\\/ ".split("/"):
  62             cw.start_new_stream('edgecases/dirs/' + f)
  63             cw.start_new_file('x/x')
  64             cw.write('x')
  65
  66         self.testcollection = cw.finish()
  67         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
  68
  69     def runTest(self):
  70         self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
  71
  72         self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
  73                                       'edgecases', 'dir1', 'dir2'])
  74         self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
  75         self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
  76         self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
  77         self.assertDirContents('edgecases',
  78                                "dirs/:/.../-/*/\x01\\/ ".split("/"))
  79         self.assertDirContents('edgecases/dirs',
  80                                ":/.../-/*/\x01\\/ ".split("/"))
  81
  82         files = {'thing1.txt': 'data 1',
  83                  'thing2.txt': 'data 2',
  84                  'dir1/thing3.txt': 'data 3',
  85                  'dir1/thing4.txt': 'data 4',
  86                  'dir2/thing5.txt': 'data 5',
  87                  'dir2/thing6.txt': 'data 6',
  88                  'dir2/dir3/thing7.txt': 'data 7',
  89                  'dir2/dir3/thing8.txt': 'data 8'}
  90
  91         for k, v in files.items():
  92             with open(os.path.join(self.mounttmp, k)) as f:
  93                 self.assertEqual(v, f.read())
  94
  95
  96 class FuseNoAPITest(MountTestBase):
  97     def setUp(self):
  98         super(FuseNoAPITest, self).setUp()
  99         keep = arvados.keep.KeepClient(local_store=self.keeptmp)
 100         self.file_data = "API-free text\n"
 101         self.file_loc = keep.put(self.file_data)
 102         self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
 103                 self.file_loc, len(self.file_data)))
 104
 105     def runTest(self):
 106         self.make_mount(fuse.MagicDirectory)
 107         self.assertDirContents(self.coll_loc, ['api-free.txt'])
 108         with open(os.path.join(
 109                 self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
 110             actual = keep_file.read(-1)
 111         self.assertEqual(self.file_data, actual)
 112
 113
 114 class FuseMagicTest(MountTestBase):
 115     def setUp(self, api=None):
 116         super(FuseMagicTest, self).setUp(api=api)
 117
 118         cw = arvados.CollectionWriter()
 119
 120         cw.start_new_file('thing1.txt')
 121         cw.write("data 1")
 122
 123         self.testcollection = cw.finish()
 124         self.test_manifest = cw.manifest_text()
 125         self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
 126
 127     def runTest(self):
 128         self.make_mount(fuse.MagicDirectory)
 129
 130         mount_ls = llfuse.listdir(self.mounttmp)
 131         self.assertIn('README', mount_ls)
 132         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
 133                              arvados.util.uuid_pattern.match(fn)
 134                              for fn in mount_ls),
 135                          "new FUSE MagicDirectory lists Collection")
 136         self.assertDirContents(self.testcollection, ['thing1.txt'])
 137         self.assertDirContents(os.path.join('by_id', self.testcollection),
 138                                ['thing1.txt'])
 139         mount_ls = llfuse.listdir(self.mounttmp)
 140         self.assertIn('README', mount_ls)
 141         self.assertIn(self.testcollection, mount_ls)
 142         self.assertIn(self.testcollection,
 143                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
 144
 145         files = {}
 146         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 147
 148         for k, v in files.items():
 149             with open(os.path.join(self.mounttmp, k)) as f:
 150                 self.assertEqual(v, f.read())
 151
 152
 153 class FuseTagsTest(MountTestBase):
 154     def runTest(self):
 155         self.make_mount(fuse.TagsDirectory)
 156
 157         d1 = llfuse.listdir(self.mounttmp)
 158         d1.sort()
 159         self.assertEqual(['foo_tag'], d1)
 160
 161         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 162         d2.sort()
 163         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 164
 165         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
 166         d3.sort()
 167         self.assertEqual(['foo'], d3)
 168
 169
 170 class FuseTagsUpdateTest(MountTestBase):
 171     def tag_collection(self, coll_uuid, tag_name):
 172         return self.api.links().create(
 173             body={'link': {'head_uuid': coll_uuid,
 174                            'link_class': 'tag',
 175                            'name': tag_name,
 176         }}).execute()
 177
 178     def runTest(self):
 179         self.make_mount(fuse.TagsDirectory, poll_time=1)
 180
 181         self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
 182
 183         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
 184         self.tag_collection(bar_uuid, 'fuse_test_tag')
 185         time.sleep(1)
 186         self.assertIn('fuse_test_tag', llfuse.listdir(self.mounttmp))
 187         self.assertDirContents('fuse_test_tag', [bar_uuid])
 188
 189         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
 190         l = self.tag_collection(baz_uuid, 'fuse_test_tag')
 191         time.sleep(1)
 192         self.assertDirContents('fuse_test_tag', [bar_uuid, baz_uuid])
 193
 194         self.api.links().delete(uuid=l['uuid']).execute()
 195         time.sleep(1)
 196         self.assertDirContents('fuse_test_tag', [bar_uuid])
 197
 198
 199 class FuseSharedTest(MountTestBase):
 200     def runTest(self):
 201         self.make_mount(fuse.SharedDirectory,
 202                         exclude=self.api.users().current().execute()['uuid'])
 203
 204         # shared_dirs is a list of the directories exposed
 205         # by fuse.SharedDirectory (i.e. any object visible
 206         # to the current user)
 207         shared_dirs = llfuse.listdir(self.mounttmp)
 208         shared_dirs.sort()
 209         self.assertIn('FUSE User', shared_dirs)
 210
 211         # fuse_user_objs is a list of the objects owned by the FUSE
 212         # test user (which present as files in the 'FUSE User'
 213         # directory)
 214         fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
 215         fuse_user_objs.sort()
 216         self.assertEqual(['FUSE Test Project',                    # project owned by user
 217                           'collection #1 owned by FUSE',          # collection owned by user
 218                           'collection #2 owned by FUSE',          # collection owned by user
 219                           'pipeline instance owned by FUSE.pipelineInstance',  # pipeline instance owned by user
 220                       ], fuse_user_objs)
 221
 222         # test_proj_files is a list of the files in the FUSE Test Project.
 223         test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
 224         test_proj_files.sort()
 225         self.assertEqual(['collection in FUSE project',
 226                           'pipeline instance in FUSE project.pipelineInstance',
 227                           'pipeline template in FUSE project.pipelineTemplate'
 228                       ], test_proj_files)
 229
 230         # Double check that we can open and read objects in this folder as a file,
 231         # and that its contents are what we expect.
 232         pipeline_template_path = os.path.join(
 233                 self.mounttmp,
 234                 'FUSE User',
 235                 'FUSE Test Project',
 236                 'pipeline template in FUSE project.pipelineTemplate')
 237         with open(pipeline_template_path) as f:
 238             j = json.load(f)
 239             self.assertEqual("pipeline template in FUSE project", j['name'])
 240
 241         # check mtime on template
 242         st = os.stat(pipeline_template_path)
 243         self.assertEqual(st.st_mtime, 1397493304)
 244
 245         # check mtime on collection
 246         st = os.stat(os.path.join(
 247                 self.mounttmp,
 248                 'FUSE User',
 249                 'collection #1 owned by FUSE'))
 250         self.assertEqual(st.st_mtime, 1391448174)
 251
 252
 253 class FuseHomeTest(MountTestBase):
 254     def runTest(self):
 255         self.make_mount(fuse.ProjectDirectory,
 256                         project_object=self.api.users().current().execute())
 257
 258         d1 = llfuse.listdir(self.mounttmp)
 259         self.assertIn('Unrestricted public data', d1)
 260
 261         d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
 262         public_project = run_test_server.fixture('groups')[
 263             'anonymously_accessible_project']
 264         found_in = 0
 265         found_not_in = 0
 266         for name, item in run_test_server.fixture('collections').iteritems():
 267             if 'name' not in item:
 268                 pass
 269             elif item['owner_uuid'] == public_project['uuid']:
 270                 self.assertIn(item['name'], d2)
 271                 found_in += 1
 272             else:
 273                 # Artificial assumption here: there is no public
 274                 # collection fixture with the same name as a
 275                 # non-public collection.
 276                 self.assertNotIn(item['name'], d2)
 277                 found_not_in += 1
 278         self.assertNotEqual(0, found_in)
 279         self.assertNotEqual(0, found_not_in)
 280
 281         d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
 282         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 283
 284
 285 def fuseModifyFileTestHelperReadStartContents(mounttmp):
 286     class Test(unittest.TestCase):
 287         def runTest(self):
 288             d1 = llfuse.listdir(mounttmp)
 289             self.assertEqual(["file1.txt"], d1)
 290             with open(os.path.join(mounttmp, "file1.txt")) as f:
 291                 self.assertEqual("blub", f.read())
 292     Test().runTest()
 293
 294 def fuseModifyFileTestHelperReadEndContents(mounttmp):
 295     class Test(unittest.TestCase):
 296         def runTest(self):
 297             d1 = llfuse.listdir(mounttmp)
 298             self.assertEqual(["file1.txt"], d1)
 299             with open(os.path.join(mounttmp, "file1.txt")) as f:
 300                 self.assertEqual("plnp", f.read())
 301     Test().runTest()
 302
 303 class FuseModifyFileTest(MountTestBase):
 304     def runTest(self):
 305         collection = arvados.collection.Collection(api_client=self.api)
 306         with collection.open("file1.txt", "w") as f:
 307             f.write("blub")
 308
 309         collection.save_new()
 310
 311         m = self.make_mount(fuse.CollectionDirectory)
 312         with llfuse.lock:
 313             m.new_collection(collection.api_response(), collection)
 314
 315         self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 316
 317         with collection.open("file1.txt", "w") as f:
 318             f.write("plnp")
 319
 320         self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 321
 322
 323 class FuseAddFileToCollectionTest(MountTestBase):
 324     def runTest(self):
 325         collection = arvados.collection.Collection(api_client=self.api)
 326         with collection.open("file1.txt", "w") as f:
 327             f.write("blub")
 328
 329         collection.save_new()
 330
 331         m = self.make_mount(fuse.CollectionDirectory)
 332         with llfuse.lock:
 333             m.new_collection(collection.api_response(), collection)
 334
 335         d1 = llfuse.listdir(self.mounttmp)
 336         self.assertEqual(["file1.txt"], d1)
 337
 338         with collection.open("file2.txt", "w") as f:
 339             f.write("plnp")
 340
 341         d1 = llfuse.listdir(self.mounttmp)
 342         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 343
 344
 345 class FuseRemoveFileFromCollectionTest(MountTestBase):
 346     def runTest(self):
 347         collection = arvados.collection.Collection(api_client=self.api)
 348         with collection.open("file1.txt", "w") as f:
 349             f.write("blub")
 350
 351         with collection.open("file2.txt", "w") as f:
 352             f.write("plnp")
 353
 354         collection.save_new()
 355
 356         m = self.make_mount(fuse.CollectionDirectory)
 357         with llfuse.lock:
 358             m.new_collection(collection.api_response(), collection)
 359
 360         d1 = llfuse.listdir(self.mounttmp)
 361         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 362
 363         collection.remove("file2.txt")
 364
 365         d1 = llfuse.listdir(self.mounttmp)
 366         self.assertEqual(["file1.txt"], d1)
 367
 368
 369 def fuseCreateFileTestHelper(mounttmp):
 370     class Test(unittest.TestCase):
 371         def runTest(self):
 372             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 373                 pass
 374     Test().runTest()
 375
 376 class FuseCreateFileTest(MountTestBase):
 377     def runTest(self):
 378         collection = arvados.collection.Collection(api_client=self.api)
 379         collection.save_new()
 380
 381         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 382         self.assertEqual(collection2["manifest_text"], "")
 383
 384         collection.save_new()
 385
 386         m = self.make_mount(fuse.CollectionDirectory)
 387         with llfuse.lock:
 388             m.new_collection(collection.api_response(), collection)
 389         self.assertTrue(m.writable())
 390
 391         self.assertNotIn("file1.txt", collection)
 392
 393         self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
 394
 395         self.assertIn("file1.txt", collection)
 396
 397         d1 = llfuse.listdir(self.mounttmp)
 398         self.assertEqual(["file1.txt"], d1)
 399
 400         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 401         self.assertRegexpMatches(collection2["manifest_text"],
 402             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 403
 404
 405 def fuseWriteFileTestHelperWriteFile(mounttmp):
 406     class Test(unittest.TestCase):
 407         def runTest(self):
 408             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 409                 f.write("Hello world!")
 410     Test().runTest()
 411
 412 def fuseWriteFileTestHelperReadFile(mounttmp):
 413     class Test(unittest.TestCase):
 414         def runTest(self):
 415             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 416                 self.assertEqual(f.read(), "Hello world!")
 417     Test().runTest()
 418
 419 class FuseWriteFileTest(MountTestBase):
 420     def runTest(self):
 421         collection = arvados.collection.Collection(api_client=self.api)
 422         collection.save_new()
 423
 424         m = self.make_mount(fuse.CollectionDirectory)
 425         with llfuse.lock:
 426             m.new_collection(collection.api_response(), collection)
 427         self.assertTrue(m.writable())
 428
 429         self.assertNotIn("file1.txt", collection)
 430
 431         self.assertEqual(0, self.operations.write_counter.get())
 432         self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 433         self.assertEqual(12, self.operations.write_counter.get())
 434
 435         with collection.open("file1.txt") as f:
 436             self.assertEqual(f.read(), "Hello world!")
 437
 438         self.assertEqual(0, self.operations.read_counter.get())
 439         self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 440         self.assertEqual(12, self.operations.read_counter.get())
 441
 442         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 443         self.assertRegexpMatches(collection2["manifest_text"],
 444             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 445
 446
 447 def fuseUpdateFileTestHelper(mounttmp):
 448     class Test(unittest.TestCase):
 449         def runTest(self):
 450             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 451                 f.write("Hello world!")
 452
 453             with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
 454                 fr = f.read()
 455                 self.assertEqual(fr, "Hello world!")
 456                 f.seek(0)
 457                 f.write("Hola mundo!")
 458                 f.seek(0)
 459                 fr = f.read()
 460                 self.assertEqual(fr, "Hola mundo!!")
 461
 462             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 463                 self.assertEqual(f.read(), "Hola mundo!!")
 464
 465     Test().runTest()
 466
 467 class FuseUpdateFileTest(MountTestBase):
 468     def runTest(self):
 469         collection = arvados.collection.Collection(api_client=self.api)
 470         collection.save_new()
 471
 472         m = self.make_mount(fuse.CollectionDirectory)
 473         with llfuse.lock:
 474             m.new_collection(collection.api_response(), collection)
 475         self.assertTrue(m.writable())
 476
 477         # See note in MountTestBase.setUp
 478         self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
 479
 480         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 481         self.assertRegexpMatches(collection2["manifest_text"],
 482             r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 483
 484
 485 def fuseMkdirTestHelper(mounttmp):
 486     class Test(unittest.TestCase):
 487         def runTest(self):
 488             with self.assertRaises(IOError):
 489                 with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 490                     f.write("Hello world!")
 491
 492             os.mkdir(os.path.join(mounttmp, "testdir"))
 493
 494             with self.assertRaises(OSError):
 495                 os.mkdir(os.path.join(mounttmp, "testdir"))
 496
 497             d1 = llfuse.listdir(mounttmp)
 498             self.assertEqual(["testdir"], d1)
 499
 500             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 501                 f.write("Hello world!")
 502
 503             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 504             self.assertEqual(["file1.txt"], d1)
 505
 506     Test().runTest()
 507
 508 class FuseMkdirTest(MountTestBase):
 509     def runTest(self):
 510         collection = arvados.collection.Collection(api_client=self.api)
 511         collection.save_new()
 512
 513         m = self.make_mount(fuse.CollectionDirectory)
 514         with llfuse.lock:
 515             m.new_collection(collection.api_response(), collection)
 516         self.assertTrue(m.writable())
 517
 518         self.pool.apply(fuseMkdirTestHelper, (self.mounttmp,))
 519
 520         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 521         self.assertRegexpMatches(collection2["manifest_text"],
 522             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 523
 524
 525 def fuseRmTestHelperWriteFile(mounttmp):
 526     class Test(unittest.TestCase):
 527         def runTest(self):
 528             os.mkdir(os.path.join(mounttmp, "testdir"))
 529
 530             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 531                 f.write("Hello world!")
 532
 533     Test().runTest()
 534
 535 def fuseRmTestHelperDeleteFile(mounttmp):
 536     class Test(unittest.TestCase):
 537         def runTest(self):
 538             # Can't delete because it's not empty
 539             with self.assertRaises(OSError):
 540                 os.rmdir(os.path.join(mounttmp, "testdir"))
 541
 542             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 543             self.assertEqual(["file1.txt"], d1)
 544
 545             # Delete file
 546             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 547
 548             # Make sure it's empty
 549             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 550             self.assertEqual([], d1)
 551
 552             # Try to delete it again
 553             with self.assertRaises(OSError):
 554                 os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 555
 556     Test().runTest()
 557
 558 def fuseRmTestHelperRmdir(mounttmp):
 559     class Test(unittest.TestCase):
 560         def runTest(self):
 561             # Should be able to delete now that it is empty
 562             os.rmdir(os.path.join(mounttmp, "testdir"))
 563
 564             # Make sure it's empty
 565             d1 = llfuse.listdir(os.path.join(mounttmp))
 566             self.assertEqual([], d1)
 567
 568             # Try to delete it again
 569             with self.assertRaises(OSError):
 570                 os.rmdir(os.path.join(mounttmp, "testdir"))
 571
 572     Test().runTest()
 573
 574 class FuseRmTest(MountTestBase):
 575     def runTest(self):
 576         collection = arvados.collection.Collection(api_client=self.api)
 577         collection.save_new()
 578
 579         m = self.make_mount(fuse.CollectionDirectory)
 580         with llfuse.lock:
 581             m.new_collection(collection.api_response(), collection)
 582         self.assertTrue(m.writable())
 583
 584         self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 585
 586         # Starting manifest
 587         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 588         self.assertRegexpMatches(collection2["manifest_text"],
 589             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 590         self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 591
 592         # Can't have empty directories :-( so manifest will be empty.
 593         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 594         self.assertEqual(collection2["manifest_text"], "")
 595
 596         self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 597
 598         # manifest should be empty now.
 599         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 600         self.assertEqual(collection2["manifest_text"], "")
 601
 602
 603 def fuseMvFileTestHelperWriteFile(mounttmp):
 604     class Test(unittest.TestCase):
 605         def runTest(self):
 606             os.mkdir(os.path.join(mounttmp, "testdir"))
 607
 608             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 609                 f.write("Hello world!")
 610
 611     Test().runTest()
 612
 613 def fuseMvFileTestHelperMoveFile(mounttmp):
 614     class Test(unittest.TestCase):
 615         def runTest(self):
 616             d1 = llfuse.listdir(os.path.join(mounttmp))
 617             self.assertEqual(["testdir"], d1)
 618             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 619             self.assertEqual(["file1.txt"], d1)
 620
 621             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 622
 623             d1 = llfuse.listdir(os.path.join(mounttmp))
 624             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
 625             d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
 626             self.assertEqual([], d1)
 627
 628     Test().runTest()
 629
 630 class FuseMvFileTest(MountTestBase):
 631     def runTest(self):
 632         collection = arvados.collection.Collection(api_client=self.api)
 633         collection.save_new()
 634
 635         m = self.make_mount(fuse.CollectionDirectory)
 636         with llfuse.lock:
 637             m.new_collection(collection.api_response(), collection)
 638         self.assertTrue(m.writable())
 639
 640         self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 641
 642         # Starting manifest
 643         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 644         self.assertRegexpMatches(collection2["manifest_text"],
 645             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 646
 647         self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 648
 649         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 650         self.assertRegexpMatches(collection2["manifest_text"],
 651             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 652
 653
 654 def fuseRenameTestHelper(mounttmp):
 655     class Test(unittest.TestCase):
 656         def runTest(self):
 657             os.mkdir(os.path.join(mounttmp, "testdir"))
 658
 659             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
 660                 f.write("Hello world!")
 661
 662     Test().runTest()
 663
 664 class FuseRenameTest(MountTestBase):
 665     def runTest(self):
 666         collection = arvados.collection.Collection(api_client=self.api)
 667         collection.save_new()
 668
 669         m = self.make_mount(fuse.CollectionDirectory)
 670         with llfuse.lock:
 671             m.new_collection(collection.api_response(), collection)
 672         self.assertTrue(m.writable())
 673
 674         self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
 675
 676         # Starting manifest
 677         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 678         self.assertRegexpMatches(collection2["manifest_text"],
 679             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 680
 681         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 682         self.assertEqual(["testdir"], d1)
 683         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
 684         self.assertEqual(["file1.txt"], d1)
 685
 686         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 687
 688         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 689         self.assertEqual(["testdir2"], sorted(d1))
 690         d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
 691         self.assertEqual(["file1.txt"], d1)
 692
 693         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
 694         self.assertRegexpMatches(collection2["manifest_text"],
 695             r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 696
 697
 698 class FuseUpdateFromEventTest(MountTestBase):
 699     def runTest(self):
 700         collection = arvados.collection.Collection(api_client=self.api)
 701         collection.save_new()
 702
 703         m = self.make_mount(fuse.CollectionDirectory)
 704         with llfuse.lock:
 705             m.new_collection(collection.api_response(), collection)
 706
 707         self.operations.listen_for_events()
 708
 709         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 710         self.assertEqual([], sorted(d1))
 711
 712         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 713             with collection2.open("file1.txt", "w") as f:
 714                 f.write("foo")
 715
 716         time.sleep(1)
 717
 718         # should show up via event bus notify
 719
 720         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 721         self.assertEqual(["file1.txt"], sorted(d1))
 722
 723
 724 def fuseFileConflictTestHelper(mounttmp):
 725     class Test(unittest.TestCase):
 726         def runTest(self):
 727             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
 728                 f.write("bar")
 729
 730             d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
 731             self.assertEqual(len(d1), 2)
 732
 733             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
 734                 self.assertEqual(f.read(), "bar")
 735
 736             self.assertRegexpMatches(d1[1],
 737                 r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 738
 739             with open(os.path.join(mounttmp, d1[1]), "r") as f:
 740                 self.assertEqual(f.read(), "foo")
 741
 742     Test().runTest()
 743
 744 class FuseFileConflictTest(MountTestBase):
 745     def runTest(self):
 746         collection = arvados.collection.Collection(api_client=self.api)
 747         collection.save_new()
 748
 749         m = self.make_mount(fuse.CollectionDirectory)
 750         with llfuse.lock:
 751             m.new_collection(collection.api_response(), collection)
 752
 753         d1 = llfuse.listdir(os.path.join(self.mounttmp))
 754         self.assertEqual([], sorted(d1))
 755
 756         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
 757             with collection2.open("file1.txt", "w") as f:
 758                 f.write("foo")
 759
 760         # See note in MountTestBase.setUp
 761         self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
 762
 763
 764 def fuseUnlinkOpenFileTest(mounttmp):
 765     class Test(unittest.TestCase):
 766         def runTest(self):
 767             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
 768                 f.write("foo")
 769
 770                 d1 = llfuse.listdir(os.path.join(mounttmp))
 771                 self.assertEqual(["file1.txt"], sorted(d1))
 772
 773                 os.remove(os.path.join(mounttmp, "file1.txt"))
 774
 775                 d1 = llfuse.listdir(os.path.join(mounttmp))
 776                 self.assertEqual([], sorted(d1))
 777
 778                 f.seek(0)
 779                 self.assertEqual(f.read(), "foo")
 780                 f.write("bar")
 781
 782                 f.seek(0)
 783                 self.assertEqual(f.read(), "foobar")
 784
 785     Test().runTest()
 786
 787 class FuseUnlinkOpenFileTest(MountTestBase):
 788     def runTest(self):
 789         collection = arvados.collection.Collection(api_client=self.api)
 790         collection.save_new()
 791
 792         m = self.make_mount(fuse.CollectionDirectory)
 793         with llfuse.lock:
 794             m.new_collection(collection.api_response(), collection)
 795
 796         # See note in MountTestBase.setUp
 797         self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
 798
 799         self.assertEqual(collection.manifest_text(), "")
 800
 801
 802 def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 803     class Test(unittest.TestCase):
 804         def runTest(self):
 805             with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
 806                 f.write("Hello world!")
 807
 808             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 809             self.assertEqual(["file1.txt"], sorted(d1))
 810             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 811             self.assertEqual([], sorted(d1))
 812
 813     Test().runTest()
 814
 815 def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 816     class Test(unittest.TestCase):
 817         def runTest(self):
 818             os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
 819
 820             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 821             self.assertEqual([], sorted(d1))
 822             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 823             self.assertEqual(["file2.txt"], sorted(d1))
 824
 825     Test().runTest()
 826
 827 class FuseMvFileBetweenCollectionsTest(MountTestBase):
 828     def runTest(self):
 829         collection1 = arvados.collection.Collection(api_client=self.api)
 830         collection1.save_new()
 831
 832         collection2 = arvados.collection.Collection(api_client=self.api)
 833         collection2.save_new()
 834
 835         m = self.make_mount(fuse.MagicDirectory)
 836
 837         # See note in MountTestBase.setUp
 838         self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
 839                                                   collection1.manifest_locator(),
 840                                                   collection2.manifest_locator()))
 841
 842         collection1.update()
 843         collection2.update()
 844
 845         self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 846         self.assertEqual(collection2.manifest_text(), "")
 847
 848         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
 849                                                   collection1.manifest_locator(),
 850                                                   collection2.manifest_locator()))
 851
 852         collection1.update()
 853         collection2.update()
 854
 855         self.assertEqual(collection1.manifest_text(), "")
 856         self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
 857
 858         collection1.stop_threads()
 859         collection2.stop_threads()
 860
 861
 862 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
 863     class Test(unittest.TestCase):
 864         def runTest(self):
 865             os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
 866             with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
 867                 f.write("Hello world!")
 868
 869             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 870             self.assertEqual(["testdir"], sorted(d1))
 871             d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
 872             self.assertEqual(["file1.txt"], sorted(d1))
 873
 874             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 875             self.assertEqual([], sorted(d1))
 876
 877     Test().runTest()
 878
 879
 880 def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
 881     class Test(unittest.TestCase):
 882         def runTest(self):
 883             os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
 884
 885             d1 = os.listdir(os.path.join(mounttmp, uuid1))
 886             self.assertEqual([], sorted(d1))
 887
 888             d1 = os.listdir(os.path.join(mounttmp, uuid2))
 889             self.assertEqual(["testdir2"], sorted(d1))
 890             d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
 891             self.assertEqual(["file1.txt"], sorted(d1))
 892
 893             with open(os.path.join(mounttmp, uuid2, "testdir2", "file1.txt"), "r") as f:
 894                 self.assertEqual(f.read(), "Hello world!")
 895
 896     Test().runTest()
 897
 898 class FuseMvDirBetweenCollectionsTest(MountTestBase):
 899     def runTest(self):
 900         collection1 = arvados.collection.Collection(api_client=self.api)
 901         collection1.save_new()
 902
 903         collection2 = arvados.collection.Collection(api_client=self.api)
 904         collection2.save_new()
 905
 906         m = self.make_mount(fuse.MagicDirectory)
 907
 908         # See note in MountTestBase.setUp
 909         self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
 910                                                   collection1.manifest_locator(),
 911                                                   collection2.manifest_locator()))
 912
 913         collection1.update()
 914         collection2.update()
 915
 916         self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 917         self.assertEqual(collection2.manifest_text(), "")
 918
 919         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
 920                                                   collection1.manifest_locator(),
 921                                                   collection2.manifest_locator()))
 922
 923         collection1.update()
 924         collection2.update()
 925
 926         self.assertEqual(collection1.manifest_text(), "")
 927         self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 928
 929         collection1.stop_threads()
 930         collection2.stop_threads()
 931
 932 def fuseProjectMkdirTestHelper1(mounttmp):
 933     class Test(unittest.TestCase):
 934         def runTest(self):
 935             os.mkdir(os.path.join(mounttmp, "testcollection"))
 936             with self.assertRaises(OSError):
 937                 os.mkdir(os.path.join(mounttmp, "testcollection"))
 938     Test().runTest()
 939
 940 def fuseProjectMkdirTestHelper2(mounttmp):
 941     class Test(unittest.TestCase):
 942         def runTest(self):
 943             with open(os.path.join(mounttmp, "testcollection", "file1.txt"), "w") as f:
 944                 f.write("Hello world!")
 945             with self.assertRaises(OSError):
 946                 os.rmdir(os.path.join(mounttmp, "testcollection"))
 947             os.remove(os.path.join(mounttmp, "testcollection", "file1.txt"))
 948             with self.assertRaises(OSError):
 949                 os.remove(os.path.join(mounttmp, "testcollection"))
 950             os.rmdir(os.path.join(mounttmp, "testcollection"))
 951     Test().runTest()
 952
 953 class FuseProjectMkdirRmdirTest(MountTestBase):
 954     def runTest(self):
 955         self.make_mount(fuse.ProjectDirectory,
 956                         project_object=self.api.users().current().execute())
 957
 958         d1 = llfuse.listdir(self.mounttmp)
 959         self.assertNotIn('testcollection', d1)
 960
 961         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
 962
 963         d1 = llfuse.listdir(self.mounttmp)
 964         self.assertIn('testcollection', d1)
 965
 966         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
 967
 968         d1 = llfuse.listdir(self.mounttmp)
 969         self.assertNotIn('testcollection', d1)
 970
 971
 972 def fuseProjectMvTestHelper1(mounttmp):
 973     class Test(unittest.TestCase):
 974         def runTest(self):
 975             d1 = llfuse.listdir(mounttmp)
 976             self.assertNotIn('testcollection', d1)
 977
 978             os.mkdir(os.path.join(mounttmp, "testcollection"))
 979
 980             d1 = llfuse.listdir(mounttmp)
 981             self.assertIn('testcollection', d1)
 982
 983             with self.assertRaises(OSError):
 984                 os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
 985
 986             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
 987
 988             d1 = llfuse.listdir(mounttmp)
 989             self.assertNotIn('testcollection', d1)
 990
 991             d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
 992             self.assertIn('testcollection', d1)
 993
 994     Test().runTest()
 995
 996 class FuseProjectMvTest(MountTestBase):
 997     def runTest(self):
 998         self.make_mount(fuse.ProjectDirectory,
 999                         project_object=self.api.users().current().execute())
1000
1001         self.pool.apply(fuseProjectMvTestHelper1, (self.mounttmp,))
1002
1003
1004 def fuseFsyncTestHelper(mounttmp, k):
1005     class Test(unittest.TestCase):
1006         def runTest(self):
1007             fd = os.open(os.path.join(mounttmp, k), os.O_RDONLY)
1008             os.fsync(fd)
1009             os.close(fd)
1010
1011     Test().runTest()
1012
1013 class FuseFsyncTest(FuseMagicTest):
1014     def runTest(self):
1015         self.make_mount(fuse.MagicDirectory)
1016         self.pool.apply(fuseFsyncTestHelper, (self.mounttmp, self.testcollection))
1017
1018
1019 class MagicDirApiError(FuseMagicTest):
1020     def setUp(self):
1021         api = mock.MagicMock()
1022         super(MagicDirApiError, self).setUp(api=api)
1023         api.collections().get().execute.side_effect = iter([Exception('API fail'), {"manifest_text": self.test_manifest}])
1024         api.keep.get.side_effect = Exception('Keep fail')
1025
1026     def runTest(self):
1027         self.make_mount(fuse.MagicDirectory)
1028
1029         self.operations.inodes.inode_cache.cap = 1
1030         self.operations.inodes.inode_cache.min_entries = 2
1031
1032         with self.assertRaises(OSError):
1033             llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1034
1035         llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
1036
1037
1038 class FuseUnitTest(unittest.TestCase):
1039     def test_sanitize_filename(self):
1040         acceptable = [
1041             "foo.txt",
1042             ".foo",
1043             "..foo",
1044             "...",
1045             "foo...",
1046             "foo..",
1047             "foo.",
1048             "-",
1049             "\x01\x02\x03",
1050             ]
1051         unacceptable = [
1052             "f\00",
1053             "\00\00",
1054             "/foo",
1055             "foo/",
1056             "//",
1057             ]
1058         for f in acceptable:
1059             self.assertEqual(f, fuse.sanitize_filename(f))
1060         for f in unacceptable:
1061             self.assertNotEqual(f, fuse.sanitize_filename(f))
1062             # The sanitized filename should be the same length, though.
1063             self.assertEqual(len(f), len(fuse.sanitize_filename(f)))
1064         # Special cases
1065         self.assertEqual("_", fuse.sanitize_filename(""))
1066         self.assertEqual("_", fuse.sanitize_filename("."))
1067         self.assertEqual("__", fuse.sanitize_filename(".."))
1068
1069
1070 class FuseMagicTestPDHOnly(MountTestBase):
1071     def setUp(self, api=None):
1072         super(FuseMagicTestPDHOnly, self).setUp(api=api)
1073
1074         cw = arvados.CollectionWriter()
1075
1076         cw.start_new_file('thing1.txt')
1077         cw.write("data 1")
1078
1079         self.testcollection = cw.finish()
1080         self.test_manifest = cw.manifest_text()
1081         created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
1082         self.testcollectionuuid = str(created['uuid'])
1083
1084     def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
1085         if skip_pdh_only is True:
1086             self.make_mount(fuse.MagicDirectory)    # in this case, the default by_id applies
1087         else:
1088             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
1089
1090         mount_ls = llfuse.listdir(self.mounttmp)
1091         self.assertIn('README', mount_ls)
1092         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
1093                              arvados.util.uuid_pattern.match(fn)
1094                              for fn in mount_ls),
1095                          "new FUSE MagicDirectory lists Collection")
1096
1097         # look up using pdh should succeed in all cases
1098         self.assertDirContents(self.testcollection, ['thing1.txt'])
1099         self.assertDirContents(os.path.join('by_id', self.testcollection),
1100                                ['thing1.txt'])
1101         mount_ls = llfuse.listdir(self.mounttmp)
1102         self.assertIn('README', mount_ls)
1103         self.assertIn(self.testcollection, mount_ls)
1104         self.assertIn(self.testcollection,
1105                       llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
1106
1107         files = {}
1108         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
1109
1110         for k, v in files.items():
1111             with open(os.path.join(self.mounttmp, k)) as f:
1112                 self.assertEqual(v, f.read())
1113
1114         # look up using uuid should fail when pdh_only is set
1115         if pdh_only is True:
1116             with self.assertRaises(OSError):
1117                 self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1118                                ['thing1.txt'])
1119         else:
1120             self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
1121                                ['thing1.txt'])
1122
1123     def test_with_pdh_only_true(self):
1124         self.verify_pdh_only(pdh_only=True)
1125
1126     def test_with_pdh_only_false(self):
1127         self.verify_pdh_only(pdh_only=False)
1128
1129     def test_with_default_by_id(self):
1130         self.verify_pdh_only(skip_pdh_only=True)
1131
1132 def _test_refresh_old_manifest(zzz):
1133     fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt'
1134     os.listdir(os.path.join(zzz))
1135     time.sleep(3)
1136     with open(os.path.join(zzz, fnm)) as f:
1137         f.read()
1138
1139 class TokenExpiryTest(MountTestBase):
1140     def setUp(self):
1141         super(TokenExpiryTest, self).setUp(local_store=False)
1142
1143     @mock.patch('arvados.keep.KeepClient.get')
1144     def runTest(self, mocked_get):
1145         logging.getLogger('arvados.arvados_fuse').setLevel(logging.DEBUG)
1146         self.api._rootDesc = {"blobSignatureTtl": 2}
1147         mnt = self.make_mount(fuse.CollectionDirectory, collection_record='zzzzz-4zz18-op4e2lbej01tcvu')
1148         mocked_get.return_value = 'fake data'
1149
1150         old_exp = int(time.time()) + 86400*14
1151         self.pool.apply(_test_refresh_old_manifest, (self.mounttmp,))
1152         want_exp = int(time.time()) + 86400*14
1153
1154         got_loc = mocked_get.call_args[0][0]
1155         got_exp = int(
1156             re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1),
1157             16)
1158         self.assertGreaterEqual(
1159             got_exp, want_exp-1,
1160             msg='now+2w = {:x}, but fuse fetched locator {} (old_exp {:x})'.format(
1161                 want_exp, got_loc, old_exp))
1162         self.assertLessEqual(
1163             got_exp, want_exp,
1164             msg='server is not using the expected 2w TTL; test is ineffective')