From d3313e652e392b1c0067bcddd65903ed0af19b44 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 21 Jan 2016 11:30:23 -0500 Subject: [PATCH] 6833: Test to confirm that enabling polling on CollectionDirectory causes tokens to be refreshed. --- sdk/python/tests/run_test_server.py | 5 ++- services/fuse/arvados_fuse/__init__.py | 8 +++-- services/fuse/tests/mount_test_base.py | 14 ++++++--- services/fuse/tests/test_mount.py | 37 ++++++++++++++++++++++ services/fuse/tests/test_token_expiry.py | 40 ------------------------ 5 files changed, 55 insertions(+), 49 deletions(-) delete mode 100644 services/fuse/tests/test_token_expiry.py diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index d6febdda6c..155bcedc62 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -422,7 +422,10 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2): # keepstore services. proxypidfile = _pidfile('keepproxy') if os.path.exists(proxypidfile): - os.kill(int(open(proxypidfile).read()), signal.SIGHUP) + try: + os.kill(int(open(proxypidfile).read()), signal.SIGHUP) + except OSError: + os.remove(proxypidfile) def _stop_keep(n): kill_server_pid(_pidfile('keep{}'.format(n))) diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py index 65f117b55a..f85af204a0 100644 --- a/services/fuse/arvados_fuse/__init__.py +++ b/services/fuse/arvados_fuse/__init__.py @@ -459,6 +459,7 @@ class Operations(llfuse.Operations): else: if parent_inode in self.inodes: p = self.inodes[parent_inode] + self.inodes.touch(p) if name == '..': inode = p.parent_inode elif isinstance(p, Directory) and name in p: @@ -500,11 +501,14 @@ class Operations(llfuse.Operations): fh = next(self._filehandles_counter) self._filehandles[fh] = FileHandle(fh, p) self.inodes.touch(p) + + _logger.debug("arv-mount open inode %i flags %x fh %i", inode, flags, fh) + return fh @catch_exceptions def read(self, fh, off, size): - _logger.debug("arv-mount read %i %i %i", fh, off, size) + _logger.debug("arv-mount read fh %i off %i size %i", fh, off, size) self.read_ops_counter.add(1) if fh in self._filehandles: @@ -587,8 +591,6 @@ class Operations(llfuse.Operations): else: raise llfuse.FUSEError(errno.EBADF) - _logger.debug("arv-mount handle.dirobj %s", handle.obj) - e = off while e < len(handle.entries): if handle.entries[e][1].inode in self.inodes: diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py index f3c020c9a4..44ec1996d2 100644 --- a/services/fuse/tests/mount_test_base.py +++ b/services/fuse/tests/mount_test_base.py @@ -17,7 +17,7 @@ import run_test_server logger = logging.getLogger('arvados.arv-mount') class MountTestBase(unittest.TestCase): - def setUp(self, api=None): + def setUp(self, api=None, local_store=True): # The underlying C implementation of open() makes a fstat() syscall # with the GIL still held. When the GETATTR message comes back to # llfuse (which in these tests is in the same interpreter process) it @@ -27,8 +27,11 @@ class MountTestBase(unittest.TestCase): # relatively easy. self.pool = multiprocessing.Pool(1) - self.keeptmp = tempfile.mkdtemp() - os.environ['KEEP_LOCAL_STORE'] = self.keeptmp + if local_store: + self.keeptmp = tempfile.mkdtemp() + os.environ['KEEP_LOCAL_STORE'] = self.keeptmp + else: + self.keeptmp = None self.mounttmp = tempfile.mkdtemp() run_test_server.run() run_test_server.authorize_with("admin") @@ -65,8 +68,9 @@ class MountTestBase(unittest.TestCase): self.operations.destroy() os.rmdir(self.mounttmp) - shutil.rmtree(self.keeptmp) - os.environ.pop('KEEP_LOCAL_STORE') + if self.keeptmp: + shutil.rmtree(self.keeptmp) + os.environ.pop('KEEP_LOCAL_STORE') run_test_server.reset() def assertDirContents(self, subdir, expect_content): diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py index 05c8685607..48158a74e0 100644 --- a/services/fuse/tests/test_mount.py +++ b/services/fuse/tests/test_mount.py @@ -16,6 +16,7 @@ import logging import multiprocessing import run_test_server import mock +import re from mount_test_base import MountTestBase @@ -1127,3 +1128,39 @@ class FuseMagicTestPDHOnly(MountTestBase): def test_with_default_by_id(self): self.verify_pdh_only(skip_pdh_only=True) + +def _test_refresh_old_manifest(zzz): + fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt' + os.listdir(os.path.join(zzz)) + time.sleep(3) + with open(os.path.join(zzz, fnm)) as f: + f.read() + +class TokenExpiryTest(MountTestBase): + def setUp(self): + super(TokenExpiryTest, self).setUp(local_store=False) + + @mock.patch('arvados.keep.KeepClient.get') + def runTest(self, mocked_get): + logging.getLogger('arvados.arvados_fuse').setLevel(logging.DEBUG) + mnt = self.make_mount(fuse.CollectionDirectory, collection_record='zzzzz-4zz18-op4e2lbej01tcvu') + mocked_get.return_value = 'fake data' + + mnt._poll = True + mnt._poll_time = 1 + + old_exp = int(time.time()) + 86400*14 + self.pool.apply(_test_refresh_old_manifest, (self.mounttmp,)) + want_exp = int(time.time()) + 86400*14 + + got_loc = mocked_get.call_args[0][0] + got_exp = int( + re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1), + 16) + self.assertGreaterEqual( + got_exp, want_exp-1, + msg='now+2w = {:x}, but fuse fetched locator {} (old_exp {:x})'.format( + want_exp, got_loc, old_exp)) + self.assertLessEqual( + got_exp, want_exp, + msg='server is not using the expected 2w TTL; test is ineffective') diff --git a/services/fuse/tests/test_token_expiry.py b/services/fuse/tests/test_token_expiry.py deleted file mode 100644 index df839e3278..0000000000 --- a/services/fuse/tests/test_token_expiry.py +++ /dev/null @@ -1,40 +0,0 @@ -import arvados -import logging -import mock -import os -import re -import time -import unittest - -from .integration_test import IntegrationTest -from .mount_test_base import MountTestBase - -logger = logging.getLogger('arvados.arv-mount') - -class TokenExpiryTest(IntegrationTest): - @mock.patch('arvados.keep.KeepClient.get') - @IntegrationTest.mount(argv=['--mount-by-id', 'zzz']) - def test_refresh_old_manifest(self, mocked_get): - mocked_get.return_value = 'fake data' - # TODO: mock something in arvados_fuse here so it thinks - # manifests/signatures expire in 1 second - self.pool_test(os.path.join(self.mnt, 'zzz')) - want_exp = int(time.time()) + 86400*14 - got_loc = mocked_get.call_args[0][0] - got_exp = int( - re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1), - 16) - self.assertGreaterEqual( - got_exp, want_exp-1, - msg='now+2w = {:x}, but fuse fetched old locator {}'.format( - want_exp, got_loc)) - self.assertLessEqual( - got_exp, want_exp, - msg='server is not using the expected 2w TTL; test is ineffective') - @staticmethod - def _test_refresh_old_manifest(self, zzz): - uuid = 'zzzzz-4zz18-op4e2lbej01tcvu' - fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt' - os.listdir(os.path.join(zzz, uuid)) - time.sleep(3) - open(os.path.join(zzz, uuid, fnm)).read() -- 2.30.2