X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/39c17737ac69d7693684fe2f95bef0ec235a28bf..f04693da1811e670d4cbb981debeecf14d79137c:/sdk/python/tests/test_keep_client.py diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py index a7b79933bb..80e6987b38 100644 --- a/sdk/python/tests/test_keep_client.py +++ b/sdk/python/tests/test_keep_client.py @@ -130,8 +130,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers): class KeepPermissionTestCase(run_test_server.TestCaseWithServers): MAIN_SERVER = {} - KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789', - 'enforce_permissions': True} + KEEP_SERVER = {'blob_signing': True} def test_KeepBasicRWTest(self): run_test_server.authorize_with('active') @@ -173,70 +172,6 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers): unsigned_bar_locator) -# KeepOptionalPermission: starts Keep with --permission-key-file -# but not --enforce-permissions (i.e. generate signatures on PUT -# requests, but do not require them for GET requests) -# -# All of these requests should succeed when permissions are optional: -# * authenticated request, signed locator -# * authenticated request, unsigned locator -# * unauthenticated request, signed locator -# * unauthenticated request, unsigned locator -class KeepOptionalPermission(run_test_server.TestCaseWithServers): - MAIN_SERVER = {} - KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789', - 'enforce_permissions': False} - - @classmethod - def setUpClass(cls): - super(KeepOptionalPermission, cls).setUpClass() - run_test_server.authorize_with("admin") - cls.api_client = arvados.api('v1') - - def setUp(self): - super(KeepOptionalPermission, self).setUp() - self.keep_client = arvados.KeepClient(api_client=self.api_client, - proxy='', local_store='') - - def _put_foo_and_check(self): - signed_locator = self.keep_client.put('foo') - self.assertRegex( - signed_locator, - r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$', - 'invalid locator from Keep.put("foo"): ' + signed_locator) - return signed_locator - - def test_KeepAuthenticatedSignedTest(self): - signed_locator = self._put_foo_and_check() - self.assertEqual(self.keep_client.get(signed_locator), - b'foo', - 'wrong content from Keep.get(md5("foo"))') - - def test_KeepAuthenticatedUnsignedTest(self): - signed_locator = self._put_foo_and_check() - self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"), - b'foo', - 'wrong content from Keep.get(md5("foo"))') - - def test_KeepUnauthenticatedSignedTest(self): - # Check that signed GET requests work even when permissions - # enforcement is off. - signed_locator = self._put_foo_and_check() - self.keep_client.api_token = '' - self.assertEqual(self.keep_client.get(signed_locator), - b'foo', - 'wrong content from Keep.get(md5("foo"))') - - def test_KeepUnauthenticatedUnsignedTest(self): - # Since --enforce-permissions is not in effect, GET requests - # need not be authenticated. - signed_locator = self._put_foo_and_check() - self.keep_client.api_token = '' - self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"), - b'foo', - 'wrong content from Keep.get(md5("foo"))') - - class KeepProxyTestCase(run_test_server.TestCaseWithServers): MAIN_SERVER = {} KEEP_SERVER = {} @@ -342,6 +277,25 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): mock.responses[0].getopt(pycurl.SSL_VERIFYPEER), None) + def test_refresh_signature(self): + blk_digest = '6f5902ac237024bdd0c176cb93063dc4+11' + blk_sig = 'da39a3ee5e6b4b0d3255bfef95601890afd80709@53bed294' + local_loc = blk_digest+'+A'+blk_sig + remote_loc = blk_digest+'+R'+blk_sig + api_client = self.mock_keep_services(count=1) + headers = {'X-Keep-Locator':local_loc} + with tutil.mock_keep_responses('', 200, **headers): + # Check that the translated locator gets returned + keep_client = arvados.KeepClient(api_client=api_client) + self.assertEqual(local_loc, keep_client.refresh_signature(remote_loc)) + # Check that refresh_signature() uses the correct method and headers + keep_client._get_or_head = mock.MagicMock() + keep_client.refresh_signature(remote_loc) + args, kwargs = keep_client._get_or_head.call_args_list[0] + self.assertIn(remote_loc, args) + self.assertEqual("HEAD", kwargs['method']) + self.assertIn('X-Keep-Signature', kwargs['headers']) + # test_*_timeout verify that KeepClient instructs pycurl to use # the appropriate connection and read timeouts. They don't care # whether pycurl actually exhibits the expected timeout behavior @@ -393,10 +347,10 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): int(arvados.KeepClient.DEFAULT_TIMEOUT[0]*1000)) self.assertEqual( mock.responses[0].getopt(pycurl.LOW_SPEED_TIME), - int(arvados.KeepClient.DEFAULT_TIMEOUT[1])) + None) self.assertEqual( mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT), - int(arvados.KeepClient.DEFAULT_TIMEOUT[2])) + None) def test_proxy_get_timeout(self): api_client = self.mock_keep_services(service_type='proxy', count=1) @@ -427,10 +381,10 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[0]*1000)) self.assertEqual( mock.responses[0].getopt(pycurl.LOW_SPEED_TIME), - int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[1])) + None) self.assertEqual( mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT), - int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[2])) + None) def test_proxy_put_timeout(self): api_client = self.mock_keep_services(service_type='proxy', count=1) @@ -541,6 +495,43 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): self.assertEqual(1, req_mock.call_count) +@tutil.skip_sleep +class KeepClientCacheTestCase(unittest.TestCase, tutil.ApiClientMock): + def setUp(self): + self.api_client = self.mock_keep_services(count=2) + self.keep_client = arvados.KeepClient(api_client=self.api_client) + self.data = b'xyzzy' + self.locator = '1271ed5ef305aadabc605b1609e24c52' + + @mock.patch('arvados.KeepClient.KeepService.get') + def test_get_request_cache(self, get_mock): + with tutil.mock_keep_responses(self.data, 200, 200): + self.keep_client.get(self.locator) + self.keep_client.get(self.locator) + # Request already cached, don't require more than one request + get_mock.assert_called_once() + + @mock.patch('arvados.KeepClient.KeepService.get') + def test_head_request_cache(self, get_mock): + with tutil.mock_keep_responses(self.data, 200, 200): + self.keep_client.head(self.locator) + self.keep_client.head(self.locator) + # Don't cache HEAD requests so that they're not confused with GET reqs + self.assertEqual(2, get_mock.call_count) + + @mock.patch('arvados.KeepClient.KeepService.get') + def test_head_and_then_get_return_different_responses(self, get_mock): + head_resp = None + get_resp = None + get_mock.side_effect = ['first response', 'second response'] + with tutil.mock_keep_responses(self.data, 200, 200): + head_resp = self.keep_client.head(self.locator) + get_resp = self.keep_client.get(self.locator) + self.assertEqual('first response', head_resp) + # First reponse was not cached because it was from a HEAD request. + self.assertNotEqual(head_resp, get_resp) + + @tutil.skip_sleep class KeepXRequestIdTestCase(unittest.TestCase, tutil.ApiClientMock): def setUp(self): @@ -830,7 +821,7 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase): loc = kc.put(self.DATA, copies=1, num_retries=0) self.server.setbandwidth(0.5*self.BANDWIDTH_LOW_LIM) with self.assertTakesGreater(self.TIMEOUT_TIME): - with self.assertRaises(arvados.errors.KeepReadError) as e: + with self.assertRaises(arvados.errors.KeepReadError): kc.get(loc, num_retries=0) with self.assertTakesGreater(self.TIMEOUT_TIME): with self.assertRaises(arvados.errors.KeepWriteError): @@ -842,14 +833,13 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase): self.server.setbandwidth(self.BANDWIDTH_LOW_LIM) self.server.setdelays(response=self.TIMEOUT_TIME) with self.assertTakesGreater(self.TIMEOUT_TIME): - with self.assertRaises(arvados.errors.KeepReadError) as e: + with self.assertRaises(arvados.errors.KeepReadError): kc.get(loc, num_retries=0) with self.assertTakesGreater(self.TIMEOUT_TIME): with self.assertRaises(arvados.errors.KeepWriteError): kc.put(self.DATA, copies=1, num_retries=0) with self.assertTakesGreater(self.TIMEOUT_TIME): - with self.assertRaises(arvados.errors.KeepReadError) as e: - kc.head(loc, num_retries=0) + kc.head(loc, num_retries=0) def test_low_bandwidth_with_server_mid_delay_failure(self): kc = self.keepClient()