X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/86183e3763202c53eb547a5432d291be9ed2986c..f89b45c97b047298c86dd58d5da8c07aa3d7d27e:/sdk/python/tests/test_keep_client.py diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py index a8b2f1570a..d6b3a2a12d 100644 --- a/sdk/python/tests/test_keep_client.py +++ b/sdk/python/tests/test_keep_client.py @@ -319,6 +319,48 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): self.assertEqual('100::1', service.hostname) self.assertEqual(10, service.port) + def test_insecure_disables_tls_verify(self): + api_client = self.mock_keep_services(count=1) + force_timeout = socket.timeout("timed out") + + api_client.insecure = True + with tutil.mock_keep_responses(b'foo', 200) as mock: + keep_client = arvados.KeepClient(api_client=api_client) + keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3') + self.assertEqual( + mock.responses[0].getopt(pycurl.SSL_VERIFYPEER), + 0) + + api_client.insecure = False + with tutil.mock_keep_responses(b'foo', 200) as mock: + keep_client = arvados.KeepClient(api_client=api_client) + keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3') + # getopt()==None here means we didn't change the + # default. If we were using real pycurl instead of a mock, + # it would return the default value 1. + self.assertEqual( + mock.responses[0].getopt(pycurl.SSL_VERIFYPEER), + None) + + def test_refresh_signature(self): + blk_digest = '6f5902ac237024bdd0c176cb93063dc4+11' + blk_sig = 'da39a3ee5e6b4b0d3255bfef95601890afd80709@53bed294' + local_loc = blk_digest+'+A'+blk_sig + remote_loc = blk_digest+'+R'+blk_sig + api_client = self.mock_keep_services(count=1) + headers = {'X-Keep-Locator':local_loc} + with tutil.mock_keep_responses('', 200, **headers): + # Check that the translated locator gets returned + keep_client = arvados.KeepClient(api_client=api_client) + self.assertEqual(local_loc, keep_client.refresh_signature(remote_loc)) + # Check that refresh_signature() uses the correct method and headers + keep_client._get_or_head = mock.MagicMock() + keep_client.refresh_signature(remote_loc) + args, kwargs = keep_client._get_or_head.call_args_list[0] + self.assertIn(remote_loc, args) + self.assertEqual("HEAD", kwargs['method']) + self.assertIn('X-Keep-Signature', kwargs['headers']) + # test_*_timeout verify that KeepClient instructs pycurl to use # the appropriate connection and read timeouts. They don't care # whether pycurl actually exhibits the expected timeout behavior @@ -370,10 +412,10 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): int(arvados.KeepClient.DEFAULT_TIMEOUT[0]*1000)) self.assertEqual( mock.responses[0].getopt(pycurl.LOW_SPEED_TIME), - int(arvados.KeepClient.DEFAULT_TIMEOUT[1])) + None) self.assertEqual( mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT), - int(arvados.KeepClient.DEFAULT_TIMEOUT[2])) + None) def test_proxy_get_timeout(self): api_client = self.mock_keep_services(service_type='proxy', count=1) @@ -404,10 +446,10 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[0]*1000)) self.assertEqual( mock.responses[0].getopt(pycurl.LOW_SPEED_TIME), - int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[1])) + None) self.assertEqual( mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT), - int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[2])) + None) def test_proxy_put_timeout(self): api_client = self.mock_keep_services(service_type='proxy', count=1) @@ -518,6 +560,43 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): self.assertEqual(1, req_mock.call_count) +@tutil.skip_sleep +class KeepClientCacheTestCase(unittest.TestCase, tutil.ApiClientMock): + def setUp(self): + self.api_client = self.mock_keep_services(count=2) + self.keep_client = arvados.KeepClient(api_client=self.api_client) + self.data = b'xyzzy' + self.locator = '1271ed5ef305aadabc605b1609e24c52' + + @mock.patch('arvados.KeepClient.KeepService.get') + def test_get_request_cache(self, get_mock): + with tutil.mock_keep_responses(self.data, 200, 200): + self.keep_client.get(self.locator) + self.keep_client.get(self.locator) + # Request already cached, don't require more than one request + get_mock.assert_called_once() + + @mock.patch('arvados.KeepClient.KeepService.get') + def test_head_request_cache(self, get_mock): + with tutil.mock_keep_responses(self.data, 200, 200): + self.keep_client.head(self.locator) + self.keep_client.head(self.locator) + # Don't cache HEAD requests so that they're not confused with GET reqs + self.assertEqual(2, get_mock.call_count) + + @mock.patch('arvados.KeepClient.KeepService.get') + def test_head_and_then_get_return_different_responses(self, get_mock): + head_resp = None + get_resp = None + get_mock.side_effect = ['first response', 'second response'] + with tutil.mock_keep_responses(self.data, 200, 200): + head_resp = self.keep_client.head(self.locator) + get_resp = self.keep_client.get(self.locator) + self.assertEqual('first response', head_resp) + # First reponse was not cached because it was from a HEAD request. + self.assertNotEqual(head_resp, get_resp) + + @tutil.skip_sleep class KeepXRequestIdTestCase(unittest.TestCase, tutil.ApiClientMock): def setUp(self): @@ -807,7 +886,7 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase): loc = kc.put(self.DATA, copies=1, num_retries=0) self.server.setbandwidth(0.5*self.BANDWIDTH_LOW_LIM) with self.assertTakesGreater(self.TIMEOUT_TIME): - with self.assertRaises(arvados.errors.KeepReadError) as e: + with self.assertRaises(arvados.errors.KeepReadError): kc.get(loc, num_retries=0) with self.assertTakesGreater(self.TIMEOUT_TIME): with self.assertRaises(arvados.errors.KeepWriteError): @@ -819,14 +898,13 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase): self.server.setbandwidth(self.BANDWIDTH_LOW_LIM) self.server.setdelays(response=self.TIMEOUT_TIME) with self.assertTakesGreater(self.TIMEOUT_TIME): - with self.assertRaises(arvados.errors.KeepReadError) as e: + with self.assertRaises(arvados.errors.KeepReadError): kc.get(loc, num_retries=0) with self.assertTakesGreater(self.TIMEOUT_TIME): with self.assertRaises(arvados.errors.KeepWriteError): kc.put(self.DATA, copies=1, num_retries=0) with self.assertTakesGreater(self.TIMEOUT_TIME): - with self.assertRaises(arvados.errors.KeepReadError) as e: - kc.head(loc, num_retries=0) + kc.head(loc, num_retries=0) def test_low_bandwidth_with_server_mid_delay_failure(self): kc = self.keepClient() @@ -1257,10 +1335,21 @@ class KeepClientAPIErrorTest(unittest.TestCase): def __getattr__(self, r): if r == "api_token": return "abc" + elif r == "insecure": + return False else: raise arvados.errors.KeepReadError() keep_client = arvados.KeepClient(api_client=ApiMock(), proxy='', local_store='') + + # The bug this is testing for is that if an API (not + # keepstore) exception is thrown as part of a get(), the next + # attempt to get that same block will result in a deadlock. + # This is why there are two get()s in a row. Unfortunately, + # the failure mode for this test is that the test suite + # deadlocks, there isn't a good way to avoid that without + # adding a special case that has no use except for this test. + with self.assertRaises(arvados.errors.KeepReadError): keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8+3") with self.assertRaises(arvados.errors.KeepReadError):