X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/8227b8a1be943fbbf3adb23a3d549dec28efbbba..3d99c1541a450411a847c1c2b87721a4c51b484e:/sdk/python/tests/test_keep_client.py diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py index 0eefa586d9..722b70b58b 100644 --- a/sdk/python/tests/test_keep_client.py +++ b/sdk/python/tests/test_keep_client.py @@ -202,22 +202,6 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers): 'wrong content from Keep.get(md5("baz"))') self.assertTrue(keep_client.using_proxy) - def test_KeepProxyTest2(self): - # Don't instantiate the proxy directly, but set the X-External-Client - # header. The API server should direct us to the proxy. - arvados.config.settings()['ARVADOS_EXTERNAL_CLIENT'] = 'true' - keep_client = arvados.KeepClient(api_client=self.api_client, - proxy='', local_store='') - baz_locator = keep_client.put('baz2') - self.assertRegex( - baz_locator, - '^91f372a266fe2bf2823cb8ec7fda31ce\+4', - 'wrong md5 hash from Keep.put("baz2"): ' + baz_locator) - self.assertEqual(keep_client.get(baz_locator), - b'baz2', - 'wrong content from Keep.get(md5("baz2"))') - self.assertTrue(keep_client.using_proxy) - def test_KeepProxyTestMultipleURIs(self): # Test using ARVADOS_KEEP_SERVICES env var overriding any # existing proxy setting and setting multiple proxies @@ -254,6 +238,17 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): self.assertEqual('100::1', service.hostname) self.assertEqual(10, service.port) + def test_recognize_proxy_services_in_controller_response(self): + keep_client = arvados.KeepClient(api_client=self.mock_keep_services( + service_type='proxy', service_host='localhost', service_port=9, count=1)) + try: + # this will fail, but it ensures we get the service + # discovery response + keep_client.put('baz2') + except: + pass + self.assertTrue(keep_client.using_proxy) + def test_insecure_disables_tls_verify(self): api_client = self.mock_keep_services(count=1) force_timeout = socket.timeout("timed out") @@ -265,6 +260,9 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): self.assertEqual( mock.responses[0].getopt(pycurl.SSL_VERIFYPEER), 0) + self.assertEqual( + mock.responses[0].getopt(pycurl.SSL_VERIFYHOST), + 0) api_client.insecure = False with tutil.mock_keep_responses(b'foo', 200) as mock: @@ -276,6 +274,9 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): self.assertEqual( mock.responses[0].getopt(pycurl.SSL_VERIFYPEER), None) + self.assertEqual( + mock.responses[0].getopt(pycurl.SSL_VERIFYHOST), + None) def test_refresh_signature(self): blk_digest = '6f5902ac237024bdd0c176cb93063dc4+11' @@ -540,26 +541,49 @@ class KeepStorageClassesTestCase(unittest.TestCase, tutil.ApiClientMock): self.data = b'xyzzy' self.locator = '1271ed5ef305aadabc605b1609e24c52' + def test_multiple_default_storage_classes_req_header(self): + api_mock = self.api_client_mock() + api_mock.config.return_value = { + 'StorageClasses': { + 'foo': { 'Default': True }, + 'bar': { 'Default': True }, + 'baz': { 'Default': False } + } + } + api_client = self.mock_keep_services(api_mock=api_mock, count=2) + keep_client = arvados.KeepClient(api_client=api_client) + resp_hdr = { + 'x-keep-storage-classes-confirmed': 'foo=1, bar=1', + 'x-keep-replicas-stored': 1 + } + with tutil.mock_keep_responses(self.locator, 200, **resp_hdr) as mock: + keep_client.put(self.data, copies=1) + req_hdr = mock.responses[0] + self.assertIn( + 'X-Keep-Storage-Classes: bar, foo', req_hdr.getopt(pycurl.HTTPHEADER)) + def test_storage_classes_req_header(self): + self.assertEqual( + self.api_client.config()['StorageClasses'], + {'default': {'Default': True}}) cases = [ # requested, expected [['foo'], 'X-Keep-Storage-Classes: foo'], [['bar', 'foo'], 'X-Keep-Storage-Classes: bar, foo'], - [[], None], + [[], 'X-Keep-Storage-Classes: default'], + [None, 'X-Keep-Storage-Classes: default'], ] for req_classes, expected_header in cases: headers = {'x-keep-replicas-stored': 1} - if len(req_classes) > 0: + if req_classes is None or len(req_classes) == 0: + confirmed_hdr = 'default=1' + elif len(req_classes) > 0: confirmed_hdr = ', '.join(["{}=1".format(cls) for cls in req_classes]) - headers.update({'x-keep-storage-classes-confirmed': confirmed_hdr}) + headers.update({'x-keep-storage-classes-confirmed': confirmed_hdr}) with tutil.mock_keep_responses(self.locator, 200, **headers) as mock: self.keep_client.put(self.data, copies=1, classes=req_classes) - resp = mock.responses[0] - if expected_header is not None: - self.assertIn(expected_header, resp.getopt(pycurl.HTTPHEADER)) - else: - for hdr in resp.getopt(pycurl.HTTPHEADER): - self.assertNotRegex(hdr, r'^X-Keep-Storage-Classes.*') + req_hdr = mock.responses[0] + self.assertIn(expected_header, req_hdr.getopt(pycurl.HTTPHEADER)) def test_partial_storage_classes_put(self): headers = { @@ -680,6 +704,23 @@ class KeepXRequestIdTestCase(unittest.TestCase, tutil.ApiClientMock): self.keep_client.head(self.locator) self.assertAutomaticRequestId(mock.responses[0]) + def test_request_id_in_exception(self): + with tutil.mock_keep_responses(b'', 400, 400, 400) as mock: + with self.assertRaisesRegex(arvados.errors.KeepReadError, self.test_id): + self.keep_client.head(self.locator, request_id=self.test_id) + + with tutil.mock_keep_responses(b'', 400, 400, 400) as mock: + with self.assertRaisesRegex(arvados.errors.KeepReadError, r'req-[a-z0-9]{20}'): + self.keep_client.get(self.locator) + + with tutil.mock_keep_responses(b'', 400, 400, 400) as mock: + with self.assertRaisesRegex(arvados.errors.KeepWriteError, self.test_id): + self.keep_client.put(self.data, request_id=self.test_id) + + with tutil.mock_keep_responses(b'', 400, 400, 400) as mock: + with self.assertRaisesRegex(arvados.errors.KeepWriteError, r'req-[a-z0-9]{20}'): + self.keep_client.put(self.data) + def assertAutomaticRequestId(self, resp): hdr = [x for x in resp.getopt(pycurl.HTTPHEADER) if x.startswith('X-Request-Id: ')][0] @@ -1277,6 +1318,8 @@ class AvoidOverreplication(unittest.TestCase, tutil.ApiClientMock): def last_result(self): if self.will_succeed: return self._result + else: + return {"status_code": 500, "body": "didn't succeed"} def finished(self): return False @@ -1368,6 +1411,8 @@ class KeepClientAPIErrorTest(unittest.TestCase): return "abc" elif r == "insecure": return False + elif r == "config": + return lambda: {} else: raise arvados.errors.KeepReadError() keep_client = arvados.KeepClient(api_client=ApiMock(),