1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
18 import urllib.parse as urlparse
20 from unittest import mock
21 from . import run_test_server
23 from apiclient import errors as apiclient_errors
24 from apiclient import http as apiclient_http
25 from arvados.api import (
29 api_kwargs_from_config,
30 _googleapiclient_log_lock,
32 from .arvados_testutil import fake_httplib2_response, mock_api_responses, queue_with
34 import googleapiclient
37 if not mimetypes.inited:
40 class ArvadosApiTest(run_test_server.TestCaseWithServers):
42 ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
43 RETRIED_4XX = frozenset([408, 409, 423])
45 def api_error_response(self, code, *errors):
46 return (fake_httplib2_response(code, **self.ERROR_HEADERS),
47 json.dumps({'errors': errors,
48 'error_token': '1234567890+12345678'}).encode())
50 def _config_from_environ(self):
53 for key, value in os.environ.items()
54 if key.startswith('ARVADOS_API_')
57 def _discoveryServiceUrl(
60 path='/discovery/v1/apis/{api}/{apiVersion}/rest',
64 host = os.environ['ARVADOS_API_HOST']
65 return urlparse.urlunsplit((scheme, host, path, None, None))
67 def test_new_api_objects_with_cache(self):
68 clients = [arvados.api('v1', cache=True) for index in [0, 1]]
69 self.assertIsNot(*clients)
71 def test_empty_list(self):
72 answer = arvados.api('v1').collections().list(
73 filters=[['uuid', '=', 'abcdef']]).execute()
74 self.assertEqual(answer['items_available'], len(answer['items']))
76 def test_nonempty_list(self):
77 answer = arvados.api('v1').collections().list().execute()
78 self.assertNotEqual(0, answer['items_available'])
79 self.assertNotEqual(0, len(answer['items']))
81 def test_timestamp_inequality_filter(self):
82 api = arvados.api('v1')
83 new_item = api.collections().create(body={}).execute()
84 for operator, should_include in [
85 ['<', False], ['>', False],
86 ['<=', True], ['>=', True], ['=', True]]:
87 response = api.collections().list(filters=[
88 ['created_at', operator, new_item['created_at']],
89 # Also filter by uuid to ensure (if it matches) it's on page 0
90 ['uuid', '=', new_item['uuid']]]).execute()
91 uuids = [item['uuid'] for item in response['items']]
92 did_include = new_item['uuid'] in uuids
94 did_include, should_include,
95 "'%s %s' filter should%s have matched '%s'" % (
96 operator, new_item['created_at'],
97 ('' if should_include else ' not'),
98 new_item['created_at']))
100 def test_exceptions_include_errors(self):
102 'arvados.collections.get': self.api_error_response(
103 422, "Bad UUID format", "Bad output format"),
105 req_builder = apiclient_http.RequestMockBuilder(mock_responses)
106 api = arvados.api('v1', requestBuilder=req_builder)
107 with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
108 api.collections().get(uuid='xyz-xyz-abcdef').execute()
109 err_s = str(err_ctx.exception)
110 for msg in ["Bad UUID format", "Bad output format"]:
111 self.assertIn(msg, err_s)
113 @mock.patch('time.sleep')
114 def test_exceptions_include_request_id(self, sleep):
115 api = arvados.api('v1')
116 api.request_id='fake-request-id'
117 api._http.orig_http_request = mock.MagicMock()
118 api._http.orig_http_request.side_effect = socket.error('mock error')
121 api.users().current().execute()
122 except Exception as e:
124 self.assertRegex(str(caught), r'fake-request-id')
126 def test_exceptions_without_errors_have_basic_info(self):
128 'arvados.collections.delete': (
129 fake_httplib2_response(500, **self.ERROR_HEADERS),
132 req_builder = apiclient_http.RequestMockBuilder(mock_responses)
133 api = arvados.api('v1', requestBuilder=req_builder)
134 with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
135 api.collections().delete(uuid='xyz-xyz-abcdef').execute()
136 self.assertIn("500", str(err_ctx.exception))
138 def test_request_too_large(self):
139 api = arvados.api('v1')
140 maxsize = api._rootDesc.get('maxRequestSize', 0)
141 with self.assertRaises(apiclient_errors.MediaUploadSizeError):
143 arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
145 def test_default_request_timeout(self):
146 api = arvados.api('v1')
147 self.assertEqual(api._http.timeout, 300,
148 "Default timeout value should be 300")
150 def test_custom_request_timeout(self):
151 api = arvados.api('v1', timeout=1234)
152 self.assertEqual(api._http.timeout, 1234,
153 "Requested timeout value was 1234")
155 def test_4xx_retried(self):
156 client = arvados.api('v1')
157 for code in self.RETRIED_4XX:
158 name = f'retried #{code}'
159 with self.subTest(name), mock.patch('time.sleep'):
160 expected = {'username': name}
161 with mock_api_responses(
163 json.dumps(expected),
168 actual = client.users().current().execute()
169 self.assertEqual(actual, expected)
171 def test_4xx_not_retried(self):
172 client = arvados.api('v1', num_retries=3)
173 # Note that googleapiclient does retry 403 *if* the response JSON
174 # includes flags that say the request was denied by rate limiting.
175 # An empty JSON response like we use here should not be retried.
176 for code in [400, 401, 403, 404, 422]:
177 with self.subTest(f'error {code}'), mock.patch('time.sleep'):
178 with mock_api_responses(
184 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
185 client.users().current().execute()
186 response = exc_check.exception.args[0]
187 self.assertEqual(response.status, code)
188 self.assertEqual(response.get('status'), str(code))
190 def test_4xx_raised_after_retry_exhaustion(self):
191 client = arvados.api('v1', num_retries=1)
192 for code in self.RETRIED_4XX:
193 with self.subTest(f'failed {code}'), mock.patch('time.sleep'):
194 with mock_api_responses(
197 [code, code, code, 200],
200 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
201 client.users().current().execute()
202 response = exc_check.exception.args[0]
203 self.assertEqual(response.status, code)
204 self.assertEqual(response.get('status'), str(code))
206 def test_api_is_threadsafe(self):
208 'host': os.environ['ARVADOS_API_HOST'],
209 'token': os.environ['ARVADOS_API_TOKEN'],
212 config_kwargs = {'apiconfig': os.environ}
213 for api_constructor, kwargs in [
215 (arvados.api, api_kwargs),
216 (arvados.api_from_config, {}),
217 (arvados.api_from_config, config_kwargs),
219 sub_kwargs = "kwargs" if kwargs else "no kwargs"
220 with self.subTest(f"{api_constructor.__name__} with {sub_kwargs}"):
221 api_client = api_constructor('v1', **kwargs)
222 self.assertTrue(hasattr(api_client, 'localapi'),
223 f"client missing localapi method")
224 self.assertTrue(hasattr(api_client, 'keep'),
225 f"client missing keep attribute")
227 def test_api_host_constructor(self):
230 client = arvados.api(
233 os.environ['ARVADOS_API_HOST'],
234 os.environ['ARVADOS_API_TOKEN'],
237 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
238 "client constructed with incorrect token")
240 def test_api_url_constructor(self):
241 client = arvados.api(
243 discoveryServiceUrl=self._discoveryServiceUrl(),
244 token=os.environ['ARVADOS_API_TOKEN'],
247 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
248 "client constructed with incorrect token")
250 def test_api_bad_args(self):
252 'host': os.environ['ARVADOS_API_HOST'],
253 'token': os.environ['ARVADOS_API_TOKEN'],
254 'discoveryServiceUrl': self._discoveryServiceUrl(),
257 # Passing only a single key is missing required info
258 *([key] for key in all_kwargs.keys()),
259 # Passing all keys is a conflict
260 list(all_kwargs.keys()),
262 kwargs = {key: all_kwargs[key] for key in use_keys}
263 kwargs_list = ', '.join(use_keys)
264 with self.subTest(f"calling arvados.api with {kwargs_list} fails"), \
265 self.assertRaises(ValueError):
266 arvados.api('v1', insecure=True, **kwargs)
268 def test_api_bad_url(self):
270 {'discoveryServiceUrl': self._discoveryServiceUrl() + '/BadTestURL'},
271 {'version': 'BadTestVersion', 'host': os.environ['ARVADOS_API_HOST']},
273 bad_key = next(iter(bad_kwargs))
274 with self.subTest(f"api fails with bad {bad_key}"), \
275 self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
276 arvados.api(**bad_kwargs, token='test_api_bad_url', insecure=True)
278 def test_normalize_api_good_args(self):
279 for version, discoveryServiceUrl, host in [
280 ('Test1', None, os.environ['ARVADOS_API_HOST']),
281 (None, self._discoveryServiceUrl(), None)
283 argname = 'discoveryServiceUrl' if host is None else 'host'
284 with self.subTest(f"normalize_api_kwargs with {argname}"):
285 actual = normalize_api_kwargs(
289 os.environ['ARVADOS_API_TOKEN'],
292 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
293 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
294 self.assertEqual(actual['version'], version or 'v1')
295 self.assertTrue(actual['insecure'])
296 self.assertNotIn('host', actual)
298 def test_normalize_api_bad_args(self):
300 self._discoveryServiceUrl(),
301 os.environ['ARVADOS_API_HOST'],
302 os.environ['ARVADOS_API_TOKEN'],
304 for arg_index, arg_value in enumerate(all_args):
305 args = [None] * len(all_args)
306 args[arg_index] = arg_value
307 with self.subTest(f"normalize_api_kwargs with only arg #{arg_index + 1}"), \
308 self.assertRaises(ValueError):
309 normalize_api_kwargs('v1', *args)
310 with self.subTest("normalize_api_kwargs with discoveryServiceUrl and host"), \
311 self.assertRaises(ValueError):
312 normalize_api_kwargs('v1', *all_args)
314 def test_api_from_config_default(self):
315 client = arvados.api_from_config('v1')
316 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
317 "client constructed with incorrect token")
319 def test_api_from_config_explicit(self):
320 config = self._config_from_environ()
321 client = arvados.api_from_config('v1', config)
322 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
323 "client constructed with incorrect token")
325 def test_api_from_bad_config(self):
326 base_config = self._config_from_environ()
327 for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
328 with self.subTest(f"api_from_config without {del_key} fails"), \
329 self.assertRaises(ValueError):
330 config = dict(base_config)
332 arvados.api_from_config('v1', config)
334 def test_api_kwargs_from_good_config(self):
335 for config in [None, self._config_from_environ()]:
336 conf_type = 'default' if config is None else 'passed'
337 with self.subTest(f"api_kwargs_from_config with {conf_type} config"):
338 version = 'Test1' if config else None
339 actual = api_kwargs_from_config(version, config)
340 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
341 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
342 self.assertEqual(actual['version'], version or 'v1')
343 self.assertTrue(actual['insecure'])
344 self.assertNotIn('host', actual)
346 def test_api_kwargs_from_bad_config(self):
347 base_config = self._config_from_environ()
348 for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
349 with self.subTest(f"api_kwargs_from_config without {del_key} fails"), \
350 self.assertRaises(ValueError):
351 config = dict(base_config)
353 api_kwargs_from_config('v1', config)
355 def test_api_client_constructor(self):
358 self._discoveryServiceUrl(),
359 os.environ['ARVADOS_API_TOKEN'],
362 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
363 "client constructed with incorrect token")
365 hasattr(client, 'localapi'),
366 "client has localapi method when it should not be thread-safe",
369 def test_api_client_bad_url(self):
370 all_args = ('v1', self._discoveryServiceUrl(), 'test_api_client_bad_url')
371 for arg_index, arg_value in [
372 (0, 'BadTestVersion'),
373 (1, all_args[1] + '/BadTestURL'),
375 with self.subTest(f"api_client fails with {arg_index}={arg_value!r}"), \
376 self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
377 args = list(all_args)
378 args[arg_index] = arg_value
379 api_client(*args, insecure=True)
381 def test_initial_retry_logs(self):
383 _googleapiclient_log_lock.release()
385 # Lock was never acquired - that's the state we want anyway
387 real_logger = logging.getLogger('googleapiclient.http')
388 mock_logger = mock.Mock(wraps=real_logger)
389 mock_logger.handlers = logging.getLogger('googleapiclient').handlers
390 mock_logger.level = logging.NOTSET
391 with mock.patch('logging.getLogger', return_value=mock_logger), \
392 mock.patch('time.sleep'), \
393 self.assertLogs(real_logger, 'INFO') as actual_logs:
395 api_client('v1', 'https://test.invalid/', 'NoToken', num_retries=1)
396 except httplib2.error.ServerNotFoundError:
398 mock_logger.addFilter.assert_called()
399 mock_logger.addHandler.assert_called()
400 mock_logger.setLevel.assert_called()
401 mock_logger.removeHandler.assert_called()
402 mock_logger.removeFilter.assert_called()
403 self.assertRegex(actual_logs.output[0], r'^INFO:googleapiclient\.http:Sleeping \d')
405 def test_configured_logger_untouched(self):
406 real_logger = logging.getLogger('googleapiclient.http')
407 mock_logger = mock.Mock(wraps=real_logger)
408 mock_logger.handlers = logging.getLogger().handlers
409 with mock.patch('logging.getLogger', return_value=mock_logger), \
410 mock.patch('time.sleep'):
412 api_client('v1', 'https://test.invalid/', 'NoToken', num_retries=1)
413 except httplib2.error.ServerNotFoundError:
415 mock_logger.addFilter.assert_not_called()
416 mock_logger.addHandler.assert_not_called()
417 mock_logger.setLevel.assert_not_called()
418 mock_logger.removeHandler.assert_not_called()
419 mock_logger.removeFilter.assert_not_called()
422 class ConstructNumRetriesTestCase(unittest.TestCase):
424 def _fake_retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs):
425 return http.request(uri, method, *args, **kwargs)
427 @contextlib.contextmanager
428 def patch_retry(self):
429 # We have this dedicated context manager that goes through `sys.modules`
430 # instead of just using `mock.patch` because of the unfortunate
431 # `arvados.api` name collision.
432 orig_func = sys.modules['arvados.api']._orig_retry_request
433 expect_name = 'googleapiclient.http._retry_request'
435 '{0.__module__}.{0.__name__}'.format(orig_func), expect_name,
436 f"test setup problem: {expect_name} not at arvados.api._orig_retry_request",
438 retry_mock = mock.Mock(wraps=self._fake_retry_request)
439 sys.modules['arvados.api']._orig_retry_request = retry_mock
443 sys.modules['arvados.api']._orig_retry_request = orig_func
445 def _iter_num_retries(self, retry_mock):
446 for call in retry_mock.call_args_list:
450 yield call.kwargs['num_retries']
452 def test_default_num_retries(self):
453 with self.patch_retry() as retry_mock:
454 client = arvados.api('v1')
455 actual = set(self._iter_num_retries(retry_mock))
456 self.assertEqual(len(actual), 1)
457 self.assertTrue(actual.pop() > 6, "num_retries lower than expected")
459 def _test_calls(self, init_arg, call_args, expected):
460 with self.patch_retry() as retry_mock:
461 client = arvados.api('v1', num_retries=init_arg)
462 for num_retries in call_args:
463 client.users().current().execute(num_retries=num_retries)
464 actual = self._iter_num_retries(retry_mock)
465 # The constructor makes two requests with its num_retries argument:
466 # one for the discovery document, and one for the config.
467 self.assertEqual(next(actual, None), init_arg)
468 self.assertEqual(next(actual, None), init_arg)
469 self.assertEqual(list(actual), expected)
471 def test_discovery_num_retries(self):
472 for num_retries in [0, 5, 55]:
473 with self.subTest(f"num_retries={num_retries}"):
474 self._test_calls(num_retries, [], [])
476 def test_num_retries_called_le_init(self):
478 with self.subTest(f"init_arg={n}"):
479 call_args = [n - 4, n - 2, n]
481 self._test_calls(n, call_args, expected)
483 def test_num_retries_called_ge_init(self):
485 with self.subTest(f"init_arg={n}"):
486 call_args = [n, n + 4, n + 8]
487 self._test_calls(n, call_args, call_args)
489 def test_num_retries_called_mixed(self):
490 self._test_calls(5, [2, 6, 4, 8], [5, 6, 5, 8])
493 class PreCloseSocketTestCase(unittest.TestCase):
495 self.api = arvados.api('v1')
496 self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
497 "test doesn't know how to intercept HTTP requests")
498 self.mock_response = {'user': 'person'}
499 self.request_success = (fake_httplib2_response(200),
500 json.dumps(self.mock_response))
501 self.api._http.orig_http_request = mock.MagicMock()
502 # All requests succeed by default. Tests override as needed.
503 self.api._http.orig_http_request.return_value = self.request_success
505 @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
506 def test_close_old_connections_non_retryable(self, sleep):
507 self._test_connection_close(expect=1)
509 @mock.patch('time.time', side_effect=itertools.count())
510 def test_no_close_fresh_connections_non_retryable(self, sleep):
511 self._test_connection_close(expect=0)
513 @mock.patch('time.time', side_effect=itertools.count())
514 def test_override_max_idle_time(self, sleep):
515 self.api._http._max_keepalive_idle = 0
516 self._test_connection_close(expect=1)
518 def _test_connection_close(self, expect=0):
519 # Do two POST requests. The second one must close all
520 # connections +expect+ times.
521 self.api.users().create(body={}).execute()
522 mock_conns = {str(i): mock.MagicMock() for i in range(2)}
523 self.api._http.connections = mock_conns.copy()
524 self.api.users().create(body={}).execute()
525 for c in mock_conns.values():
526 self.assertEqual(c.close.call_count, expect)
529 class ThreadSafeAPIClientTestCase(run_test_server.TestCaseWithServers):
532 def test_constructor(self):
535 for key, value in os.environ.items()
536 if key.startswith('ARVADOS_API_')
542 key[12:].lower(): value
543 for key, value in env_mapping.items()
546 base_params['insecure'] = base_params.pop('host_insecure')
549 expected_keep_params = {}
550 for config, params, subtest in [
551 (None, {}, "default arguments"),
552 (None, extra_params, "extra params"),
553 (env_mapping, {}, "explicit config"),
554 (env_mapping, extra_params, "explicit config and params"),
555 ({}, base_params, "params only"),
557 with self.subTest(f"test constructor with {subtest}"):
558 expected_timeout = params.get('timeout', 300)
559 expected_params = dict(params)
560 keep_params = dict(expected_keep_params)
561 client = ThreadSafeAPIClient(config, keep_params, params, 'v1')
562 self.assertTrue(hasattr(client, 'localapi'), "client missing localapi method")
563 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'])
564 self.assertEqual(client._http.timeout, expected_timeout)
565 self.assertEqual(params, expected_params,
566 "api_params was modified in-place")
567 self.assertEqual(keep_params, expected_keep_params,
568 "keep_params was modified in-place")
570 def test_constructor_no_args(self):
571 client = ThreadSafeAPIClient()
572 self.assertTrue(hasattr(client, 'localapi'), "client missing localapi method")
573 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'])
574 self.assertTrue(client.insecure)
576 def test_constructor_bad_version(self):
577 with self.assertRaises(googleapiclient.errors.UnknownApiNameOrVersion):
578 ThreadSafeAPIClient(version='BadTestVersion')
580 def test_pre_v3_0_name(self):
581 from arvados.safeapi import ThreadSafeApiCache
582 self.assertIs(ThreadSafeApiCache, ThreadSafeAPIClient)
585 if __name__ == '__main__':