1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import absolute_import
6 from builtins import str
7 from builtins import range
20 import urllib.parse as urlparse
23 from . import run_test_server
25 from apiclient import errors as apiclient_errors
26 from apiclient import http as apiclient_http
27 from arvados.api import (
30 api_kwargs_from_config,
33 from .arvados_testutil import fake_httplib2_response, mock_api_responses, queue_with
35 if not mimetypes.inited:
38 class ArvadosApiTest(run_test_server.TestCaseWithServers):
40 ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
41 RETRIED_4XX = frozenset([408, 409, 422, 423])
43 def api_error_response(self, code, *errors):
44 return (fake_httplib2_response(code, **self.ERROR_HEADERS),
45 json.dumps({'errors': errors,
46 'error_token': '1234567890+12345678'}).encode())
48 def _config_from_environ(self):
51 for key, value in os.environ.items()
52 if key.startswith('ARVADOS_API_')
55 def _discoveryServiceUrl(
58 path='/discovery/v1/apis/{api}/{apiVersion}/rest',
62 host = os.environ['ARVADOS_API_HOST']
63 return urlparse.urlunsplit((scheme, host, path, None, None))
65 def test_new_api_objects_with_cache(self):
66 clients = [arvados.api('v1', cache=True) for index in [0, 1]]
67 self.assertIsNot(*clients)
69 def test_empty_list(self):
70 answer = arvados.api('v1').humans().list(
71 filters=[['uuid', '=', None]]).execute()
72 self.assertEqual(answer['items_available'], len(answer['items']))
74 def test_nonempty_list(self):
75 answer = arvados.api('v1').collections().list().execute()
76 self.assertNotEqual(0, answer['items_available'])
77 self.assertNotEqual(0, len(answer['items']))
79 def test_timestamp_inequality_filter(self):
80 api = arvados.api('v1')
81 new_item = api.specimens().create(body={}).execute()
82 for operator, should_include in [
83 ['<', False], ['>', False],
84 ['<=', True], ['>=', True], ['=', True]]:
85 response = api.specimens().list(filters=[
86 ['created_at', operator, new_item['created_at']],
87 # Also filter by uuid to ensure (if it matches) it's on page 0
88 ['uuid', '=', new_item['uuid']]]).execute()
89 uuids = [item['uuid'] for item in response['items']]
90 did_include = new_item['uuid'] in uuids
92 did_include, should_include,
93 "'%s %s' filter should%s have matched '%s'" % (
94 operator, new_item['created_at'],
95 ('' if should_include else ' not'),
96 new_item['created_at']))
98 def test_exceptions_include_errors(self):
100 'arvados.humans.get': self.api_error_response(
101 422, "Bad UUID format", "Bad output format"),
103 req_builder = apiclient_http.RequestMockBuilder(mock_responses)
104 api = arvados.api('v1', requestBuilder=req_builder)
105 with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
106 api.humans().get(uuid='xyz-xyz-abcdef').execute()
107 err_s = str(err_ctx.exception)
108 for msg in ["Bad UUID format", "Bad output format"]:
109 self.assertIn(msg, err_s)
111 @mock.patch('time.sleep')
112 def test_exceptions_include_request_id(self, sleep):
113 api = arvados.api('v1')
114 api.request_id='fake-request-id'
115 api._http.orig_http_request = mock.MagicMock()
116 api._http.orig_http_request.side_effect = socket.error('mock error')
119 api.users().current().execute()
120 except Exception as e:
122 self.assertRegex(str(caught), r'fake-request-id')
124 def test_exceptions_without_errors_have_basic_info(self):
126 'arvados.humans.delete': (
127 fake_httplib2_response(500, **self.ERROR_HEADERS),
130 req_builder = apiclient_http.RequestMockBuilder(mock_responses)
131 api = arvados.api('v1', requestBuilder=req_builder)
132 with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
133 api.humans().delete(uuid='xyz-xyz-abcdef').execute()
134 self.assertIn("500", str(err_ctx.exception))
136 def test_request_too_large(self):
137 api = arvados.api('v1')
138 maxsize = api._rootDesc.get('maxRequestSize', 0)
139 with self.assertRaises(apiclient_errors.MediaUploadSizeError):
141 arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
143 def test_default_request_timeout(self):
144 api = arvados.api('v1')
145 self.assertEqual(api._http.timeout, 300,
146 "Default timeout value should be 300")
148 def test_custom_request_timeout(self):
149 api = arvados.api('v1', timeout=1234)
150 self.assertEqual(api._http.timeout, 1234,
151 "Requested timeout value was 1234")
153 def test_4xx_retried(self):
154 client = arvados.api('v1')
155 for code in self.RETRIED_4XX:
156 name = f'retried #{code}'
157 with self.subTest(name), mock.patch('time.sleep'):
158 expected = {'username': name}
159 with mock_api_responses(
161 json.dumps(expected),
166 actual = client.users().current().execute()
167 self.assertEqual(actual, expected)
169 def test_4xx_not_retried(self):
170 client = arvados.api('v1', num_retries=3)
171 for code in [400, 401, 404]:
172 with self.subTest(f'error {code}'), mock.patch('time.sleep'):
173 with mock_api_responses(
179 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
180 client.users().current().execute()
181 response = exc_check.exception.args[0]
182 self.assertEqual(response.status, code)
183 self.assertEqual(response.get('status'), str(code))
185 def test_4xx_raised_after_retry_exhaustion(self):
186 client = arvados.api('v1', num_retries=1)
187 for code in self.RETRIED_4XX:
188 with self.subTest(f'failed {code}'), mock.patch('time.sleep'):
189 with mock_api_responses(
192 [code, code, code, 200],
195 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
196 client.users().current().execute()
197 response = exc_check.exception.args[0]
198 self.assertEqual(response.status, code)
199 self.assertEqual(response.get('status'), str(code))
201 def test_ordered_json_model(self):
203 'arvados.humans.get': (
205 json.dumps(collections.OrderedDict(
206 (c, int(c, 16)) for c in string.hexdigits
210 req_builder = apiclient_http.RequestMockBuilder(mock_responses)
211 api = arvados.api('v1',
212 requestBuilder=req_builder, model=OrderedJsonModel())
213 result = api.humans().get(uuid='test').execute()
214 self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
216 def test_api_is_threadsafe(self):
218 'host': os.environ['ARVADOS_API_HOST'],
219 'token': os.environ['ARVADOS_API_TOKEN'],
222 config_kwargs = {'apiconfig': os.environ}
223 for api_constructor, kwargs in [
225 (arvados.api, api_kwargs),
226 (arvados.api_from_config, {}),
227 (arvados.api_from_config, config_kwargs),
229 sub_kwargs = "kwargs" if kwargs else "no kwargs"
230 with self.subTest(f"{api_constructor.__name__} with {sub_kwargs}"):
231 api_client = api_constructor('v1', **kwargs)
232 self.assertTrue(hasattr(api_client, 'localapi'),
233 f"client missing localapi method")
234 self.assertTrue(hasattr(api_client, 'keep'),
235 f"client missing keep attribute")
237 def test_api_host_constructor(self):
240 client = arvados.api(
243 os.environ['ARVADOS_API_HOST'],
244 os.environ['ARVADOS_API_TOKEN'],
247 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
248 "client constructed with incorrect token")
250 def test_api_url_constructor(self):
251 client = arvados.api(
253 discoveryServiceUrl=self._discoveryServiceUrl(),
254 token=os.environ['ARVADOS_API_TOKEN'],
257 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
258 "client constructed with incorrect token")
260 def test_api_bad_args(self):
262 'host': os.environ['ARVADOS_API_HOST'],
263 'token': os.environ['ARVADOS_API_TOKEN'],
264 'discoveryServiceUrl': self._discoveryServiceUrl(),
267 # Passing only a single key is missing required info
268 *([key] for key in all_kwargs.keys()),
269 # Passing all keys is a conflict
270 list(all_kwargs.keys()),
272 kwargs = {key: all_kwargs[key] for key in use_keys}
273 kwargs_list = ', '.join(use_keys)
274 with self.subTest(f"calling arvados.api with {kwargs_list} fails"), \
275 self.assertRaises(ValueError):
276 arvados.api('v1', insecure=True, **kwargs)
278 def test_api_bad_url(self):
280 {'discoveryServiceUrl': self._discoveryServiceUrl() + '/BadTestURL'},
281 {'version': 'BadTestVersion', 'host': os.environ['ARVADOS_API_HOST']},
283 bad_key = next(iter(bad_kwargs))
284 with self.subTest(f"api fails with bad {bad_key}"), \
285 self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
286 arvados.api(**bad_kwargs, token='test_api_bad_url', insecure=True)
288 def test_normalize_api_good_args(self):
289 for version, discoveryServiceUrl, host in [
290 ('Test1', None, os.environ['ARVADOS_API_HOST']),
291 (None, self._discoveryServiceUrl(), None)
293 argname = 'discoveryServiceUrl' if host is None else 'host'
294 with self.subTest(f"normalize_api_kwargs with {argname}"):
295 actual = normalize_api_kwargs(
299 os.environ['ARVADOS_API_TOKEN'],
302 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
303 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
304 self.assertEqual(actual['version'], version or 'v1')
305 self.assertTrue(actual['insecure'])
306 self.assertNotIn('host', actual)
308 def test_normalize_api_bad_args(self):
310 self._discoveryServiceUrl(),
311 os.environ['ARVADOS_API_HOST'],
312 os.environ['ARVADOS_API_TOKEN'],
314 for arg_index, arg_value in enumerate(all_args):
315 args = [None] * len(all_args)
316 args[arg_index] = arg_value
317 with self.subTest(f"normalize_api_kwargs with only arg #{arg_index + 1}"), \
318 self.assertRaises(ValueError):
319 normalize_api_kwargs('v1', *args)
320 with self.subTest("normalize_api_kwargs with discoveryServiceUrl and host"), \
321 self.assertRaises(ValueError):
322 normalize_api_kwargs('v1', *all_args)
324 def test_api_from_config_default(self):
325 client = arvados.api_from_config('v1')
326 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
327 "client constructed with incorrect token")
329 def test_api_from_config_explicit(self):
330 config = self._config_from_environ()
331 client = arvados.api_from_config('v1', config)
332 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
333 "client constructed with incorrect token")
335 def test_api_from_bad_config(self):
336 base_config = self._config_from_environ()
337 for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
338 with self.subTest(f"api_from_config without {del_key} fails"), \
339 self.assertRaises(ValueError):
340 config = dict(base_config)
342 arvados.api_from_config('v1', config)
344 def test_api_kwargs_from_good_config(self):
345 for config in [None, self._config_from_environ()]:
346 conf_type = 'default' if config is None else 'passed'
347 with self.subTest(f"api_kwargs_from_config with {conf_type} config"):
348 version = 'Test1' if config else None
349 actual = api_kwargs_from_config(version, config)
350 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
351 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
352 self.assertEqual(actual['version'], version or 'v1')
353 self.assertTrue(actual['insecure'])
354 self.assertNotIn('host', actual)
356 def test_api_kwargs_from_bad_config(self):
357 base_config = self._config_from_environ()
358 for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
359 with self.subTest(f"api_kwargs_from_config without {del_key} fails"), \
360 self.assertRaises(ValueError):
361 config = dict(base_config)
363 api_kwargs_from_config('v1', config)
365 def test_api_client_constructor(self):
368 self._discoveryServiceUrl(),
369 os.environ['ARVADOS_API_TOKEN'],
372 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
373 "client constructed with incorrect token")
375 hasattr(client, 'localapi'),
376 "client has localapi method when it should not be thread-safe",
379 def test_api_client_bad_url(self):
380 all_args = ('v1', self._discoveryServiceUrl(), 'test_api_client_bad_url')
381 for arg_index, arg_value in [
382 (0, 'BadTestVersion'),
383 (1, all_args[1] + '/BadTestURL'),
385 with self.subTest(f"api_client fails with {arg_index}={arg_value!r}"), \
386 self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
387 args = list(all_args)
388 args[arg_index] = arg_value
389 api_client(*args, insecure=True)
392 class ConstructNumRetriesTestCase(unittest.TestCase):
394 def _fake_retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs):
395 return http.request(uri, method, *args, **kwargs)
397 @contextlib.contextmanager
398 def patch_retry(self):
399 # We have this dedicated context manager that goes through `sys.modules`
400 # instead of just using `mock.patch` because of the unfortunate
401 # `arvados.api` name collision.
402 orig_func = sys.modules['arvados.api']._orig_retry_request
403 expect_name = 'googleapiclient.http._retry_request'
405 '{0.__module__}.{0.__name__}'.format(orig_func), expect_name,
406 f"test setup problem: {expect_name} not at arvados.api._orig_retry_request",
408 retry_mock = mock.Mock(wraps=self._fake_retry_request)
409 sys.modules['arvados.api']._orig_retry_request = retry_mock
413 sys.modules['arvados.api']._orig_retry_request = orig_func
415 def _iter_num_retries(self, retry_mock):
416 for call in retry_mock.call_args_list:
420 yield call.kwargs['num_retries']
422 def test_default_num_retries(self):
423 with self.patch_retry() as retry_mock:
424 client = arvados.api('v1')
425 actual = set(self._iter_num_retries(retry_mock))
426 self.assertEqual(len(actual), 1)
427 self.assertTrue(actual.pop() > 6, "num_retries lower than expected")
429 def _test_calls(self, init_arg, call_args, expected):
430 with self.patch_retry() as retry_mock:
431 client = arvados.api('v1', num_retries=init_arg)
432 for num_retries in call_args:
433 client.users().current().execute(num_retries=num_retries)
434 actual = self._iter_num_retries(retry_mock)
435 # The constructor makes two requests with its num_retries argument:
436 # one for the discovery document, and one for the config.
437 self.assertEqual(next(actual, None), init_arg)
438 self.assertEqual(next(actual, None), init_arg)
439 self.assertEqual(list(actual), expected)
441 def test_discovery_num_retries(self):
442 for num_retries in [0, 5, 55]:
443 with self.subTest(f"num_retries={num_retries}"):
444 self._test_calls(num_retries, [], [])
446 def test_num_retries_called_le_init(self):
448 with self.subTest(f"init_arg={n}"):
449 call_args = [n - 4, n - 2, n]
451 self._test_calls(n, call_args, expected)
453 def test_num_retries_called_ge_init(self):
455 with self.subTest(f"init_arg={n}"):
456 call_args = [n, n + 4, n + 8]
457 self._test_calls(n, call_args, call_args)
459 def test_num_retries_called_mixed(self):
460 self._test_calls(5, [2, 6, 4, 8], [5, 6, 5, 8])
463 class PreCloseSocketTestCase(unittest.TestCase):
465 self.api = arvados.api('v1')
466 self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
467 "test doesn't know how to intercept HTTP requests")
468 self.mock_response = {'user': 'person'}
469 self.request_success = (fake_httplib2_response(200),
470 json.dumps(self.mock_response))
471 self.api._http.orig_http_request = mock.MagicMock()
472 # All requests succeed by default. Tests override as needed.
473 self.api._http.orig_http_request.return_value = self.request_success
475 @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
476 def test_close_old_connections_non_retryable(self, sleep):
477 self._test_connection_close(expect=1)
479 @mock.patch('time.time', side_effect=itertools.count())
480 def test_no_close_fresh_connections_non_retryable(self, sleep):
481 self._test_connection_close(expect=0)
483 @mock.patch('time.time', side_effect=itertools.count())
484 def test_override_max_idle_time(self, sleep):
485 self.api._http._max_keepalive_idle = 0
486 self._test_connection_close(expect=1)
488 def _test_connection_close(self, expect=0):
489 # Do two POST requests. The second one must close all
490 # connections +expect+ times.
491 self.api.users().create(body={}).execute()
492 mock_conns = {str(i): mock.MagicMock() for i in range(2)}
493 self.api._http.connections = mock_conns.copy()
494 self.api.users().create(body={}).execute()
495 for c in mock_conns.values():
496 self.assertEqual(c.close.call_count, expect)
499 if __name__ == '__main__':