1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import absolute_import
6 from builtins import str
7 from builtins import range
20 from . import run_test_server
22 from apiclient import errors as apiclient_errors
23 from apiclient import http as apiclient_http
24 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
25 from .arvados_testutil import fake_httplib2_response, queue_with
27 if not mimetypes.inited:
30 class ArvadosApiTest(run_test_server.TestCaseWithServers):
32 ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
34 def api_error_response(self, code, *errors):
35 return (fake_httplib2_response(code, **self.ERROR_HEADERS),
36 json.dumps({'errors': errors,
37 'error_token': '1234567890+12345678'}).encode())
39 def test_new_api_objects_with_cache(self):
40 clients = [arvados.api('v1', cache=True) for index in [0, 1]]
41 self.assertIsNot(*clients)
43 def test_empty_list(self):
44 answer = arvados.api('v1').humans().list(
45 filters=[['uuid', '=', None]]).execute()
46 self.assertEqual(answer['items_available'], len(answer['items']))
48 def test_nonempty_list(self):
49 answer = arvados.api('v1').collections().list().execute()
50 self.assertNotEqual(0, answer['items_available'])
51 self.assertNotEqual(0, len(answer['items']))
53 def test_timestamp_inequality_filter(self):
54 api = arvados.api('v1')
55 new_item = api.specimens().create(body={}).execute()
56 for operator, should_include in [
57 ['<', False], ['>', False],
58 ['<=', True], ['>=', True], ['=', True]]:
59 response = api.specimens().list(filters=[
60 ['created_at', operator, new_item['created_at']],
61 # Also filter by uuid to ensure (if it matches) it's on page 0
62 ['uuid', '=', new_item['uuid']]]).execute()
63 uuids = [item['uuid'] for item in response['items']]
64 did_include = new_item['uuid'] in uuids
66 did_include, should_include,
67 "'%s %s' filter should%s have matched '%s'" % (
68 operator, new_item['created_at'],
69 ('' if should_include else ' not'),
70 new_item['created_at']))
72 def test_exceptions_include_errors(self):
74 'arvados.humans.get': self.api_error_response(
75 422, "Bad UUID format", "Bad output format"),
77 req_builder = apiclient_http.RequestMockBuilder(mock_responses)
78 api = arvados.api('v1', requestBuilder=req_builder)
79 with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
80 api.humans().get(uuid='xyz-xyz-abcdef').execute()
81 err_s = str(err_ctx.exception)
82 for msg in ["Bad UUID format", "Bad output format"]:
83 self.assertIn(msg, err_s)
85 def test_exceptions_without_errors_have_basic_info(self):
87 'arvados.humans.delete': (
88 fake_httplib2_response(500, **self.ERROR_HEADERS),
91 req_builder = apiclient_http.RequestMockBuilder(mock_responses)
92 api = arvados.api('v1', requestBuilder=req_builder)
93 with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
94 api.humans().delete(uuid='xyz-xyz-abcdef').execute()
95 self.assertIn("500", str(err_ctx.exception))
97 def test_request_too_large(self):
98 api = arvados.api('v1')
99 maxsize = api._rootDesc.get('maxRequestSize', 0)
100 with self.assertRaises(apiclient_errors.MediaUploadSizeError):
102 arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
104 def test_default_request_timeout(self):
105 api = arvados.api('v1')
106 self.assertEqual(api._http.timeout, 300,
107 "Default timeout value should be 300")
109 def test_custom_request_timeout(self):
110 api = arvados.api('v1', timeout=1234)
111 self.assertEqual(api._http.timeout, 1234,
112 "Requested timeout value was 1234")
114 def test_ordered_json_model(self):
116 'arvados.humans.get': (
118 json.dumps(collections.OrderedDict(
119 (c, int(c, 16)) for c in string.hexdigits
123 req_builder = apiclient_http.RequestMockBuilder(mock_responses)
124 api = arvados.api('v1',
125 requestBuilder=req_builder, model=OrderedJsonModel())
126 result = api.humans().get(uuid='test').execute()
127 self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
130 class RetryREST(unittest.TestCase):
132 self.api = arvados.api('v1')
133 self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
134 "test doesn't know how to intercept HTTP requests")
135 self.mock_response = {'user': 'person'}
136 self.request_success = (fake_httplib2_response(200),
137 json.dumps(self.mock_response))
138 self.api._http.orig_http_request = mock.MagicMock()
139 # All requests succeed by default. Tests override as needed.
140 self.api._http.orig_http_request.return_value = self.request_success
142 @mock.patch('time.sleep')
143 def test_socket_error_retry_get(self, sleep):
144 self.api._http.orig_http_request.side_effect = (
145 socket.error('mock error'),
146 self.request_success,
148 self.assertEqual(self.api.users().current().execute(),
150 self.assertGreater(self.api._http.orig_http_request.call_count, 1,
151 "client got the right response without retrying")
152 self.assertEqual(sleep.call_args_list,
153 [mock.call(RETRY_DELAY_INITIAL)])
155 @mock.patch('time.sleep')
156 def test_same_automatic_request_id_on_retry(self, sleep):
157 self.api._http.orig_http_request.side_effect = (
158 socket.error('mock error'),
159 self.request_success,
161 self.api.users().current().execute()
162 calls = self.api._http.orig_http_request.call_args_list
163 self.assertEqual(len(calls), 2)
165 calls[0][1]['headers']['X-Request-Id'],
166 calls[1][1]['headers']['X-Request-Id'])
167 self.assertRegex(calls[0][1]['headers']['X-Request-Id'], r'^req-[a-z0-9]{20}$')
169 @mock.patch('time.sleep')
170 def test_provided_request_id_on_retry(self, sleep):
171 self.api.request_id='fake-request-id'
172 self.api._http.orig_http_request.side_effect = (
173 socket.error('mock error'),
174 self.request_success,
176 self.api.users().current().execute()
177 calls = self.api._http.orig_http_request.call_args_list
178 self.assertEqual(len(calls), 2)
180 self.assertEqual(call[1]['headers']['X-Request-Id'], 'fake-request-id')
182 @mock.patch('time.sleep')
183 def test_socket_error_retry_delay(self, sleep):
184 self.api._http.orig_http_request.side_effect = socket.error('mock')
185 self.api._http._retry_count = 3
186 with self.assertRaises(socket.error):
187 self.api.users().current().execute()
188 self.assertEqual(self.api._http.orig_http_request.call_count, 4)
189 self.assertEqual(sleep.call_args_list, [
190 mock.call(RETRY_DELAY_INITIAL),
191 mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
192 mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
195 @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
196 def test_close_old_connections_non_retryable(self, sleep):
197 self._test_connection_close(expect=1)
199 @mock.patch('time.time', side_effect=itertools.count())
200 def test_no_close_fresh_connections_non_retryable(self, sleep):
201 self._test_connection_close(expect=0)
203 @mock.patch('time.time', side_effect=itertools.count())
204 def test_override_max_idle_time(self, sleep):
205 self.api._http._max_keepalive_idle = 0
206 self._test_connection_close(expect=1)
208 def _test_connection_close(self, expect=0):
209 # Do two POST requests. The second one must close all
210 # connections +expect+ times.
211 self.api.users().create(body={}).execute()
212 mock_conns = {str(i): mock.MagicMock() for i in range(2)}
213 self.api._http.connections = mock_conns.copy()
214 self.api.users().create(body={}).execute()
215 for c in mock_conns.values():
216 self.assertEqual(c.close.call_count, expect)
218 @mock.patch('time.sleep')
219 def test_socket_error_no_retry_post(self, sleep):
220 self.api._http.orig_http_request.side_effect = (
221 socket.error('mock error'),
222 self.request_success,
224 with self.assertRaises(socket.error):
225 self.api.users().create(body={}).execute()
226 self.assertEqual(self.api._http.orig_http_request.call_count, 1,
227 "client should try non-retryable method exactly once")
228 self.assertEqual(sleep.call_args_list, [])
231 if __name__ == '__main__':