3 from __future__ import absolute_import
16 from . import run_test_server
18 from apiclient import errors as apiclient_errors
19 from apiclient import http as apiclient_http
20 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
21 from .arvados_testutil import fake_httplib2_response, queue_with
23 if not mimetypes.inited:
26 class ArvadosApiTest(run_test_server.TestCaseWithServers):
28 ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
30 def api_error_response(self, code, *errors):
31 return (fake_httplib2_response(code, **self.ERROR_HEADERS),
32 json.dumps({'errors': errors,
33 'error_token': '1234567890+12345678'}))
35 def test_new_api_objects_with_cache(self):
36 clients = [arvados.api('v1', cache=True) for index in [0, 1]]
37 self.assertIsNot(*clients)
39 def test_empty_list(self):
40 answer = arvados.api('v1').humans().list(
41 filters=[['uuid', '=', None]]).execute()
42 self.assertEqual(answer['items_available'], len(answer['items']))
44 def test_nonempty_list(self):
45 answer = arvados.api('v1').collections().list().execute()
46 self.assertNotEqual(0, answer['items_available'])
47 self.assertNotEqual(0, len(answer['items']))
49 def test_timestamp_inequality_filter(self):
50 api = arvados.api('v1')
51 new_item = api.specimens().create(body={}).execute()
52 for operator, should_include in [
53 ['<', False], ['>', False],
54 ['<=', True], ['>=', True], ['=', True]]:
55 response = api.specimens().list(filters=[
56 ['created_at', operator, new_item['created_at']],
57 # Also filter by uuid to ensure (if it matches) it's on page 0
58 ['uuid', '=', new_item['uuid']]]).execute()
59 uuids = [item['uuid'] for item in response['items']]
60 did_include = new_item['uuid'] in uuids
62 did_include, should_include,
63 "'%s %s' filter should%s have matched '%s'" % (
64 operator, new_item['created_at'],
65 ('' if should_include else ' not'),
66 new_item['created_at']))
68 def test_exceptions_include_errors(self):
70 'arvados.humans.get': self.api_error_response(
71 422, "Bad UUID format", "Bad output format"),
73 req_builder = apiclient_http.RequestMockBuilder(mock_responses)
74 api = arvados.api('v1', requestBuilder=req_builder)
75 with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
76 api.humans().get(uuid='xyz-xyz-abcdef').execute()
77 err_s = str(err_ctx.exception)
78 for msg in ["Bad UUID format", "Bad output format"]:
79 self.assertIn(msg, err_s)
81 def test_exceptions_without_errors_have_basic_info(self):
83 'arvados.humans.delete': (
84 fake_httplib2_response(500, **self.ERROR_HEADERS),
87 req_builder = apiclient_http.RequestMockBuilder(mock_responses)
88 api = arvados.api('v1', requestBuilder=req_builder)
89 with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
90 api.humans().delete(uuid='xyz-xyz-abcdef').execute()
91 self.assertIn("500", str(err_ctx.exception))
93 def test_request_too_large(self):
94 api = arvados.api('v1')
95 maxsize = api._rootDesc.get('maxRequestSize', 0)
96 with self.assertRaises(apiclient_errors.MediaUploadSizeError):
98 arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
100 def test_ordered_json_model(self):
102 'arvados.humans.get': (None, json.dumps(collections.OrderedDict(
103 (c, int(c, 16)) for c in string.hexdigits))),
105 req_builder = apiclient_http.RequestMockBuilder(mock_responses)
106 api = arvados.api('v1',
107 requestBuilder=req_builder, model=OrderedJsonModel())
108 result = api.humans().get(uuid='test').execute()
109 self.assertEqual(string.hexdigits, ''.join(result.keys()))
112 class RetryREST(unittest.TestCase):
114 self.api = arvados.api('v1')
115 self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
116 "test doesn't know how to intercept HTTP requests")
117 self.mock_response = {'user': 'person'}
118 self.request_success = (fake_httplib2_response(200),
119 json.dumps(self.mock_response))
120 self.api._http.orig_http_request = mock.MagicMock()
121 # All requests succeed by default. Tests override as needed.
122 self.api._http.orig_http_request.return_value = self.request_success
124 @mock.patch('time.sleep')
125 def test_socket_error_retry_get(self, sleep):
126 self.api._http.orig_http_request.side_effect = (
127 socket.error('mock error'),
128 self.request_success,
130 self.assertEqual(self.api.users().current().execute(),
132 self.assertGreater(self.api._http.orig_http_request.call_count, 1,
133 "client got the right response without retrying")
134 self.assertEqual(sleep.call_args_list,
135 [mock.call(RETRY_DELAY_INITIAL)])
137 @mock.patch('time.sleep')
138 def test_socket_error_retry_delay(self, sleep):
139 self.api._http.orig_http_request.side_effect = socket.error('mock')
140 self.api._http._retry_count = 3
141 with self.assertRaises(socket.error):
142 self.api.users().current().execute()
143 self.assertEqual(self.api._http.orig_http_request.call_count, 4)
144 self.assertEqual(sleep.call_args_list, [
145 mock.call(RETRY_DELAY_INITIAL),
146 mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
147 mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
150 @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
151 def test_close_old_connections_non_retryable(self, sleep):
152 self._test_connection_close(expect=1)
154 @mock.patch('time.time', side_effect=itertools.count())
155 def test_no_close_fresh_connections_non_retryable(self, sleep):
156 self._test_connection_close(expect=0)
158 @mock.patch('time.time', side_effect=itertools.count())
159 def test_override_max_idle_time(self, sleep):
160 self.api._http._max_keepalive_idle = 0
161 self._test_connection_close(expect=1)
163 def _test_connection_close(self, expect=0):
164 # Do two POST requests. The second one must close all
165 # connections +expect+ times.
166 self.api.users().create(body={}).execute()
167 mock_conns = {str(i): mock.MagicMock() for i in range(2)}
168 self.api._http.connections = mock_conns.copy()
169 self.api.users().create(body={}).execute()
170 for c in mock_conns.itervalues():
171 self.assertEqual(c.close.call_count, expect)
173 @mock.patch('time.sleep')
174 def test_socket_error_no_retry_post(self, sleep):
175 self.api._http.orig_http_request.side_effect = (
176 socket.error('mock error'),
177 self.request_success,
179 with self.assertRaises(socket.error):
180 self.api.users().create(body={}).execute()
181 self.assertEqual(self.api._http.orig_http_request.call_count, 1,
182 "client should try non-retryable method exactly once")
183 self.assertEqual(sleep.call_args_list, [])
186 if __name__ == '__main__':