11308: Remove superfluous hashbangs
[arvados.git] / sdk / python / tests / test_api.py
1 from __future__ import absolute_import
2 from builtins import str
3 from builtins import range
4 import arvados
5 import collections
6 import httplib2
7 import itertools
8 import json
9 import mimetypes
10 import os
11 import socket
12 import string
13 import unittest
14
15 import mock
16 from . import run_test_server
17
18 from apiclient import errors as apiclient_errors
19 from apiclient import http as apiclient_http
20 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
21 from .arvados_testutil import fake_httplib2_response, queue_with
22
23 if not mimetypes.inited:
24     mimetypes.init()
25
26 class ArvadosApiTest(run_test_server.TestCaseWithServers):
27     MAIN_SERVER = {}
28     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
29
30     def api_error_response(self, code, *errors):
31         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
32                 json.dumps({'errors': errors,
33                             'error_token': '1234567890+12345678'}).encode())
34
35     def test_new_api_objects_with_cache(self):
36         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
37         self.assertIsNot(*clients)
38
39     def test_empty_list(self):
40         answer = arvados.api('v1').humans().list(
41             filters=[['uuid', '=', None]]).execute()
42         self.assertEqual(answer['items_available'], len(answer['items']))
43
44     def test_nonempty_list(self):
45         answer = arvados.api('v1').collections().list().execute()
46         self.assertNotEqual(0, answer['items_available'])
47         self.assertNotEqual(0, len(answer['items']))
48
49     def test_timestamp_inequality_filter(self):
50         api = arvados.api('v1')
51         new_item = api.specimens().create(body={}).execute()
52         for operator, should_include in [
53                 ['<', False], ['>', False],
54                 ['<=', True], ['>=', True], ['=', True]]:
55             response = api.specimens().list(filters=[
56                 ['created_at', operator, new_item['created_at']],
57                 # Also filter by uuid to ensure (if it matches) it's on page 0
58                 ['uuid', '=', new_item['uuid']]]).execute()
59             uuids = [item['uuid'] for item in response['items']]
60             did_include = new_item['uuid'] in uuids
61             self.assertEqual(
62                 did_include, should_include,
63                 "'%s %s' filter should%s have matched '%s'" % (
64                     operator, new_item['created_at'],
65                     ('' if should_include else ' not'),
66                     new_item['created_at']))
67
68     def test_exceptions_include_errors(self):
69         mock_responses = {
70             'arvados.humans.get': self.api_error_response(
71                 422, "Bad UUID format", "Bad output format"),
72             }
73         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
74         api = arvados.api('v1', requestBuilder=req_builder)
75         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
76             api.humans().get(uuid='xyz-xyz-abcdef').execute()
77         err_s = str(err_ctx.exception)
78         for msg in ["Bad UUID format", "Bad output format"]:
79             self.assertIn(msg, err_s)
80
81     def test_exceptions_without_errors_have_basic_info(self):
82         mock_responses = {
83             'arvados.humans.delete': (
84                 fake_httplib2_response(500, **self.ERROR_HEADERS),
85                 b"")
86             }
87         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
88         api = arvados.api('v1', requestBuilder=req_builder)
89         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
90             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
91         self.assertIn("500", str(err_ctx.exception))
92
93     def test_request_too_large(self):
94         api = arvados.api('v1')
95         maxsize = api._rootDesc.get('maxRequestSize', 0)
96         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
97             text = "X" * maxsize
98             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
99
100     def test_ordered_json_model(self):
101         mock_responses = {
102             'arvados.humans.get': (
103                 None,
104                 json.dumps(collections.OrderedDict(
105                     (c, int(c, 16)) for c in string.hexdigits
106                 )).encode(),
107             ),
108         }
109         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
110         api = arvados.api('v1',
111                           requestBuilder=req_builder, model=OrderedJsonModel())
112         result = api.humans().get(uuid='test').execute()
113         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
114
115
116 class RetryREST(unittest.TestCase):
117     def setUp(self):
118         self.api = arvados.api('v1')
119         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
120                         "test doesn't know how to intercept HTTP requests")
121         self.mock_response = {'user': 'person'}
122         self.request_success = (fake_httplib2_response(200),
123                                 json.dumps(self.mock_response))
124         self.api._http.orig_http_request = mock.MagicMock()
125         # All requests succeed by default. Tests override as needed.
126         self.api._http.orig_http_request.return_value = self.request_success
127
128     @mock.patch('time.sleep')
129     def test_socket_error_retry_get(self, sleep):
130         self.api._http.orig_http_request.side_effect = (
131             socket.error('mock error'),
132             self.request_success,
133         )
134         self.assertEqual(self.api.users().current().execute(),
135                          self.mock_response)
136         self.assertGreater(self.api._http.orig_http_request.call_count, 1,
137                            "client got the right response without retrying")
138         self.assertEqual(sleep.call_args_list,
139                          [mock.call(RETRY_DELAY_INITIAL)])
140
141     @mock.patch('time.sleep')
142     def test_socket_error_retry_delay(self, sleep):
143         self.api._http.orig_http_request.side_effect = socket.error('mock')
144         self.api._http._retry_count = 3
145         with self.assertRaises(socket.error):
146             self.api.users().current().execute()
147         self.assertEqual(self.api._http.orig_http_request.call_count, 4)
148         self.assertEqual(sleep.call_args_list, [
149             mock.call(RETRY_DELAY_INITIAL),
150             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
151             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
152         ])
153
154     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
155     def test_close_old_connections_non_retryable(self, sleep):
156         self._test_connection_close(expect=1)
157
158     @mock.patch('time.time', side_effect=itertools.count())
159     def test_no_close_fresh_connections_non_retryable(self, sleep):
160         self._test_connection_close(expect=0)
161
162     @mock.patch('time.time', side_effect=itertools.count())
163     def test_override_max_idle_time(self, sleep):
164         self.api._http._max_keepalive_idle = 0
165         self._test_connection_close(expect=1)
166
167     def _test_connection_close(self, expect=0):
168         # Do two POST requests. The second one must close all
169         # connections +expect+ times.
170         self.api.users().create(body={}).execute()
171         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
172         self.api._http.connections = mock_conns.copy()
173         self.api.users().create(body={}).execute()
174         for c in mock_conns.values():
175             self.assertEqual(c.close.call_count, expect)
176
177     @mock.patch('time.sleep')
178     def test_socket_error_no_retry_post(self, sleep):
179         self.api._http.orig_http_request.side_effect = (
180             socket.error('mock error'),
181             self.request_success,
182         )
183         with self.assertRaises(socket.error):
184             self.api.users().create(body={}).execute()
185         self.assertEqual(self.api._http.orig_http_request.call_count, 1,
186                          "client should try non-retryable method exactly once")
187         self.assertEqual(sleep.call_args_list, [])
188
189
190 if __name__ == '__main__':
191     unittest.main()