sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import str
   7 from builtins import range
   8 import arvados
   9 import collections
  10 import httplib2
  11 import itertools
  12 import json
  13 import mimetypes
  14 import os
  15 import socket
  16 import string
  17 import unittest
  18
  19 import mock
  20 from . import run_test_server
  21
  22 from apiclient import errors as apiclient_errors
  23 from apiclient import http as apiclient_http
  24 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
  25 from .arvados_testutil import fake_httplib2_response, queue_with
  26
  27 if not mimetypes.inited:
  28     mimetypes.init()
  29
  30 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  31     MAIN_SERVER = {}
  32     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  33
  34     def api_error_response(self, code, *errors):
  35         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  36                 json.dumps({'errors': errors,
  37                             'error_token': '1234567890+12345678'}).encode())
  38
  39     def test_new_api_objects_with_cache(self):
  40         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  41         self.assertIsNot(*clients)
  42
  43     def test_empty_list(self):
  44         answer = arvados.api('v1').humans().list(
  45             filters=[['uuid', '=', None]]).execute()
  46         self.assertEqual(answer['items_available'], len(answer['items']))
  47
  48     def test_nonempty_list(self):
  49         answer = arvados.api('v1').collections().list().execute()
  50         self.assertNotEqual(0, answer['items_available'])
  51         self.assertNotEqual(0, len(answer['items']))
  52
  53     def test_timestamp_inequality_filter(self):
  54         api = arvados.api('v1')
  55         new_item = api.specimens().create(body={}).execute()
  56         for operator, should_include in [
  57                 ['<', False], ['>', False],
  58                 ['<=', True], ['>=', True], ['=', True]]:
  59             response = api.specimens().list(filters=[
  60                 ['created_at', operator, new_item['created_at']],
  61                 # Also filter by uuid to ensure (if it matches) it's on page 0
  62                 ['uuid', '=', new_item['uuid']]]).execute()
  63             uuids = [item['uuid'] for item in response['items']]
  64             did_include = new_item['uuid'] in uuids
  65             self.assertEqual(
  66                 did_include, should_include,
  67                 "'%s %s' filter should%s have matched '%s'" % (
  68                     operator, new_item['created_at'],
  69                     ('' if should_include else ' not'),
  70                     new_item['created_at']))
  71
  72     def test_exceptions_include_errors(self):
  73         mock_responses = {
  74             'arvados.humans.get': self.api_error_response(
  75                 422, "Bad UUID format", "Bad output format"),
  76             }
  77         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  78         api = arvados.api('v1', requestBuilder=req_builder)
  79         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  80             api.humans().get(uuid='xyz-xyz-abcdef').execute()
  81         err_s = str(err_ctx.exception)
  82         for msg in ["Bad UUID format", "Bad output format"]:
  83             self.assertIn(msg, err_s)
  84
  85     @mock.patch('time.sleep')
  86     def test_exceptions_include_request_id(self, sleep):
  87         api = arvados.api('v1')
  88         api.request_id='fake-request-id'
  89         api._http.orig_http_request = mock.MagicMock()
  90         api._http.orig_http_request.side_effect = socket.error('mock error')
  91         caught = None
  92         try:
  93             api.users().current().execute()
  94         except Exception as e:
  95             caught = e
  96         self.assertRegex(str(caught), r'fake-request-id')
  97
  98     def test_exceptions_without_errors_have_basic_info(self):
  99         mock_responses = {
 100             'arvados.humans.delete': (
 101                 fake_httplib2_response(500, **self.ERROR_HEADERS),
 102                 b"")
 103             }
 104         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 105         api = arvados.api('v1', requestBuilder=req_builder)
 106         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 107             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
 108         self.assertIn("500", str(err_ctx.exception))
 109
 110     def test_request_too_large(self):
 111         api = arvados.api('v1')
 112         maxsize = api._rootDesc.get('maxRequestSize', 0)
 113         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 114             text = "X" * maxsize
 115             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 116
 117     def test_default_request_timeout(self):
 118         api = arvados.api('v1')
 119         self.assertEqual(api._http.timeout, 300,
 120             "Default timeout value should be 300")
 121
 122     def test_custom_request_timeout(self):
 123         api = arvados.api('v1', timeout=1234)
 124         self.assertEqual(api._http.timeout, 1234,
 125             "Requested timeout value was 1234")
 126
 127     def test_ordered_json_model(self):
 128         mock_responses = {
 129             'arvados.humans.get': (
 130                 None,
 131                 json.dumps(collections.OrderedDict(
 132                     (c, int(c, 16)) for c in string.hexdigits
 133                 )).encode(),
 134             ),
 135         }
 136         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 137         api = arvados.api('v1',
 138                           requestBuilder=req_builder, model=OrderedJsonModel())
 139         result = api.humans().get(uuid='test').execute()
 140         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 141
 142     def test_api_is_threadsafe(self):
 143         api_kwargs = {
 144             'host': os.environ['ARVADOS_API_HOST'],
 145             'token': os.environ['ARVADOS_API_TOKEN'],
 146             'insecure': True,
 147         }
 148         config_kwargs = {'apiconfig': os.environ}
 149         for api_constructor, kwargs in [
 150                 (arvados.api, {}),
 151                 (arvados.api, api_kwargs),
 152                 (arvados.api_from_config, {}),
 153                 (arvados.api_from_config, config_kwargs),
 154         ]:
 155             sub_kwargs = "kwargs" if kwargs else "no kwargs"
 156             with self.subTest(f"{api_constructor.__name__} with {sub_kwargs}"):
 157                 api_client = api_constructor('v1', **kwargs)
 158                 self.assertTrue(hasattr(api_client, 'localapi'),
 159                                 f"client missing localapi method")
 160                 self.assertTrue(hasattr(api_client, 'keep'),
 161                                 f"client missing keep attribute")
 162
 163
 164 class RetryREST(unittest.TestCase):
 165     def setUp(self):
 166         self.api = arvados.api('v1')
 167         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 168                         "test doesn't know how to intercept HTTP requests")
 169         self.mock_response = {'user': 'person'}
 170         self.request_success = (fake_httplib2_response(200),
 171                                 json.dumps(self.mock_response))
 172         self.api._http.orig_http_request = mock.MagicMock()
 173         # All requests succeed by default. Tests override as needed.
 174         self.api._http.orig_http_request.return_value = self.request_success
 175
 176     @mock.patch('time.sleep')
 177     def test_socket_error_retry_get(self, sleep):
 178         self.api._http.orig_http_request.side_effect = (
 179             socket.error('mock error'),
 180             self.request_success,
 181         )
 182         self.assertEqual(self.api.users().current().execute(),
 183                          self.mock_response)
 184         self.assertGreater(self.api._http.orig_http_request.call_count, 1,
 185                            "client got the right response without retrying")
 186         self.assertEqual(sleep.call_args_list,
 187                          [mock.call(RETRY_DELAY_INITIAL)])
 188
 189     @mock.patch('time.sleep')
 190     def test_same_automatic_request_id_on_retry(self, sleep):
 191         self.api._http.orig_http_request.side_effect = (
 192             socket.error('mock error'),
 193             self.request_success,
 194         )
 195         self.api.users().current().execute()
 196         calls = self.api._http.orig_http_request.call_args_list
 197         self.assertEqual(len(calls), 2)
 198         self.assertEqual(
 199             calls[0][1]['headers']['X-Request-Id'],
 200             calls[1][1]['headers']['X-Request-Id'])
 201         self.assertRegex(calls[0][1]['headers']['X-Request-Id'], r'^req-[a-z0-9]{20}$')
 202
 203     @mock.patch('time.sleep')
 204     def test_provided_request_id_on_retry(self, sleep):
 205         self.api.request_id='fake-request-id'
 206         self.api._http.orig_http_request.side_effect = (
 207             socket.error('mock error'),
 208             self.request_success,
 209         )
 210         self.api.users().current().execute()
 211         calls = self.api._http.orig_http_request.call_args_list
 212         self.assertEqual(len(calls), 2)
 213         for call in calls:
 214             self.assertEqual(call[1]['headers']['X-Request-Id'], 'fake-request-id')
 215
 216     @mock.patch('time.sleep')
 217     def test_socket_error_retry_delay(self, sleep):
 218         self.api._http.orig_http_request.side_effect = socket.error('mock')
 219         self.api._http._retry_count = 3
 220         with self.assertRaises(socket.error):
 221             self.api.users().current().execute()
 222         self.assertEqual(self.api._http.orig_http_request.call_count, 4)
 223         self.assertEqual(sleep.call_args_list, [
 224             mock.call(RETRY_DELAY_INITIAL),
 225             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
 226             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
 227         ])
 228
 229     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 230     def test_close_old_connections_non_retryable(self, sleep):
 231         self._test_connection_close(expect=1)
 232
 233     @mock.patch('time.time', side_effect=itertools.count())
 234     def test_no_close_fresh_connections_non_retryable(self, sleep):
 235         self._test_connection_close(expect=0)
 236
 237     @mock.patch('time.time', side_effect=itertools.count())
 238     def test_override_max_idle_time(self, sleep):
 239         self.api._http._max_keepalive_idle = 0
 240         self._test_connection_close(expect=1)
 241
 242     def _test_connection_close(self, expect=0):
 243         # Do two POST requests. The second one must close all
 244         # connections +expect+ times.
 245         self.api.users().create(body={}).execute()
 246         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 247         self.api._http.connections = mock_conns.copy()
 248         self.api.users().create(body={}).execute()
 249         for c in mock_conns.values():
 250             self.assertEqual(c.close.call_count, expect)
 251
 252     @mock.patch('time.sleep')
 253     def test_socket_error_no_retry_post(self, sleep):
 254         self.api._http.orig_http_request.side_effect = (
 255             socket.error('mock error'),
 256             self.request_success,
 257         )
 258         with self.assertRaises(socket.error):
 259             self.api.users().create(body={}).execute()
 260         self.assertEqual(self.api._http.orig_http_request.call_count, 1,
 261                          "client should try non-retryable method exactly once")
 262         self.assertEqual(sleep.call_args_list, [])
 263
 264
 265 if __name__ == '__main__':
 266     unittest.main()