sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import str
   7 from builtins import range
   8 import arvados
   9 import collections
  10 import httplib2
  11 import itertools
  12 import json
  13 import mimetypes
  14 import os
  15 import socket
  16 import string
  17 import unittest
  18
  19 import mock
  20 from . import run_test_server
  21
  22 from apiclient import errors as apiclient_errors
  23 from apiclient import http as apiclient_http
  24 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
  25 from .arvados_testutil import fake_httplib2_response, queue_with
  26
  27 if not mimetypes.inited:
  28     mimetypes.init()
  29
  30 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  31     MAIN_SERVER = {}
  32     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  33
  34     def api_error_response(self, code, *errors):
  35         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  36                 json.dumps({'errors': errors,
  37                             'error_token': '1234567890+12345678'}).encode())
  38
  39     def test_new_api_objects_with_cache(self):
  40         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  41         self.assertIsNot(*clients)
  42
  43     def test_empty_list(self):
  44         answer = arvados.api('v1').humans().list(
  45             filters=[['uuid', '=', None]]).execute()
  46         self.assertEqual(answer['items_available'], len(answer['items']))
  47
  48     def test_nonempty_list(self):
  49         answer = arvados.api('v1').collections().list().execute()
  50         self.assertNotEqual(0, answer['items_available'])
  51         self.assertNotEqual(0, len(answer['items']))
  52
  53     def test_timestamp_inequality_filter(self):
  54         api = arvados.api('v1')
  55         new_item = api.specimens().create(body={}).execute()
  56         for operator, should_include in [
  57                 ['<', False], ['>', False],
  58                 ['<=', True], ['>=', True], ['=', True]]:
  59             response = api.specimens().list(filters=[
  60                 ['created_at', operator, new_item['created_at']],
  61                 # Also filter by uuid to ensure (if it matches) it's on page 0
  62                 ['uuid', '=', new_item['uuid']]]).execute()
  63             uuids = [item['uuid'] for item in response['items']]
  64             did_include = new_item['uuid'] in uuids
  65             self.assertEqual(
  66                 did_include, should_include,
  67                 "'%s %s' filter should%s have matched '%s'" % (
  68                     operator, new_item['created_at'],
  69                     ('' if should_include else ' not'),
  70                     new_item['created_at']))
  71
  72     def test_exceptions_include_errors(self):
  73         mock_responses = {
  74             'arvados.humans.get': self.api_error_response(
  75                 422, "Bad UUID format", "Bad output format"),
  76             }
  77         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  78         api = arvados.api('v1', requestBuilder=req_builder)
  79         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  80             api.humans().get(uuid='xyz-xyz-abcdef').execute()
  81         err_s = str(err_ctx.exception)
  82         for msg in ["Bad UUID format", "Bad output format"]:
  83             self.assertIn(msg, err_s)
  84
  85     def test_exceptions_without_errors_have_basic_info(self):
  86         mock_responses = {
  87             'arvados.humans.delete': (
  88                 fake_httplib2_response(500, **self.ERROR_HEADERS),
  89                 b"")
  90             }
  91         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  92         api = arvados.api('v1', requestBuilder=req_builder)
  93         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  94             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
  95         self.assertIn("500", str(err_ctx.exception))
  96
  97     def test_request_too_large(self):
  98         api = arvados.api('v1')
  99         maxsize = api._rootDesc.get('maxRequestSize', 0)
 100         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 101             text = "X" * maxsize
 102             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 103
 104     def test_default_request_timeout(self):
 105         api = arvados.api('v1')
 106         self.assertEqual(api._http.timeout, 300,
 107             "Default timeout value should be 300")
 108
 109     def test_custom_request_timeout(self):
 110         api = arvados.api('v1', timeout=1234)
 111         self.assertEqual(api._http.timeout, 1234,
 112             "Requested timeout value was 1234")
 113
 114     def test_ordered_json_model(self):
 115         mock_responses = {
 116             'arvados.humans.get': (
 117                 None,
 118                 json.dumps(collections.OrderedDict(
 119                     (c, int(c, 16)) for c in string.hexdigits
 120                 )).encode(),
 121             ),
 122         }
 123         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 124         api = arvados.api('v1',
 125                           requestBuilder=req_builder, model=OrderedJsonModel())
 126         result = api.humans().get(uuid='test').execute()
 127         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 128
 129
 130 class RetryREST(unittest.TestCase):
 131     def setUp(self):
 132         self.api = arvados.api('v1')
 133         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 134                         "test doesn't know how to intercept HTTP requests")
 135         self.mock_response = {'user': 'person'}
 136         self.request_success = (fake_httplib2_response(200),
 137                                 json.dumps(self.mock_response))
 138         self.api._http.orig_http_request = mock.MagicMock()
 139         # All requests succeed by default. Tests override as needed.
 140         self.api._http.orig_http_request.return_value = self.request_success
 141
 142     @mock.patch('time.sleep')
 143     def test_socket_error_retry_get(self, sleep):
 144         self.api._http.orig_http_request.side_effect = (
 145             socket.error('mock error'),
 146             self.request_success,
 147         )
 148         self.assertEqual(self.api.users().current().execute(),
 149                          self.mock_response)
 150         self.assertGreater(self.api._http.orig_http_request.call_count, 1,
 151                            "client got the right response without retrying")
 152         self.assertEqual(sleep.call_args_list,
 153                          [mock.call(RETRY_DELAY_INITIAL)])
 154
 155     @mock.patch('time.sleep')
 156     def test_same_automatic_request_id_on_retry(self, sleep):
 157         self.api._http.orig_http_request.side_effect = (
 158             socket.error('mock error'),
 159             self.request_success,
 160         )
 161         self.api.users().current().execute()
 162         calls = self.api._http.orig_http_request.call_args_list
 163         self.assertEqual(len(calls), 2)
 164         self.assertEqual(
 165             calls[0][1]['headers']['X-Request-Id'],
 166             calls[1][1]['headers']['X-Request-Id'])
 167         self.assertRegex(calls[0][1]['headers']['X-Request-Id'], r'^req-[a-z0-9]{20}$')
 168
 169     @mock.patch('time.sleep')
 170     def test_provided_request_id_on_retry(self, sleep):
 171         self.api.request_id='fake-request-id'
 172         self.api._http.orig_http_request.side_effect = (
 173             socket.error('mock error'),
 174             self.request_success,
 175         )
 176         self.api.users().current().execute()
 177         calls = self.api._http.orig_http_request.call_args_list
 178         self.assertEqual(len(calls), 2)
 179         for call in calls:
 180             self.assertEqual(call[1]['headers']['X-Request-Id'], 'fake-request-id')
 181
 182     @mock.patch('time.sleep')
 183     def test_socket_error_retry_delay(self, sleep):
 184         self.api._http.orig_http_request.side_effect = socket.error('mock')
 185         self.api._http._retry_count = 3
 186         with self.assertRaises(socket.error):
 187             self.api.users().current().execute()
 188         self.assertEqual(self.api._http.orig_http_request.call_count, 4)
 189         self.assertEqual(sleep.call_args_list, [
 190             mock.call(RETRY_DELAY_INITIAL),
 191             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
 192             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
 193         ])
 194
 195     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 196     def test_close_old_connections_non_retryable(self, sleep):
 197         self._test_connection_close(expect=1)
 198
 199     @mock.patch('time.time', side_effect=itertools.count())
 200     def test_no_close_fresh_connections_non_retryable(self, sleep):
 201         self._test_connection_close(expect=0)
 202
 203     @mock.patch('time.time', side_effect=itertools.count())
 204     def test_override_max_idle_time(self, sleep):
 205         self.api._http._max_keepalive_idle = 0
 206         self._test_connection_close(expect=1)
 207
 208     def _test_connection_close(self, expect=0):
 209         # Do two POST requests. The second one must close all
 210         # connections +expect+ times.
 211         self.api.users().create(body={}).execute()
 212         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 213         self.api._http.connections = mock_conns.copy()
 214         self.api.users().create(body={}).execute()
 215         for c in mock_conns.values():
 216             self.assertEqual(c.close.call_count, expect)
 217
 218     @mock.patch('time.sleep')
 219     def test_socket_error_no_retry_post(self, sleep):
 220         self.api._http.orig_http_request.side_effect = (
 221             socket.error('mock error'),
 222             self.request_success,
 223         )
 224         with self.assertRaises(socket.error):
 225             self.api.users().create(body={}).execute()
 226         self.assertEqual(self.api._http.orig_http_request.call_count, 1,
 227                          "client should try non-retryable method exactly once")
 228         self.assertEqual(sleep.call_args_list, [])
 229
 230
 231 if __name__ == '__main__':
 232     unittest.main()