sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import str
   7 from builtins import range
   8 import arvados
   9 import collections
  10 import httplib2
  11 import itertools
  12 import json
  13 import mimetypes
  14 import os
  15 import socket
  16 import string
  17 import unittest
  18
  19 import mock
  20 from . import run_test_server
  21
  22 from apiclient import errors as apiclient_errors
  23 from apiclient import http as apiclient_http
  24 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
  25 from .arvados_testutil import fake_httplib2_response, queue_with
  26
  27 if not mimetypes.inited:
  28     mimetypes.init()
  29
  30 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  31     MAIN_SERVER = {}
  32     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  33
  34     def api_error_response(self, code, *errors):
  35         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  36                 json.dumps({'errors': errors,
  37                             'error_token': '1234567890+12345678'}).encode())
  38
  39     def test_new_api_objects_with_cache(self):
  40         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  41         self.assertIsNot(*clients)
  42
  43     def test_empty_list(self):
  44         answer = arvados.api('v1').humans().list(
  45             filters=[['uuid', '=', None]]).execute()
  46         self.assertEqual(answer['items_available'], len(answer['items']))
  47
  48     def test_nonempty_list(self):
  49         answer = arvados.api('v1').collections().list().execute()
  50         self.assertNotEqual(0, answer['items_available'])
  51         self.assertNotEqual(0, len(answer['items']))
  52
  53     def test_timestamp_inequality_filter(self):
  54         api = arvados.api('v1')
  55         new_item = api.specimens().create(body={}).execute()
  56         for operator, should_include in [
  57                 ['<', False], ['>', False],
  58                 ['<=', True], ['>=', True], ['=', True]]:
  59             response = api.specimens().list(filters=[
  60                 ['created_at', operator, new_item['created_at']],
  61                 # Also filter by uuid to ensure (if it matches) it's on page 0
  62                 ['uuid', '=', new_item['uuid']]]).execute()
  63             uuids = [item['uuid'] for item in response['items']]
  64             did_include = new_item['uuid'] in uuids
  65             self.assertEqual(
  66                 did_include, should_include,
  67                 "'%s %s' filter should%s have matched '%s'" % (
  68                     operator, new_item['created_at'],
  69                     ('' if should_include else ' not'),
  70                     new_item['created_at']))
  71
  72     def test_exceptions_include_errors(self):
  73         mock_responses = {
  74             'arvados.humans.get': self.api_error_response(
  75                 422, "Bad UUID format", "Bad output format"),
  76             }
  77         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  78         api = arvados.api('v1', requestBuilder=req_builder)
  79         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  80             api.humans().get(uuid='xyz-xyz-abcdef').execute()
  81         err_s = str(err_ctx.exception)
  82         for msg in ["Bad UUID format", "Bad output format"]:
  83             self.assertIn(msg, err_s)
  84
  85     @mock.patch('time.sleep')
  86     def test_exceptions_include_request_id(self, sleep):
  87         api = arvados.api('v1')
  88         api.request_id='fake-request-id'
  89         api._http.orig_http_request = mock.MagicMock()
  90         api._http.orig_http_request.side_effect = socket.error('mock error')
  91         caught = None
  92         try:
  93             api.users().current().execute()
  94         except Exception as e:
  95             caught = e
  96         self.assertRegex(str(caught), r'fake-request-id')
  97
  98     def test_exceptions_without_errors_have_basic_info(self):
  99         mock_responses = {
 100             'arvados.humans.delete': (
 101                 fake_httplib2_response(500, **self.ERROR_HEADERS),
 102                 b"")
 103             }
 104         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 105         api = arvados.api('v1', requestBuilder=req_builder)
 106         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 107             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
 108         self.assertIn("500", str(err_ctx.exception))
 109
 110     def test_request_too_large(self):
 111         api = arvados.api('v1')
 112         maxsize = api._rootDesc.get('maxRequestSize', 0)
 113         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 114             text = "X" * maxsize
 115             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 116
 117     def test_default_request_timeout(self):
 118         api = arvados.api('v1')
 119         self.assertEqual(api._http.timeout, 300,
 120             "Default timeout value should be 300")
 121
 122     def test_custom_request_timeout(self):
 123         api = arvados.api('v1', timeout=1234)
 124         self.assertEqual(api._http.timeout, 1234,
 125             "Requested timeout value was 1234")
 126
 127     def test_ordered_json_model(self):
 128         mock_responses = {
 129             'arvados.humans.get': (
 130                 None,
 131                 json.dumps(collections.OrderedDict(
 132                     (c, int(c, 16)) for c in string.hexdigits
 133                 )).encode(),
 134             ),
 135         }
 136         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 137         api = arvados.api('v1',
 138                           requestBuilder=req_builder, model=OrderedJsonModel())
 139         result = api.humans().get(uuid='test').execute()
 140         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 141
 142
 143 class RetryREST(unittest.TestCase):
 144     def setUp(self):
 145         self.api = arvados.api('v1')
 146         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 147                         "test doesn't know how to intercept HTTP requests")
 148         self.mock_response = {'user': 'person'}
 149         self.request_success = (fake_httplib2_response(200),
 150                                 json.dumps(self.mock_response))
 151         self.api._http.orig_http_request = mock.MagicMock()
 152         # All requests succeed by default. Tests override as needed.
 153         self.api._http.orig_http_request.return_value = self.request_success
 154
 155     @mock.patch('time.sleep')
 156     def test_socket_error_retry_get(self, sleep):
 157         self.api._http.orig_http_request.side_effect = (
 158             socket.error('mock error'),
 159             self.request_success,
 160         )
 161         self.assertEqual(self.api.users().current().execute(),
 162                          self.mock_response)
 163         self.assertGreater(self.api._http.orig_http_request.call_count, 1,
 164                            "client got the right response without retrying")
 165         self.assertEqual(sleep.call_args_list,
 166                          [mock.call(RETRY_DELAY_INITIAL)])
 167
 168     @mock.patch('time.sleep')
 169     def test_same_automatic_request_id_on_retry(self, sleep):
 170         self.api._http.orig_http_request.side_effect = (
 171             socket.error('mock error'),
 172             self.request_success,
 173         )
 174         self.api.users().current().execute()
 175         calls = self.api._http.orig_http_request.call_args_list
 176         self.assertEqual(len(calls), 2)
 177         self.assertEqual(
 178             calls[0][1]['headers']['X-Request-Id'],
 179             calls[1][1]['headers']['X-Request-Id'])
 180         self.assertRegex(calls[0][1]['headers']['X-Request-Id'], r'^req-[a-z0-9]{20}$')
 181
 182     @mock.patch('time.sleep')
 183     def test_provided_request_id_on_retry(self, sleep):
 184         self.api.request_id='fake-request-id'
 185         self.api._http.orig_http_request.side_effect = (
 186             socket.error('mock error'),
 187             self.request_success,
 188         )
 189         self.api.users().current().execute()
 190         calls = self.api._http.orig_http_request.call_args_list
 191         self.assertEqual(len(calls), 2)
 192         for call in calls:
 193             self.assertEqual(call[1]['headers']['X-Request-Id'], 'fake-request-id')
 194
 195     @mock.patch('time.sleep')
 196     def test_socket_error_retry_delay(self, sleep):
 197         self.api._http.orig_http_request.side_effect = socket.error('mock')
 198         self.api._http._retry_count = 3
 199         with self.assertRaises(socket.error):
 200             self.api.users().current().execute()
 201         self.assertEqual(self.api._http.orig_http_request.call_count, 4)
 202         self.assertEqual(sleep.call_args_list, [
 203             mock.call(RETRY_DELAY_INITIAL),
 204             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
 205             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
 206         ])
 207
 208     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 209     def test_close_old_connections_non_retryable(self, sleep):
 210         self._test_connection_close(expect=1)
 211
 212     @mock.patch('time.time', side_effect=itertools.count())
 213     def test_no_close_fresh_connections_non_retryable(self, sleep):
 214         self._test_connection_close(expect=0)
 215
 216     @mock.patch('time.time', side_effect=itertools.count())
 217     def test_override_max_idle_time(self, sleep):
 218         self.api._http._max_keepalive_idle = 0
 219         self._test_connection_close(expect=1)
 220
 221     def _test_connection_close(self, expect=0):
 222         # Do two POST requests. The second one must close all
 223         # connections +expect+ times.
 224         self.api.users().create(body={}).execute()
 225         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 226         self.api._http.connections = mock_conns.copy()
 227         self.api.users().create(body={}).execute()
 228         for c in mock_conns.values():
 229             self.assertEqual(c.close.call_count, expect)
 230
 231     @mock.patch('time.sleep')
 232     def test_socket_error_no_retry_post(self, sleep):
 233         self.api._http.orig_http_request.side_effect = (
 234             socket.error('mock error'),
 235             self.request_success,
 236         )
 237         with self.assertRaises(socket.error):
 238             self.api.users().create(body={}).execute()
 239         self.assertEqual(self.api._http.orig_http_request.call_count, 1,
 240                          "client should try non-retryable method exactly once")
 241         self.assertEqual(sleep.call_args_list, [])
 242
 243
 244 if __name__ == '__main__':
 245     unittest.main()