sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import str
   7 from builtins import range
   8 import arvados
   9 import collections
  10 import httplib2
  11 import itertools
  12 import json
  13 import mimetypes
  14 import os
  15 import socket
  16 import string
  17 import unittest
  18
  19 import mock
  20 from . import run_test_server
  21
  22 from apiclient import errors as apiclient_errors
  23 from apiclient import http as apiclient_http
  24 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
  25 from .arvados_testutil import fake_httplib2_response, queue_with
  26
  27 if not mimetypes.inited:
  28     mimetypes.init()
  29
  30 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  31     MAIN_SERVER = {}
  32     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  33
  34     def api_error_response(self, code, *errors):
  35         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  36                 json.dumps({'errors': errors,
  37                             'error_token': '1234567890+12345678'}).encode())
  38
  39     def test_new_api_objects_with_cache(self):
  40         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  41         self.assertIsNot(*clients)
  42
  43     def test_empty_list(self):
  44         answer = arvados.api('v1').humans().list(
  45             filters=[['uuid', '=', None]]).execute()
  46         self.assertEqual(answer['items_available'], len(answer['items']))
  47
  48     def test_nonempty_list(self):
  49         answer = arvados.api('v1').collections().list().execute()
  50         self.assertNotEqual(0, answer['items_available'])
  51         self.assertNotEqual(0, len(answer['items']))
  52
  53     def test_timestamp_inequality_filter(self):
  54         api = arvados.api('v1')
  55         new_item = api.specimens().create(body={}).execute()
  56         for operator, should_include in [
  57                 ['<', False], ['>', False],
  58                 ['<=', True], ['>=', True], ['=', True]]:
  59             response = api.specimens().list(filters=[
  60                 ['created_at', operator, new_item['created_at']],
  61                 # Also filter by uuid to ensure (if it matches) it's on page 0
  62                 ['uuid', '=', new_item['uuid']]]).execute()
  63             uuids = [item['uuid'] for item in response['items']]
  64             did_include = new_item['uuid'] in uuids
  65             self.assertEqual(
  66                 did_include, should_include,
  67                 "'%s %s' filter should%s have matched '%s'" % (
  68                     operator, new_item['created_at'],
  69                     ('' if should_include else ' not'),
  70                     new_item['created_at']))
  71
  72     def test_exceptions_include_errors(self):
  73         mock_responses = {
  74             'arvados.humans.get': self.api_error_response(
  75                 422, "Bad UUID format", "Bad output format"),
  76             }
  77         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  78         api = arvados.api('v1', requestBuilder=req_builder)
  79         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  80             api.humans().get(uuid='xyz-xyz-abcdef').execute()
  81         err_s = str(err_ctx.exception)
  82         for msg in ["Bad UUID format", "Bad output format"]:
  83             self.assertIn(msg, err_s)
  84
  85     def test_exceptions_without_errors_have_basic_info(self):
  86         mock_responses = {
  87             'arvados.humans.delete': (
  88                 fake_httplib2_response(500, **self.ERROR_HEADERS),
  89                 b"")
  90             }
  91         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  92         api = arvados.api('v1', requestBuilder=req_builder)
  93         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  94             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
  95         self.assertIn("500", str(err_ctx.exception))
  96
  97     def test_request_too_large(self):
  98         api = arvados.api('v1')
  99         maxsize = api._rootDesc.get('maxRequestSize', 0)
 100         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 101             text = "X" * maxsize
 102             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 103
 104     # Checks for bug #17171
 105     def test_default_request_timeout(self):
 106         api = arvados.api('v1')
 107         self.assertEqual(api._http.timeout, 10,
 108             "Default timeout value should be 10")
 109
 110     # Checks for bug #17989
 111     def test_custom_request_timeout(self):
 112         api = arvados.api('v1', timeout=1234)
 113         self.assertEqual(api._http.timeout, 1234,
 114             "Requested timeout value was 1234")
 115
 116     def test_ordered_json_model(self):
 117         mock_responses = {
 118             'arvados.humans.get': (
 119                 None,
 120                 json.dumps(collections.OrderedDict(
 121                     (c, int(c, 16)) for c in string.hexdigits
 122                 )).encode(),
 123             ),
 124         }
 125         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 126         api = arvados.api('v1',
 127                           requestBuilder=req_builder, model=OrderedJsonModel())
 128         result = api.humans().get(uuid='test').execute()
 129         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 130
 131
 132 class RetryREST(unittest.TestCase):
 133     def setUp(self):
 134         self.api = arvados.api('v1')
 135         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 136                         "test doesn't know how to intercept HTTP requests")
 137         self.mock_response = {'user': 'person'}
 138         self.request_success = (fake_httplib2_response(200),
 139                                 json.dumps(self.mock_response))
 140         self.api._http.orig_http_request = mock.MagicMock()
 141         # All requests succeed by default. Tests override as needed.
 142         self.api._http.orig_http_request.return_value = self.request_success
 143
 144     @mock.patch('time.sleep')
 145     def test_socket_error_retry_get(self, sleep):
 146         self.api._http.orig_http_request.side_effect = (
 147             socket.error('mock error'),
 148             self.request_success,
 149         )
 150         self.assertEqual(self.api.users().current().execute(),
 151                          self.mock_response)
 152         self.assertGreater(self.api._http.orig_http_request.call_count, 1,
 153                            "client got the right response without retrying")
 154         self.assertEqual(sleep.call_args_list,
 155                          [mock.call(RETRY_DELAY_INITIAL)])
 156
 157     @mock.patch('time.sleep')
 158     def test_same_automatic_request_id_on_retry(self, sleep):
 159         self.api._http.orig_http_request.side_effect = (
 160             socket.error('mock error'),
 161             self.request_success,
 162         )
 163         self.api.users().current().execute()
 164         calls = self.api._http.orig_http_request.call_args_list
 165         self.assertEqual(len(calls), 2)
 166         self.assertEqual(
 167             calls[0][1]['headers']['X-Request-Id'],
 168             calls[1][1]['headers']['X-Request-Id'])
 169         self.assertRegex(calls[0][1]['headers']['X-Request-Id'], r'^req-[a-z0-9]{20}$')
 170
 171     @mock.patch('time.sleep')
 172     def test_provided_request_id_on_retry(self, sleep):
 173         self.api.request_id='fake-request-id'
 174         self.api._http.orig_http_request.side_effect = (
 175             socket.error('mock error'),
 176             self.request_success,
 177         )
 178         self.api.users().current().execute()
 179         calls = self.api._http.orig_http_request.call_args_list
 180         self.assertEqual(len(calls), 2)
 181         for call in calls:
 182             self.assertEqual(call[1]['headers']['X-Request-Id'], 'fake-request-id')
 183
 184     @mock.patch('time.sleep')
 185     def test_socket_error_retry_delay(self, sleep):
 186         self.api._http.orig_http_request.side_effect = socket.error('mock')
 187         self.api._http._retry_count = 3
 188         with self.assertRaises(socket.error):
 189             self.api.users().current().execute()
 190         self.assertEqual(self.api._http.orig_http_request.call_count, 4)
 191         self.assertEqual(sleep.call_args_list, [
 192             mock.call(RETRY_DELAY_INITIAL),
 193             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
 194             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
 195         ])
 196
 197     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 198     def test_close_old_connections_non_retryable(self, sleep):
 199         self._test_connection_close(expect=1)
 200
 201     @mock.patch('time.time', side_effect=itertools.count())
 202     def test_no_close_fresh_connections_non_retryable(self, sleep):
 203         self._test_connection_close(expect=0)
 204
 205     @mock.patch('time.time', side_effect=itertools.count())
 206     def test_override_max_idle_time(self, sleep):
 207         self.api._http._max_keepalive_idle = 0
 208         self._test_connection_close(expect=1)
 209
 210     def _test_connection_close(self, expect=0):
 211         # Do two POST requests. The second one must close all
 212         # connections +expect+ times.
 213         self.api.users().create(body={}).execute()
 214         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 215         self.api._http.connections = mock_conns.copy()
 216         self.api.users().create(body={}).execute()
 217         for c in mock_conns.values():
 218             self.assertEqual(c.close.call_count, expect)
 219
 220     @mock.patch('time.sleep')
 221     def test_socket_error_no_retry_post(self, sleep):
 222         self.api._http.orig_http_request.side_effect = (
 223             socket.error('mock error'),
 224             self.request_success,
 225         )
 226         with self.assertRaises(socket.error):
 227             self.api.users().create(body={}).execute()
 228         self.assertEqual(self.api._http.orig_http_request.call_count, 1,
 229                          "client should try non-retryable method exactly once")
 230         self.assertEqual(sleep.call_args_list, [])
 231
 232
 233 if __name__ == '__main__':
 234     unittest.main()