sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import str
   7 from builtins import range
   8 import arvados
   9 import collections
  10 import httplib2
  11 import itertools
  12 import json
  13 import mimetypes
  14 import os
  15 import socket
  16 import string
  17 import unittest
  18
  19 import mock
  20 from . import run_test_server
  21
  22 from apiclient import errors as apiclient_errors
  23 from apiclient import http as apiclient_http
  24 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
  25 from .arvados_testutil import fake_httplib2_response, queue_with
  26
  27 if not mimetypes.inited:
  28     mimetypes.init()
  29
  30 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  31     MAIN_SERVER = {}
  32     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  33
  34     def api_error_response(self, code, *errors):
  35         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  36                 json.dumps({'errors': errors,
  37                             'error_token': '1234567890+12345678'}).encode())
  38
  39     def test_new_api_objects_with_cache(self):
  40         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  41         self.assertIsNot(*clients)
  42
  43     def test_empty_list(self):
  44         answer = arvados.api('v1').humans().list(
  45             filters=[['uuid', '=', None]]).execute()
  46         self.assertEqual(answer['items_available'], len(answer['items']))
  47
  48     def test_nonempty_list(self):
  49         answer = arvados.api('v1').collections().list().execute()
  50         self.assertNotEqual(0, answer['items_available'])
  51         self.assertNotEqual(0, len(answer['items']))
  52
  53     def test_timestamp_inequality_filter(self):
  54         api = arvados.api('v1')
  55         new_item = api.specimens().create(body={}).execute()
  56         for operator, should_include in [
  57                 ['<', False], ['>', False],
  58                 ['<=', True], ['>=', True], ['=', True]]:
  59             response = api.specimens().list(filters=[
  60                 ['created_at', operator, new_item['created_at']],
  61                 # Also filter by uuid to ensure (if it matches) it's on page 0
  62                 ['uuid', '=', new_item['uuid']]]).execute()
  63             uuids = [item['uuid'] for item in response['items']]
  64             did_include = new_item['uuid'] in uuids
  65             self.assertEqual(
  66                 did_include, should_include,
  67                 "'%s %s' filter should%s have matched '%s'" % (
  68                     operator, new_item['created_at'],
  69                     ('' if should_include else ' not'),
  70                     new_item['created_at']))
  71
  72     def test_exceptions_include_errors(self):
  73         mock_responses = {
  74             'arvados.humans.get': self.api_error_response(
  75                 422, "Bad UUID format", "Bad output format"),
  76             }
  77         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  78         api = arvados.api('v1', requestBuilder=req_builder)
  79         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  80             api.humans().get(uuid='xyz-xyz-abcdef').execute()
  81         err_s = str(err_ctx.exception)
  82         for msg in ["Bad UUID format", "Bad output format"]:
  83             self.assertIn(msg, err_s)
  84
  85     def test_exceptions_without_errors_have_basic_info(self):
  86         mock_responses = {
  87             'arvados.humans.delete': (
  88                 fake_httplib2_response(500, **self.ERROR_HEADERS),
  89                 b"")
  90             }
  91         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  92         api = arvados.api('v1', requestBuilder=req_builder)
  93         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  94             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
  95         self.assertIn("500", str(err_ctx.exception))
  96
  97     def test_request_too_large(self):
  98         api = arvados.api('v1')
  99         maxsize = api._rootDesc.get('maxRequestSize', 0)
 100         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 101             text = "X" * maxsize
 102             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 103
 104     # Checks for bug #17171
 105     def test_default_request_timeout(self):
 106         api = arvados.api('v1')
 107         self.assertEqual(api._http.timeout, 10,
 108             "Default timeout value should be 10")
 109
 110     def test_ordered_json_model(self):
 111         mock_responses = {
 112             'arvados.humans.get': (
 113                 None,
 114                 json.dumps(collections.OrderedDict(
 115                     (c, int(c, 16)) for c in string.hexdigits
 116                 )).encode(),
 117             ),
 118         }
 119         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 120         api = arvados.api('v1',
 121                           requestBuilder=req_builder, model=OrderedJsonModel())
 122         result = api.humans().get(uuid='test').execute()
 123         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 124
 125
 126 class RetryREST(unittest.TestCase):
 127     def setUp(self):
 128         self.api = arvados.api('v1')
 129         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 130                         "test doesn't know how to intercept HTTP requests")
 131         self.mock_response = {'user': 'person'}
 132         self.request_success = (fake_httplib2_response(200),
 133                                 json.dumps(self.mock_response))
 134         self.api._http.orig_http_request = mock.MagicMock()
 135         # All requests succeed by default. Tests override as needed.
 136         self.api._http.orig_http_request.return_value = self.request_success
 137
 138     @mock.patch('time.sleep')
 139     def test_socket_error_retry_get(self, sleep):
 140         self.api._http.orig_http_request.side_effect = (
 141             socket.error('mock error'),
 142             self.request_success,
 143         )
 144         self.assertEqual(self.api.users().current().execute(),
 145                          self.mock_response)
 146         self.assertGreater(self.api._http.orig_http_request.call_count, 1,
 147                            "client got the right response without retrying")
 148         self.assertEqual(sleep.call_args_list,
 149                          [mock.call(RETRY_DELAY_INITIAL)])
 150
 151     @mock.patch('time.sleep')
 152     def test_same_automatic_request_id_on_retry(self, sleep):
 153         self.api._http.orig_http_request.side_effect = (
 154             socket.error('mock error'),
 155             self.request_success,
 156         )
 157         self.api.users().current().execute()
 158         calls = self.api._http.orig_http_request.call_args_list
 159         self.assertEqual(len(calls), 2)
 160         self.assertEqual(
 161             calls[0][1]['headers']['X-Request-Id'],
 162             calls[1][1]['headers']['X-Request-Id'])
 163         self.assertRegex(calls[0][1]['headers']['X-Request-Id'], r'^req-[a-z0-9]{20}$')
 164
 165     @mock.patch('time.sleep')
 166     def test_provided_request_id_on_retry(self, sleep):
 167         self.api.request_id='fake-request-id'
 168         self.api._http.orig_http_request.side_effect = (
 169             socket.error('mock error'),
 170             self.request_success,
 171         )
 172         self.api.users().current().execute()
 173         calls = self.api._http.orig_http_request.call_args_list
 174         self.assertEqual(len(calls), 2)
 175         for call in calls:
 176             self.assertEqual(call[1]['headers']['X-Request-Id'], 'fake-request-id')
 177
 178     @mock.patch('time.sleep')
 179     def test_socket_error_retry_delay(self, sleep):
 180         self.api._http.orig_http_request.side_effect = socket.error('mock')
 181         self.api._http._retry_count = 3
 182         with self.assertRaises(socket.error):
 183             self.api.users().current().execute()
 184         self.assertEqual(self.api._http.orig_http_request.call_count, 4)
 185         self.assertEqual(sleep.call_args_list, [
 186             mock.call(RETRY_DELAY_INITIAL),
 187             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
 188             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
 189         ])
 190
 191     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 192     def test_close_old_connections_non_retryable(self, sleep):
 193         self._test_connection_close(expect=1)
 194
 195     @mock.patch('time.time', side_effect=itertools.count())
 196     def test_no_close_fresh_connections_non_retryable(self, sleep):
 197         self._test_connection_close(expect=0)
 198
 199     @mock.patch('time.time', side_effect=itertools.count())
 200     def test_override_max_idle_time(self, sleep):
 201         self.api._http._max_keepalive_idle = 0
 202         self._test_connection_close(expect=1)
 203
 204     def _test_connection_close(self, expect=0):
 205         # Do two POST requests. The second one must close all
 206         # connections +expect+ times.
 207         self.api.users().create(body={}).execute()
 208         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 209         self.api._http.connections = mock_conns.copy()
 210         self.api.users().create(body={}).execute()
 211         for c in mock_conns.values():
 212             self.assertEqual(c.close.call_count, expect)
 213
 214     @mock.patch('time.sleep')
 215     def test_socket_error_no_retry_post(self, sleep):
 216         self.api._http.orig_http_request.side_effect = (
 217             socket.error('mock error'),
 218             self.request_success,
 219         )
 220         with self.assertRaises(socket.error):
 221             self.api.users().create(body={}).execute()
 222         self.assertEqual(self.api._http.orig_http_request.call_count, 1,
 223                          "client should try non-retryable method exactly once")
 224         self.assertEqual(sleep.call_args_list, [])
 225
 226
 227 if __name__ == '__main__':
 228     unittest.main()