sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import str
   7 from builtins import range
   8 import arvados
   9 import collections
  10 import httplib2
  11 import itertools
  12 import json
  13 import mimetypes
  14 import os
  15 import socket
  16 import string
  17 import unittest
  18
  19 import mock
  20 from . import run_test_server
  21
  22 from apiclient import errors as apiclient_errors
  23 from apiclient import http as apiclient_http
  24 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
  25 from .arvados_testutil import fake_httplib2_response, queue_with
  26
  27 if not mimetypes.inited:
  28     mimetypes.init()
  29
  30 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  31     MAIN_SERVER = {}
  32     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  33
  34     def api_error_response(self, code, *errors):
  35         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  36                 json.dumps({'errors': errors,
  37                             'error_token': '1234567890+12345678'}).encode())
  38
  39     def test_new_api_objects_with_cache(self):
  40         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  41         self.assertIsNot(*clients)
  42
  43     def test_empty_list(self):
  44         answer = arvados.api('v1').humans().list(
  45             filters=[['uuid', '=', None]]).execute()
  46         self.assertEqual(answer['items_available'], len(answer['items']))
  47
  48     def test_nonempty_list(self):
  49         answer = arvados.api('v1').collections().list().execute()
  50         self.assertNotEqual(0, answer['items_available'])
  51         self.assertNotEqual(0, len(answer['items']))
  52
  53     def test_timestamp_inequality_filter(self):
  54         api = arvados.api('v1')
  55         new_item = api.specimens().create(body={}).execute()
  56         for operator, should_include in [
  57                 ['<', False], ['>', False],
  58                 ['<=', True], ['>=', True], ['=', True]]:
  59             response = api.specimens().list(filters=[
  60                 ['created_at', operator, new_item['created_at']],
  61                 # Also filter by uuid to ensure (if it matches) it's on page 0
  62                 ['uuid', '=', new_item['uuid']]]).execute()
  63             uuids = [item['uuid'] for item in response['items']]
  64             did_include = new_item['uuid'] in uuids
  65             self.assertEqual(
  66                 did_include, should_include,
  67                 "'%s %s' filter should%s have matched '%s'" % (
  68                     operator, new_item['created_at'],
  69                     ('' if should_include else ' not'),
  70                     new_item['created_at']))
  71
  72     def test_exceptions_include_errors(self):
  73         mock_responses = {
  74             'arvados.humans.get': self.api_error_response(
  75                 422, "Bad UUID format", "Bad output format"),
  76             }
  77         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  78         api = arvados.api('v1', requestBuilder=req_builder)
  79         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  80             api.humans().get(uuid='xyz-xyz-abcdef').execute()
  81         err_s = str(err_ctx.exception)
  82         for msg in ["Bad UUID format", "Bad output format"]:
  83             self.assertIn(msg, err_s)
  84
  85     def test_exceptions_without_errors_have_basic_info(self):
  86         mock_responses = {
  87             'arvados.humans.delete': (
  88                 fake_httplib2_response(500, **self.ERROR_HEADERS),
  89                 b"")
  90             }
  91         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  92         api = arvados.api('v1', requestBuilder=req_builder)
  93         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  94             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
  95         self.assertIn("500", str(err_ctx.exception))
  96
  97     def test_request_too_large(self):
  98         api = arvados.api('v1')
  99         maxsize = api._rootDesc.get('maxRequestSize', 0)
 100         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 101             text = "X" * maxsize
 102             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 103
 104     def test_ordered_json_model(self):
 105         mock_responses = {
 106             'arvados.humans.get': (
 107                 None,
 108                 json.dumps(collections.OrderedDict(
 109                     (c, int(c, 16)) for c in string.hexdigits
 110                 )).encode(),
 111             ),
 112         }
 113         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 114         api = arvados.api('v1',
 115                           requestBuilder=req_builder, model=OrderedJsonModel())
 116         result = api.humans().get(uuid='test').execute()
 117         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 118
 119
 120 class RetryREST(unittest.TestCase):
 121     def setUp(self):
 122         self.api = arvados.api('v1')
 123         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 124                         "test doesn't know how to intercept HTTP requests")
 125         self.mock_response = {'user': 'person'}
 126         self.request_success = (fake_httplib2_response(200),
 127                                 json.dumps(self.mock_response))
 128         self.api._http.orig_http_request = mock.MagicMock()
 129         # All requests succeed by default. Tests override as needed.
 130         self.api._http.orig_http_request.return_value = self.request_success
 131
 132     @mock.patch('time.sleep')
 133     def test_socket_error_retry_get(self, sleep):
 134         self.api._http.orig_http_request.side_effect = (
 135             socket.error('mock error'),
 136             self.request_success,
 137         )
 138         self.assertEqual(self.api.users().current().execute(),
 139                          self.mock_response)
 140         self.assertGreater(self.api._http.orig_http_request.call_count, 1,
 141                            "client got the right response without retrying")
 142         self.assertEqual(sleep.call_args_list,
 143                          [mock.call(RETRY_DELAY_INITIAL)])
 144
 145     @mock.patch('time.sleep')
 146     def test_same_automatic_request_id_on_retry(self, sleep):
 147         self.api._http.orig_http_request.side_effect = (
 148             socket.error('mock error'),
 149             self.request_success,
 150         )
 151         self.api.users().current().execute()
 152         calls = self.api._http.orig_http_request.call_args_list
 153         self.assertEqual(len(calls), 2)
 154         self.assertEqual(
 155             calls[0][1]['headers']['X-Request-Id'],
 156             calls[1][1]['headers']['X-Request-Id'])
 157         self.assertRegex(calls[0][1]['headers']['X-Request-Id'], r'^req-[a-z0-9]{20}$')
 158
 159     @mock.patch('time.sleep')
 160     def test_provided_request_id_on_retry(self, sleep):
 161         self.api.request_id='fake-request-id'
 162         self.api._http.orig_http_request.side_effect = (
 163             socket.error('mock error'),
 164             self.request_success,
 165         )
 166         self.api.users().current().execute()
 167         calls = self.api._http.orig_http_request.call_args_list
 168         self.assertEqual(len(calls), 2)
 169         for call in calls:
 170             self.assertEqual(call[1]['headers']['X-Request-Id'], 'fake-request-id')
 171
 172     @mock.patch('time.sleep')
 173     def test_socket_error_retry_delay(self, sleep):
 174         self.api._http.orig_http_request.side_effect = socket.error('mock')
 175         self.api._http._retry_count = 3
 176         with self.assertRaises(socket.error):
 177             self.api.users().current().execute()
 178         self.assertEqual(self.api._http.orig_http_request.call_count, 4)
 179         self.assertEqual(sleep.call_args_list, [
 180             mock.call(RETRY_DELAY_INITIAL),
 181             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
 182             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
 183         ])
 184
 185     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 186     def test_close_old_connections_non_retryable(self, sleep):
 187         self._test_connection_close(expect=1)
 188
 189     @mock.patch('time.time', side_effect=itertools.count())
 190     def test_no_close_fresh_connections_non_retryable(self, sleep):
 191         self._test_connection_close(expect=0)
 192
 193     @mock.patch('time.time', side_effect=itertools.count())
 194     def test_override_max_idle_time(self, sleep):
 195         self.api._http._max_keepalive_idle = 0
 196         self._test_connection_close(expect=1)
 197
 198     def _test_connection_close(self, expect=0):
 199         # Do two POST requests. The second one must close all
 200         # connections +expect+ times.
 201         self.api.users().create(body={}).execute()
 202         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 203         self.api._http.connections = mock_conns.copy()
 204         self.api.users().create(body={}).execute()
 205         for c in mock_conns.values():
 206             self.assertEqual(c.close.call_count, expect)
 207
 208     @mock.patch('time.sleep')
 209     def test_socket_error_no_retry_post(self, sleep):
 210         self.api._http.orig_http_request.side_effect = (
 211             socket.error('mock error'),
 212             self.request_success,
 213         )
 214         with self.assertRaises(socket.error):
 215             self.api.users().create(body={}).execute()
 216         self.assertEqual(self.api._http.orig_http_request.call_count, 1,
 217                          "client should try non-retryable method exactly once")
 218         self.assertEqual(sleep.call_args_list, [])
 219
 220
 221 if __name__ == '__main__':
 222     unittest.main()