sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import str
   7 from builtins import range
   8 import arvados
   9 import collections
  10 import httplib2
  11 import itertools
  12 import json
  13 import mimetypes
  14 import os
  15 import socket
  16 import string
  17 import unittest
  18
  19 import mock
  20 from . import run_test_server
  21
  22 from apiclient import errors as apiclient_errors
  23 from apiclient import http as apiclient_http
  24 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
  25 from .arvados_testutil import fake_httplib2_response, queue_with
  26
  27 if not mimetypes.inited:
  28     mimetypes.init()
  29
  30 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  31     MAIN_SERVER = {}
  32     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  33
  34     def api_error_response(self, code, *errors):
  35         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  36                 json.dumps({'errors': errors,
  37                             'error_token': '1234567890+12345678'}).encode())
  38
  39     def test_new_api_objects_with_cache(self):
  40         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  41         self.assertIsNot(*clients)
  42
  43     def test_empty_list(self):
  44         answer = arvados.api('v1').humans().list(
  45             filters=[['uuid', '=', None]]).execute()
  46         self.assertEqual(answer['items_available'], len(answer['items']))
  47
  48     def test_nonempty_list(self):
  49         answer = arvados.api('v1').collections().list().execute()
  50         self.assertNotEqual(0, answer['items_available'])
  51         self.assertNotEqual(0, len(answer['items']))
  52
  53     def test_timestamp_inequality_filter(self):
  54         api = arvados.api('v1')
  55         new_item = api.specimens().create(body={}).execute()
  56         for operator, should_include in [
  57                 ['<', False], ['>', False],
  58                 ['<=', True], ['>=', True], ['=', True]]:
  59             response = api.specimens().list(filters=[
  60                 ['created_at', operator, new_item['created_at']],
  61                 # Also filter by uuid to ensure (if it matches) it's on page 0
  62                 ['uuid', '=', new_item['uuid']]]).execute()
  63             uuids = [item['uuid'] for item in response['items']]
  64             did_include = new_item['uuid'] in uuids
  65             self.assertEqual(
  66                 did_include, should_include,
  67                 "'%s %s' filter should%s have matched '%s'" % (
  68                     operator, new_item['created_at'],
  69                     ('' if should_include else ' not'),
  70                     new_item['created_at']))
  71
  72     def test_exceptions_include_errors(self):
  73         mock_responses = {
  74             'arvados.humans.get': self.api_error_response(
  75                 422, "Bad UUID format", "Bad output format"),
  76             }
  77         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  78         api = arvados.api('v1', requestBuilder=req_builder)
  79         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  80             api.humans().get(uuid='xyz-xyz-abcdef').execute()
  81         err_s = str(err_ctx.exception)
  82         for msg in ["Bad UUID format", "Bad output format"]:
  83             self.assertIn(msg, err_s)
  84
  85     def test_exceptions_without_errors_have_basic_info(self):
  86         mock_responses = {
  87             'arvados.humans.delete': (
  88                 fake_httplib2_response(500, **self.ERROR_HEADERS),
  89                 b"")
  90             }
  91         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  92         api = arvados.api('v1', requestBuilder=req_builder)
  93         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  94             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
  95         self.assertIn("500", str(err_ctx.exception))
  96
  97     def test_request_too_large(self):
  98         api = arvados.api('v1')
  99         maxsize = api._rootDesc.get('maxRequestSize', 0)
 100         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 101             text = "X" * maxsize
 102             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 103
 104     def test_ordered_json_model(self):
 105         mock_responses = {
 106             'arvados.humans.get': (
 107                 None,
 108                 json.dumps(collections.OrderedDict(
 109                     (c, int(c, 16)) for c in string.hexdigits
 110                 )).encode(),
 111             ),
 112         }
 113         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 114         api = arvados.api('v1',
 115                           requestBuilder=req_builder, model=OrderedJsonModel())
 116         result = api.humans().get(uuid='test').execute()
 117         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 118
 119
 120 class RetryREST(unittest.TestCase):
 121     def setUp(self):
 122         self.api = arvados.api('v1')
 123         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 124                         "test doesn't know how to intercept HTTP requests")
 125         self.mock_response = {'user': 'person'}
 126         self.request_success = (fake_httplib2_response(200),
 127                                 json.dumps(self.mock_response))
 128         self.api._http.orig_http_request = mock.MagicMock()
 129         # All requests succeed by default. Tests override as needed.
 130         self.api._http.orig_http_request.return_value = self.request_success
 131
 132     @mock.patch('time.sleep')
 133     def test_socket_error_retry_get(self, sleep):
 134         self.api._http.orig_http_request.side_effect = (
 135             socket.error('mock error'),
 136             self.request_success,
 137         )
 138         self.assertEqual(self.api.users().current().execute(),
 139                          self.mock_response)
 140         self.assertGreater(self.api._http.orig_http_request.call_count, 1,
 141                            "client got the right response without retrying")
 142         self.assertEqual(sleep.call_args_list,
 143                          [mock.call(RETRY_DELAY_INITIAL)])
 144
 145     @mock.patch('time.sleep')
 146     def test_socket_error_retry_delay(self, sleep):
 147         self.api._http.orig_http_request.side_effect = socket.error('mock')
 148         self.api._http._retry_count = 3
 149         with self.assertRaises(socket.error):
 150             self.api.users().current().execute()
 151         self.assertEqual(self.api._http.orig_http_request.call_count, 4)
 152         self.assertEqual(sleep.call_args_list, [
 153             mock.call(RETRY_DELAY_INITIAL),
 154             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
 155             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
 156         ])
 157
 158     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 159     def test_close_old_connections_non_retryable(self, sleep):
 160         self._test_connection_close(expect=1)
 161
 162     @mock.patch('time.time', side_effect=itertools.count())
 163     def test_no_close_fresh_connections_non_retryable(self, sleep):
 164         self._test_connection_close(expect=0)
 165
 166     @mock.patch('time.time', side_effect=itertools.count())
 167     def test_override_max_idle_time(self, sleep):
 168         self.api._http._max_keepalive_idle = 0
 169         self._test_connection_close(expect=1)
 170
 171     def _test_connection_close(self, expect=0):
 172         # Do two POST requests. The second one must close all
 173         # connections +expect+ times.
 174         self.api.users().create(body={}).execute()
 175         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 176         self.api._http.connections = mock_conns.copy()
 177         self.api.users().create(body={}).execute()
 178         for c in mock_conns.values():
 179             self.assertEqual(c.close.call_count, expect)
 180
 181     @mock.patch('time.sleep')
 182     def test_socket_error_no_retry_post(self, sleep):
 183         self.api._http.orig_http_request.side_effect = (
 184             socket.error('mock error'),
 185             self.request_success,
 186         )
 187         with self.assertRaises(socket.error):
 188             self.api.users().create(body={}).execute()
 189         self.assertEqual(self.api._http.orig_http_request.call_count, 1,
 190                          "client should try non-retryable method exactly once")
 191         self.assertEqual(sleep.call_args_list, [])
 192
 193
 194 if __name__ == '__main__':
 195     unittest.main()