sdk/python/tests/test_api.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import absolute_import
   4 from builtins import str
   5 from builtins import range
   6 import arvados
   7 import collections
   8 import httplib2
   9 import itertools
  10 import json
  11 import mimetypes
  12 import os
  13 import socket
  14 import string
  15 import unittest
  16
  17 import mock
  18 from . import run_test_server
  19
  20 from apiclient import errors as apiclient_errors
  21 from apiclient import http as apiclient_http
  22 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
  23 from .arvados_testutil import fake_httplib2_response, queue_with
  24
  25 if not mimetypes.inited:
  26     mimetypes.init()
  27
  28 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  29     MAIN_SERVER = {}
  30     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  31
  32     def api_error_response(self, code, *errors):
  33         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  34                 json.dumps({'errors': errors,
  35                             'error_token': '1234567890+12345678'}).encode())
  36
  37     def test_new_api_objects_with_cache(self):
  38         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  39         self.assertIsNot(*clients)
  40
  41     def test_empty_list(self):
  42         answer = arvados.api('v1').humans().list(
  43             filters=[['uuid', '=', None]]).execute()
  44         self.assertEqual(answer['items_available'], len(answer['items']))
  45
  46     def test_nonempty_list(self):
  47         answer = arvados.api('v1').collections().list().execute()
  48         self.assertNotEqual(0, answer['items_available'])
  49         self.assertNotEqual(0, len(answer['items']))
  50
  51     def test_timestamp_inequality_filter(self):
  52         api = arvados.api('v1')
  53         new_item = api.specimens().create(body={}).execute()
  54         for operator, should_include in [
  55                 ['<', False], ['>', False],
  56                 ['<=', True], ['>=', True], ['=', True]]:
  57             response = api.specimens().list(filters=[
  58                 ['created_at', operator, new_item['created_at']],
  59                 # Also filter by uuid to ensure (if it matches) it's on page 0
  60                 ['uuid', '=', new_item['uuid']]]).execute()
  61             uuids = [item['uuid'] for item in response['items']]
  62             did_include = new_item['uuid'] in uuids
  63             self.assertEqual(
  64                 did_include, should_include,
  65                 "'%s %s' filter should%s have matched '%s'" % (
  66                     operator, new_item['created_at'],
  67                     ('' if should_include else ' not'),
  68                     new_item['created_at']))
  69
  70     def test_exceptions_include_errors(self):
  71         mock_responses = {
  72             'arvados.humans.get': self.api_error_response(
  73                 422, "Bad UUID format", "Bad output format"),
  74             }
  75         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  76         api = arvados.api('v1', requestBuilder=req_builder)
  77         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  78             api.humans().get(uuid='xyz-xyz-abcdef').execute()
  79         err_s = str(err_ctx.exception)
  80         for msg in ["Bad UUID format", "Bad output format"]:
  81             self.assertIn(msg, err_s)
  82
  83     def test_exceptions_without_errors_have_basic_info(self):
  84         mock_responses = {
  85             'arvados.humans.delete': (
  86                 fake_httplib2_response(500, **self.ERROR_HEADERS),
  87                 b"")
  88             }
  89         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  90         api = arvados.api('v1', requestBuilder=req_builder)
  91         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  92             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
  93         self.assertIn("500", str(err_ctx.exception))
  94
  95     def test_request_too_large(self):
  96         api = arvados.api('v1')
  97         maxsize = api._rootDesc.get('maxRequestSize', 0)
  98         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
  99             text = "X" * maxsize
 100             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 101
 102     def test_ordered_json_model(self):
 103         mock_responses = {
 104             'arvados.humans.get': (
 105                 None,
 106                 json.dumps(collections.OrderedDict(
 107                     (c, int(c, 16)) for c in string.hexdigits
 108                 )).encode(),
 109             ),
 110         }
 111         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 112         api = arvados.api('v1',
 113                           requestBuilder=req_builder, model=OrderedJsonModel())
 114         result = api.humans().get(uuid='test').execute()
 115         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 116
 117
 118 class RetryREST(unittest.TestCase):
 119     def setUp(self):
 120         self.api = arvados.api('v1')
 121         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 122                         "test doesn't know how to intercept HTTP requests")
 123         self.mock_response = {'user': 'person'}
 124         self.request_success = (fake_httplib2_response(200),
 125                                 json.dumps(self.mock_response))
 126         self.api._http.orig_http_request = mock.MagicMock()
 127         # All requests succeed by default. Tests override as needed.
 128         self.api._http.orig_http_request.return_value = self.request_success
 129
 130     @mock.patch('time.sleep')
 131     def test_socket_error_retry_get(self, sleep):
 132         self.api._http.orig_http_request.side_effect = (
 133             socket.error('mock error'),
 134             self.request_success,
 135         )
 136         self.assertEqual(self.api.users().current().execute(),
 137                          self.mock_response)
 138         self.assertGreater(self.api._http.orig_http_request.call_count, 1,
 139                            "client got the right response without retrying")
 140         self.assertEqual(sleep.call_args_list,
 141                          [mock.call(RETRY_DELAY_INITIAL)])
 142
 143     @mock.patch('time.sleep')
 144     def test_socket_error_retry_delay(self, sleep):
 145         self.api._http.orig_http_request.side_effect = socket.error('mock')
 146         self.api._http._retry_count = 3
 147         with self.assertRaises(socket.error):
 148             self.api.users().current().execute()
 149         self.assertEqual(self.api._http.orig_http_request.call_count, 4)
 150         self.assertEqual(sleep.call_args_list, [
 151             mock.call(RETRY_DELAY_INITIAL),
 152             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
 153             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
 154         ])
 155
 156     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 157     def test_close_old_connections_non_retryable(self, sleep):
 158         self._test_connection_close(expect=1)
 159
 160     @mock.patch('time.time', side_effect=itertools.count())
 161     def test_no_close_fresh_connections_non_retryable(self, sleep):
 162         self._test_connection_close(expect=0)
 163
 164     @mock.patch('time.time', side_effect=itertools.count())
 165     def test_override_max_idle_time(self, sleep):
 166         self.api._http._max_keepalive_idle = 0
 167         self._test_connection_close(expect=1)
 168
 169     def _test_connection_close(self, expect=0):
 170         # Do two POST requests. The second one must close all
 171         # connections +expect+ times.
 172         self.api.users().create(body={}).execute()
 173         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 174         self.api._http.connections = mock_conns.copy()
 175         self.api.users().create(body={}).execute()
 176         for c in mock_conns.values():
 177             self.assertEqual(c.close.call_count, expect)
 178
 179     @mock.patch('time.sleep')
 180     def test_socket_error_no_retry_post(self, sleep):
 181         self.api._http.orig_http_request.side_effect = (
 182             socket.error('mock error'),
 183             self.request_success,
 184         )
 185         with self.assertRaises(socket.error):
 186             self.api.users().create(body={}).execute()
 187         self.assertEqual(self.api._http.orig_http_request.call_count, 1,
 188                          "client should try non-retryable method exactly once")
 189         self.assertEqual(sleep.call_args_list, [])
 190
 191
 192 if __name__ == '__main__':
 193     unittest.main()