sdk/python/tests/test_api.py

   1 #!/usr/bin/env python
   2
   3 import arvados
   4 import collections
   5 import httplib2
   6 import itertools
   7 import json
   8 import mimetypes
   9 import os
  10 import socket
  11 import string
  12 import unittest
  13
  14 import mock
  15 import run_test_server
  16
  17 from apiclient import errors as apiclient_errors
  18 from apiclient import http as apiclient_http
  19 from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
  20 from arvados_testutil import fake_httplib2_response, queue_with
  21
  22 if not mimetypes.inited:
  23     mimetypes.init()
  24
  25 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  26     MAIN_SERVER = {}
  27     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  28
  29     def api_error_response(self, code, *errors):
  30         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  31                 json.dumps({'errors': errors,
  32                             'error_token': '1234567890+12345678'}))
  33
  34     def test_new_api_objects_with_cache(self):
  35         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  36         self.assertIsNot(*clients)
  37
  38     def test_empty_list(self):
  39         answer = arvados.api('v1').humans().list(
  40             filters=[['uuid', 'is', None]]).execute()
  41         self.assertEqual(answer['items_available'], len(answer['items']))
  42
  43     def test_nonempty_list(self):
  44         answer = arvados.api('v1').collections().list().execute()
  45         self.assertNotEqual(0, answer['items_available'])
  46         self.assertNotEqual(0, len(answer['items']))
  47
  48     def test_timestamp_inequality_filter(self):
  49         api = arvados.api('v1')
  50         new_item = api.specimens().create(body={}).execute()
  51         for operator, should_include in [
  52                 ['<', False], ['>', False],
  53                 ['<=', True], ['>=', True], ['=', True]]:
  54             response = api.specimens().list(filters=[
  55                 ['created_at', operator, new_item['created_at']],
  56                 # Also filter by uuid to ensure (if it matches) it's on page 0
  57                 ['uuid', '=', new_item['uuid']]]).execute()
  58             uuids = [item['uuid'] for item in response['items']]
  59             did_include = new_item['uuid'] in uuids
  60             self.assertEqual(
  61                 did_include, should_include,
  62                 "'%s %s' filter should%s have matched '%s'" % (
  63                     operator, new_item['created_at'],
  64                     ('' if should_include else ' not'),
  65                     new_item['created_at']))
  66
  67     def test_exceptions_include_errors(self):
  68         mock_responses = {
  69             'arvados.humans.get': self.api_error_response(
  70                 422, "Bad UUID format", "Bad output format"),
  71             }
  72         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  73         api = arvados.api('v1', requestBuilder=req_builder)
  74         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  75             api.humans().get(uuid='xyz-xyz-abcdef').execute()
  76         err_s = str(err_ctx.exception)
  77         for msg in ["Bad UUID format", "Bad output format"]:
  78             self.assertIn(msg, err_s)
  79
  80     def test_exceptions_without_errors_have_basic_info(self):
  81         mock_responses = {
  82             'arvados.humans.delete': (
  83                 fake_httplib2_response(500, **self.ERROR_HEADERS),
  84                 "")
  85             }
  86         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
  87         api = arvados.api('v1', requestBuilder=req_builder)
  88         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
  89             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
  90         self.assertIn("500", str(err_ctx.exception))
  91
  92     def test_request_too_large(self):
  93         api = arvados.api('v1')
  94         maxsize = api._rootDesc.get('maxRequestSize', 0)
  95         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
  96             text = "X" * maxsize
  97             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
  98
  99     def test_ordered_json_model(self):
 100         mock_responses = {
 101             'arvados.humans.get': (None, json.dumps(collections.OrderedDict(
 102                         (c, int(c, 16)) for c in string.hexdigits))),
 103             }
 104         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 105         api = arvados.api('v1',
 106                           requestBuilder=req_builder, model=OrderedJsonModel())
 107         result = api.humans().get(uuid='test').execute()
 108         self.assertEqual(string.hexdigits, ''.join(result.keys()))
 109
 110
 111 class RetryREST(unittest.TestCase):
 112     def setUp(self):
 113         self.api = arvados.api('v1')
 114         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 115                         "test doesn't know how to intercept HTTP requests")
 116         self.mock_response = {'user': 'person'}
 117         self.request_success = (fake_httplib2_response(200),
 118                                 json.dumps(self.mock_response))
 119         self.api._http.orig_http_request = mock.MagicMock()
 120         # All requests succeed by default. Tests override as needed.
 121         self.api._http.orig_http_request.return_value = self.request_success
 122
 123     @mock.patch('time.sleep')
 124     def test_socket_error_retry_get(self, sleep):
 125         self.api._http.orig_http_request.side_effect = (
 126             socket.error('mock error'),
 127             self.request_success,
 128         )
 129         self.assertEqual(self.api.users().current().execute(),
 130                          self.mock_response)
 131         self.assertGreater(self.api._http.orig_http_request.call_count, 1,
 132                            "client got the right response without retrying")
 133         self.assertEqual(sleep.call_args_list,
 134                          [mock.call(RETRY_DELAY_INITIAL)])
 135
 136     @mock.patch('time.sleep')
 137     def test_socket_error_retry_delay(self, sleep):
 138         self.api._http.orig_http_request.side_effect = socket.error('mock')
 139         self.api._http._retry_count = 3
 140         with self.assertRaises(socket.error):
 141             self.api.users().current().execute()
 142         self.assertEqual(self.api._http.orig_http_request.call_count, 4)
 143         self.assertEqual(sleep.call_args_list, [
 144             mock.call(RETRY_DELAY_INITIAL),
 145             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF),
 146             mock.call(RETRY_DELAY_INITIAL * RETRY_DELAY_BACKOFF**2),
 147         ])
 148
 149     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 150     def test_close_old_connections_non_retryable(self, sleep):
 151         self._test_connection_close(expect=1)
 152
 153     @mock.patch('time.time', side_effect=itertools.count())
 154     def test_no_close_fresh_connections_non_retryable(self, sleep):
 155         self._test_connection_close(expect=0)
 156
 157     @mock.patch('time.time', side_effect=itertools.count())
 158     def test_override_max_idle_time(self, sleep):
 159         self.api._http._max_keepalive_idle = 0
 160         self._test_connection_close(expect=1)
 161
 162     def _test_connection_close(self, expect=0):
 163         # Do two POST requests. The second one must close all
 164         # connections +expect+ times.
 165         self.api.users().create(body={}).execute()
 166         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 167         self.api._http.connections = mock_conns.copy()
 168         self.api.users().create(body={}).execute()
 169         for c in mock_conns.itervalues():
 170             self.assertEqual(c.close.call_count, expect)
 171
 172     @mock.patch('time.sleep')
 173     def test_socket_error_no_retry_post(self, sleep):
 174         self.api._http.orig_http_request.side_effect = (
 175             socket.error('mock error'),
 176             self.request_success,
 177         )
 178         with self.assertRaises(socket.error):
 179             self.api.users().create(body={}).execute()
 180         self.assertEqual(self.api._http.orig_http_request.call_count, 1,
 181                          "client should try non-retryable method exactly once")
 182         self.assertEqual(sleep.call_args_list, [])
 183
 184
 185 if __name__ == '__main__':
 186     unittest.main()