sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import str
   7 from builtins import range
   8 import arvados
   9 import collections
  10 import contextlib
  11 import httplib2
  12 import itertools
  13 import json
  14 import mimetypes
  15 import os
  16 import socket
  17 import string
  18 import sys
  19 import unittest
  20 import urllib.parse as urlparse
  21
  22 import mock
  23 from . import run_test_server
  24
  25 from apiclient import errors as apiclient_errors
  26 from apiclient import http as apiclient_http
  27 from arvados.api import (
  28     api_client,
  29     normalize_api_kwargs,
  30     api_kwargs_from_config,
  31     OrderedJsonModel,
  32 )
  33 from .arvados_testutil import fake_httplib2_response, mock_api_responses, queue_with
  34
  35 if not mimetypes.inited:
  36     mimetypes.init()
  37
  38 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  39     MAIN_SERVER = {}
  40     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  41     RETRIED_4XX = frozenset([408, 409, 423])
  42
  43     def api_error_response(self, code, *errors):
  44         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  45                 json.dumps({'errors': errors,
  46                             'error_token': '1234567890+12345678'}).encode())
  47
  48     def _config_from_environ(self):
  49         return {
  50             key: value
  51             for key, value in os.environ.items()
  52             if key.startswith('ARVADOS_API_')
  53         }
  54
  55     def _discoveryServiceUrl(
  56             self,
  57             host=None,
  58             path='/discovery/v1/apis/{api}/{apiVersion}/rest',
  59             scheme='https',
  60     ):
  61         if host is None:
  62             host = os.environ['ARVADOS_API_HOST']
  63         return urlparse.urlunsplit((scheme, host, path, None, None))
  64
  65     def test_new_api_objects_with_cache(self):
  66         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  67         self.assertIsNot(*clients)
  68
  69     def test_empty_list(self):
  70         answer = arvados.api('v1').humans().list(
  71             filters=[['uuid', '=', None]]).execute()
  72         self.assertEqual(answer['items_available'], len(answer['items']))
  73
  74     def test_nonempty_list(self):
  75         answer = arvados.api('v1').collections().list().execute()
  76         self.assertNotEqual(0, answer['items_available'])
  77         self.assertNotEqual(0, len(answer['items']))
  78
  79     def test_timestamp_inequality_filter(self):
  80         api = arvados.api('v1')
  81         new_item = api.specimens().create(body={}).execute()
  82         for operator, should_include in [
  83                 ['<', False], ['>', False],
  84                 ['<=', True], ['>=', True], ['=', True]]:
  85             response = api.specimens().list(filters=[
  86                 ['created_at', operator, new_item['created_at']],
  87                 # Also filter by uuid to ensure (if it matches) it's on page 0
  88                 ['uuid', '=', new_item['uuid']]]).execute()
  89             uuids = [item['uuid'] for item in response['items']]
  90             did_include = new_item['uuid'] in uuids
  91             self.assertEqual(
  92                 did_include, should_include,
  93                 "'%s %s' filter should%s have matched '%s'" % (
  94                     operator, new_item['created_at'],
  95                     ('' if should_include else ' not'),
  96                     new_item['created_at']))
  97
  98     def test_exceptions_include_errors(self):
  99         mock_responses = {
 100             'arvados.humans.get': self.api_error_response(
 101                 422, "Bad UUID format", "Bad output format"),
 102             }
 103         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 104         api = arvados.api('v1', requestBuilder=req_builder)
 105         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 106             api.humans().get(uuid='xyz-xyz-abcdef').execute()
 107         err_s = str(err_ctx.exception)
 108         for msg in ["Bad UUID format", "Bad output format"]:
 109             self.assertIn(msg, err_s)
 110
 111     @mock.patch('time.sleep')
 112     def test_exceptions_include_request_id(self, sleep):
 113         api = arvados.api('v1')
 114         api.request_id='fake-request-id'
 115         api._http.orig_http_request = mock.MagicMock()
 116         api._http.orig_http_request.side_effect = socket.error('mock error')
 117         caught = None
 118         try:
 119             api.users().current().execute()
 120         except Exception as e:
 121             caught = e
 122         self.assertRegex(str(caught), r'fake-request-id')
 123
 124     def test_exceptions_without_errors_have_basic_info(self):
 125         mock_responses = {
 126             'arvados.humans.delete': (
 127                 fake_httplib2_response(500, **self.ERROR_HEADERS),
 128                 b"")
 129             }
 130         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 131         api = arvados.api('v1', requestBuilder=req_builder)
 132         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 133             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
 134         self.assertIn("500", str(err_ctx.exception))
 135
 136     def test_request_too_large(self):
 137         api = arvados.api('v1')
 138         maxsize = api._rootDesc.get('maxRequestSize', 0)
 139         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 140             text = "X" * maxsize
 141             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 142
 143     def test_default_request_timeout(self):
 144         api = arvados.api('v1')
 145         self.assertEqual(api._http.timeout, 300,
 146             "Default timeout value should be 300")
 147
 148     def test_custom_request_timeout(self):
 149         api = arvados.api('v1', timeout=1234)
 150         self.assertEqual(api._http.timeout, 1234,
 151             "Requested timeout value was 1234")
 152
 153     def test_4xx_retried(self):
 154         client = arvados.api('v1')
 155         for code in self.RETRIED_4XX:
 156             name = f'retried #{code}'
 157             with self.subTest(name), mock.patch('time.sleep'):
 158                 expected = {'username': name}
 159                 with mock_api_responses(
 160                         client,
 161                         json.dumps(expected),
 162                         [code, code, 200],
 163                         self.ERROR_HEADERS,
 164                         'orig_http_request',
 165                 ):
 166                     actual = client.users().current().execute()
 167                 self.assertEqual(actual, expected)
 168
 169     def test_4xx_not_retried(self):
 170         client = arvados.api('v1', num_retries=3)
 171         # Note that googleapiclient does retry 403 *if* the response JSON
 172         # includes flags that say the request was denied by rate limiting.
 173         # An empty JSON response like we use here should not be retried.
 174         for code in [400, 401, 403, 404, 422]:
 175             with self.subTest(f'error {code}'), mock.patch('time.sleep'):
 176                 with mock_api_responses(
 177                         client,
 178                         b'{}',
 179                         [code, 200],
 180                         self.ERROR_HEADERS,
 181                         'orig_http_request',
 182                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
 183                     client.users().current().execute()
 184                 response = exc_check.exception.args[0]
 185                 self.assertEqual(response.status, code)
 186                 self.assertEqual(response.get('status'), str(code))
 187
 188     def test_4xx_raised_after_retry_exhaustion(self):
 189         client = arvados.api('v1', num_retries=1)
 190         for code in self.RETRIED_4XX:
 191             with self.subTest(f'failed {code}'), mock.patch('time.sleep'):
 192                 with mock_api_responses(
 193                         client,
 194                         b'{}',
 195                         [code, code, code, 200],
 196                         self.ERROR_HEADERS,
 197                         'orig_http_request',
 198                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
 199                     client.users().current().execute()
 200                 response = exc_check.exception.args[0]
 201                 self.assertEqual(response.status, code)
 202                 self.assertEqual(response.get('status'), str(code))
 203
 204     def test_ordered_json_model(self):
 205         mock_responses = {
 206             'arvados.humans.get': (
 207                 None,
 208                 json.dumps(collections.OrderedDict(
 209                     (c, int(c, 16)) for c in string.hexdigits
 210                 )).encode(),
 211             ),
 212         }
 213         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 214         api = arvados.api('v1',
 215                           requestBuilder=req_builder, model=OrderedJsonModel())
 216         result = api.humans().get(uuid='test').execute()
 217         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 218
 219     def test_api_is_threadsafe(self):
 220         api_kwargs = {
 221             'host': os.environ['ARVADOS_API_HOST'],
 222             'token': os.environ['ARVADOS_API_TOKEN'],
 223             'insecure': True,
 224         }
 225         config_kwargs = {'apiconfig': os.environ}
 226         for api_constructor, kwargs in [
 227                 (arvados.api, {}),
 228                 (arvados.api, api_kwargs),
 229                 (arvados.api_from_config, {}),
 230                 (arvados.api_from_config, config_kwargs),
 231         ]:
 232             sub_kwargs = "kwargs" if kwargs else "no kwargs"
 233             with self.subTest(f"{api_constructor.__name__} with {sub_kwargs}"):
 234                 api_client = api_constructor('v1', **kwargs)
 235                 self.assertTrue(hasattr(api_client, 'localapi'),
 236                                 f"client missing localapi method")
 237                 self.assertTrue(hasattr(api_client, 'keep'),
 238                                 f"client missing keep attribute")
 239
 240     def test_api_host_constructor(self):
 241         cache = True
 242         insecure = True
 243         client = arvados.api(
 244             'v1',
 245             cache,
 246             os.environ['ARVADOS_API_HOST'],
 247             os.environ['ARVADOS_API_TOKEN'],
 248             insecure,
 249         )
 250         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 251                          "client constructed with incorrect token")
 252
 253     def test_api_url_constructor(self):
 254         client = arvados.api(
 255             'v1',
 256             discoveryServiceUrl=self._discoveryServiceUrl(),
 257             token=os.environ['ARVADOS_API_TOKEN'],
 258             insecure=True,
 259         )
 260         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 261                          "client constructed with incorrect token")
 262
 263     def test_api_bad_args(self):
 264         all_kwargs = {
 265             'host': os.environ['ARVADOS_API_HOST'],
 266             'token': os.environ['ARVADOS_API_TOKEN'],
 267             'discoveryServiceUrl': self._discoveryServiceUrl(),
 268         }
 269         for use_keys in [
 270                 # Passing only a single key is missing required info
 271                 *([key] for key in all_kwargs.keys()),
 272                 # Passing all keys is a conflict
 273                 list(all_kwargs.keys()),
 274         ]:
 275             kwargs = {key: all_kwargs[key] for key in use_keys}
 276             kwargs_list = ', '.join(use_keys)
 277             with self.subTest(f"calling arvados.api with {kwargs_list} fails"), \
 278                  self.assertRaises(ValueError):
 279                 arvados.api('v1', insecure=True, **kwargs)
 280
 281     def test_api_bad_url(self):
 282         for bad_kwargs in [
 283                 {'discoveryServiceUrl': self._discoveryServiceUrl() + '/BadTestURL'},
 284                 {'version': 'BadTestVersion', 'host': os.environ['ARVADOS_API_HOST']},
 285         ]:
 286             bad_key = next(iter(bad_kwargs))
 287             with self.subTest(f"api fails with bad {bad_key}"), \
 288                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 289                 arvados.api(**bad_kwargs, token='test_api_bad_url', insecure=True)
 290
 291     def test_normalize_api_good_args(self):
 292         for version, discoveryServiceUrl, host in [
 293                 ('Test1', None, os.environ['ARVADOS_API_HOST']),
 294                 (None, self._discoveryServiceUrl(), None)
 295         ]:
 296             argname = 'discoveryServiceUrl' if host is None else 'host'
 297             with self.subTest(f"normalize_api_kwargs with {argname}"):
 298                 actual = normalize_api_kwargs(
 299                     version,
 300                     discoveryServiceUrl,
 301                     host,
 302                     os.environ['ARVADOS_API_TOKEN'],
 303                     insecure=True,
 304                 )
 305                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 306                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 307                 self.assertEqual(actual['version'], version or 'v1')
 308                 self.assertTrue(actual['insecure'])
 309                 self.assertNotIn('host', actual)
 310
 311     def test_normalize_api_bad_args(self):
 312         all_args = (
 313             self._discoveryServiceUrl(),
 314             os.environ['ARVADOS_API_HOST'],
 315             os.environ['ARVADOS_API_TOKEN'],
 316         )
 317         for arg_index, arg_value in enumerate(all_args):
 318             args = [None] * len(all_args)
 319             args[arg_index] = arg_value
 320             with self.subTest(f"normalize_api_kwargs with only arg #{arg_index + 1}"), \
 321                  self.assertRaises(ValueError):
 322                 normalize_api_kwargs('v1', *args)
 323         with self.subTest("normalize_api_kwargs with discoveryServiceUrl and host"), \
 324              self.assertRaises(ValueError):
 325             normalize_api_kwargs('v1', *all_args)
 326
 327     def test_api_from_config_default(self):
 328         client = arvados.api_from_config('v1')
 329         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 330                          "client constructed with incorrect token")
 331
 332     def test_api_from_config_explicit(self):
 333         config = self._config_from_environ()
 334         client = arvados.api_from_config('v1', config)
 335         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 336                          "client constructed with incorrect token")
 337
 338     def test_api_from_bad_config(self):
 339         base_config = self._config_from_environ()
 340         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 341             with self.subTest(f"api_from_config without {del_key} fails"), \
 342                  self.assertRaises(ValueError):
 343                 config = dict(base_config)
 344                 del config[del_key]
 345                 arvados.api_from_config('v1', config)
 346
 347     def test_api_kwargs_from_good_config(self):
 348         for config in [None, self._config_from_environ()]:
 349             conf_type = 'default' if config is None else 'passed'
 350             with self.subTest(f"api_kwargs_from_config with {conf_type} config"):
 351                 version = 'Test1' if config else None
 352                 actual = api_kwargs_from_config(version, config)
 353                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 354                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 355                 self.assertEqual(actual['version'], version or 'v1')
 356                 self.assertTrue(actual['insecure'])
 357                 self.assertNotIn('host', actual)
 358
 359     def test_api_kwargs_from_bad_config(self):
 360         base_config = self._config_from_environ()
 361         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 362             with self.subTest(f"api_kwargs_from_config without {del_key} fails"), \
 363                  self.assertRaises(ValueError):
 364                 config = dict(base_config)
 365                 del config[del_key]
 366                 api_kwargs_from_config('v1', config)
 367
 368     def test_api_client_constructor(self):
 369         client = api_client(
 370             'v1',
 371             self._discoveryServiceUrl(),
 372             os.environ['ARVADOS_API_TOKEN'],
 373             insecure=True,
 374         )
 375         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 376                          "client constructed with incorrect token")
 377         self.assertFalse(
 378             hasattr(client, 'localapi'),
 379             "client has localapi method when it should not be thread-safe",
 380         )
 381
 382     def test_api_client_bad_url(self):
 383         all_args = ('v1', self._discoveryServiceUrl(), 'test_api_client_bad_url')
 384         for arg_index, arg_value in [
 385                 (0, 'BadTestVersion'),
 386                 (1, all_args[1] + '/BadTestURL'),
 387         ]:
 388             with self.subTest(f"api_client fails with {arg_index}={arg_value!r}"), \
 389                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 390                 args = list(all_args)
 391                 args[arg_index] = arg_value
 392                 api_client(*args, insecure=True)
 393
 394
 395 class ConstructNumRetriesTestCase(unittest.TestCase):
 396     @staticmethod
 397     def _fake_retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs):
 398         return http.request(uri, method, *args, **kwargs)
 399
 400     @contextlib.contextmanager
 401     def patch_retry(self):
 402         # We have this dedicated context manager that goes through `sys.modules`
 403         # instead of just using `mock.patch` because of the unfortunate
 404         # `arvados.api` name collision.
 405         orig_func = sys.modules['arvados.api']._orig_retry_request
 406         expect_name = 'googleapiclient.http._retry_request'
 407         self.assertEqual(
 408             '{0.__module__}.{0.__name__}'.format(orig_func), expect_name,
 409             f"test setup problem: {expect_name} not at arvados.api._orig_retry_request",
 410         )
 411         retry_mock = mock.Mock(wraps=self._fake_retry_request)
 412         sys.modules['arvados.api']._orig_retry_request = retry_mock
 413         try:
 414             yield retry_mock
 415         finally:
 416             sys.modules['arvados.api']._orig_retry_request = orig_func
 417
 418     def _iter_num_retries(self, retry_mock):
 419         for call in retry_mock.call_args_list:
 420             try:
 421                 yield call.args[1]
 422             except IndexError:
 423                 yield call.kwargs['num_retries']
 424
 425     def test_default_num_retries(self):
 426         with self.patch_retry() as retry_mock:
 427             client = arvados.api('v1')
 428         actual = set(self._iter_num_retries(retry_mock))
 429         self.assertEqual(len(actual), 1)
 430         self.assertTrue(actual.pop() > 6, "num_retries lower than expected")
 431
 432     def _test_calls(self, init_arg, call_args, expected):
 433         with self.patch_retry() as retry_mock:
 434             client = arvados.api('v1', num_retries=init_arg)
 435             for num_retries in call_args:
 436                 client.users().current().execute(num_retries=num_retries)
 437         actual = self._iter_num_retries(retry_mock)
 438         # The constructor makes two requests with its num_retries argument:
 439         # one for the discovery document, and one for the config.
 440         self.assertEqual(next(actual, None), init_arg)
 441         self.assertEqual(next(actual, None), init_arg)
 442         self.assertEqual(list(actual), expected)
 443
 444     def test_discovery_num_retries(self):
 445         for num_retries in [0, 5, 55]:
 446             with self.subTest(f"num_retries={num_retries}"):
 447                 self._test_calls(num_retries, [], [])
 448
 449     def test_num_retries_called_le_init(self):
 450         for n in [6, 10]:
 451             with self.subTest(f"init_arg={n}"):
 452                 call_args = [n - 4, n - 2, n]
 453                 expected = [n] * 3
 454                 self._test_calls(n, call_args, expected)
 455
 456     def test_num_retries_called_ge_init(self):
 457         for n in [0, 10]:
 458             with self.subTest(f"init_arg={n}"):
 459                 call_args = [n, n + 4, n + 8]
 460                 self._test_calls(n, call_args, call_args)
 461
 462     def test_num_retries_called_mixed(self):
 463         self._test_calls(5, [2, 6, 4, 8], [5, 6, 5, 8])
 464
 465
 466 class PreCloseSocketTestCase(unittest.TestCase):
 467     def setUp(self):
 468         self.api = arvados.api('v1')
 469         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 470                         "test doesn't know how to intercept HTTP requests")
 471         self.mock_response = {'user': 'person'}
 472         self.request_success = (fake_httplib2_response(200),
 473                                 json.dumps(self.mock_response))
 474         self.api._http.orig_http_request = mock.MagicMock()
 475         # All requests succeed by default. Tests override as needed.
 476         self.api._http.orig_http_request.return_value = self.request_success
 477
 478     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 479     def test_close_old_connections_non_retryable(self, sleep):
 480         self._test_connection_close(expect=1)
 481
 482     @mock.patch('time.time', side_effect=itertools.count())
 483     def test_no_close_fresh_connections_non_retryable(self, sleep):
 484         self._test_connection_close(expect=0)
 485
 486     @mock.patch('time.time', side_effect=itertools.count())
 487     def test_override_max_idle_time(self, sleep):
 488         self.api._http._max_keepalive_idle = 0
 489         self._test_connection_close(expect=1)
 490
 491     def _test_connection_close(self, expect=0):
 492         # Do two POST requests. The second one must close all
 493         # connections +expect+ times.
 494         self.api.users().create(body={}).execute()
 495         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 496         self.api._http.connections = mock_conns.copy()
 497         self.api.users().create(body={}).execute()
 498         for c in mock_conns.values():
 499             self.assertEqual(c.close.call_count, expect)
 500
 501
 502 if __name__ == '__main__':
 503     unittest.main()