sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import str
   7 from builtins import range
   8 import arvados
   9 import collections
  10 import contextlib
  11 import httplib2
  12 import itertools
  13 import json
  14 import mimetypes
  15 import os
  16 import socket
  17 import string
  18 import sys
  19 import unittest
  20 import urllib.parse as urlparse
  21
  22 import mock
  23 from . import run_test_server
  24
  25 from apiclient import errors as apiclient_errors
  26 from apiclient import http as apiclient_http
  27 from arvados.api import (
  28     api_client,
  29     normalize_api_kwargs,
  30     api_kwargs_from_config,
  31     OrderedJsonModel,
  32 )
  33 from .arvados_testutil import fake_httplib2_response, mock_api_responses, queue_with
  34
  35 if not mimetypes.inited:
  36     mimetypes.init()
  37
  38 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  39     MAIN_SERVER = {}
  40     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  41     RETRIED_4XX = frozenset([408, 409, 423])
  42
  43     def api_error_response(self, code, *errors):
  44         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  45                 json.dumps({'errors': errors,
  46                             'error_token': '1234567890+12345678'}).encode())
  47
  48     def _config_from_environ(self):
  49         return {
  50             key: value
  51             for key, value in os.environ.items()
  52             if key.startswith('ARVADOS_API_')
  53         }
  54
  55     def _discoveryServiceUrl(
  56             self,
  57             host=None,
  58             path='/discovery/v1/apis/{api}/{apiVersion}/rest',
  59             scheme='https',
  60     ):
  61         if host is None:
  62             host = os.environ['ARVADOS_API_HOST']
  63         return urlparse.urlunsplit((scheme, host, path, None, None))
  64
  65     def test_new_api_objects_with_cache(self):
  66         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  67         self.assertIsNot(*clients)
  68
  69     def test_empty_list(self):
  70         answer = arvados.api('v1').humans().list(
  71             filters=[['uuid', '=', None]]).execute()
  72         self.assertEqual(answer['items_available'], len(answer['items']))
  73
  74     def test_nonempty_list(self):
  75         answer = arvados.api('v1').collections().list().execute()
  76         self.assertNotEqual(0, answer['items_available'])
  77         self.assertNotEqual(0, len(answer['items']))
  78
  79     def test_timestamp_inequality_filter(self):
  80         api = arvados.api('v1')
  81         new_item = api.specimens().create(body={}).execute()
  82         for operator, should_include in [
  83                 ['<', False], ['>', False],
  84                 ['<=', True], ['>=', True], ['=', True]]:
  85             response = api.specimens().list(filters=[
  86                 ['created_at', operator, new_item['created_at']],
  87                 # Also filter by uuid to ensure (if it matches) it's on page 0
  88                 ['uuid', '=', new_item['uuid']]]).execute()
  89             uuids = [item['uuid'] for item in response['items']]
  90             did_include = new_item['uuid'] in uuids
  91             self.assertEqual(
  92                 did_include, should_include,
  93                 "'%s %s' filter should%s have matched '%s'" % (
  94                     operator, new_item['created_at'],
  95                     ('' if should_include else ' not'),
  96                     new_item['created_at']))
  97
  98     def test_exceptions_include_errors(self):
  99         mock_responses = {
 100             'arvados.humans.get': self.api_error_response(
 101                 422, "Bad UUID format", "Bad output format"),
 102             }
 103         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 104         api = arvados.api('v1', requestBuilder=req_builder)
 105         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 106             api.humans().get(uuid='xyz-xyz-abcdef').execute()
 107         err_s = str(err_ctx.exception)
 108         for msg in ["Bad UUID format", "Bad output format"]:
 109             self.assertIn(msg, err_s)
 110
 111     @mock.patch('time.sleep')
 112     def test_exceptions_include_request_id(self, sleep):
 113         api = arvados.api('v1')
 114         api.request_id='fake-request-id'
 115         api._http.orig_http_request = mock.MagicMock()
 116         api._http.orig_http_request.side_effect = socket.error('mock error')
 117         caught = None
 118         try:
 119             api.users().current().execute()
 120         except Exception as e:
 121             caught = e
 122         self.assertRegex(str(caught), r'fake-request-id')
 123
 124     def test_exceptions_without_errors_have_basic_info(self):
 125         mock_responses = {
 126             'arvados.humans.delete': (
 127                 fake_httplib2_response(500, **self.ERROR_HEADERS),
 128                 b"")
 129             }
 130         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 131         api = arvados.api('v1', requestBuilder=req_builder)
 132         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 133             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
 134         self.assertIn("500", str(err_ctx.exception))
 135
 136     def test_request_too_large(self):
 137         api = arvados.api('v1')
 138         maxsize = api._rootDesc.get('maxRequestSize', 0)
 139         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 140             text = "X" * maxsize
 141             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 142
 143     def test_default_request_timeout(self):
 144         api = arvados.api('v1')
 145         self.assertEqual(api._http.timeout, 300,
 146             "Default timeout value should be 300")
 147
 148     def test_custom_request_timeout(self):
 149         api = arvados.api('v1', timeout=1234)
 150         self.assertEqual(api._http.timeout, 1234,
 151             "Requested timeout value was 1234")
 152
 153     def test_4xx_retried(self):
 154         client = arvados.api('v1')
 155         for code in self.RETRIED_4XX:
 156             name = f'retried #{code}'
 157             with self.subTest(name), mock.patch('time.sleep'):
 158                 expected = {'username': name}
 159                 with mock_api_responses(
 160                         client,
 161                         json.dumps(expected),
 162                         [code, code, 200],
 163                         self.ERROR_HEADERS,
 164                         'orig_http_request',
 165                 ):
 166                     actual = client.users().current().execute()
 167                 self.assertEqual(actual, expected)
 168
 169     def test_4xx_not_retried(self):
 170         client = arvados.api('v1', num_retries=3)
 171         for code in [400, 401, 404, 422]:
 172             with self.subTest(f'error {code}'), mock.patch('time.sleep'):
 173                 with mock_api_responses(
 174                         client,
 175                         b'{}',
 176                         [code, 200],
 177                         self.ERROR_HEADERS,
 178                         'orig_http_request',
 179                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
 180                     client.users().current().execute()
 181                 response = exc_check.exception.args[0]
 182                 self.assertEqual(response.status, code)
 183                 self.assertEqual(response.get('status'), str(code))
 184
 185     def test_4xx_raised_after_retry_exhaustion(self):
 186         client = arvados.api('v1', num_retries=1)
 187         for code in self.RETRIED_4XX:
 188             with self.subTest(f'failed {code}'), mock.patch('time.sleep'):
 189                 with mock_api_responses(
 190                         client,
 191                         b'{}',
 192                         [code, code, code, 200],
 193                         self.ERROR_HEADERS,
 194                         'orig_http_request',
 195                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
 196                     client.users().current().execute()
 197                 response = exc_check.exception.args[0]
 198                 self.assertEqual(response.status, code)
 199                 self.assertEqual(response.get('status'), str(code))
 200
 201     def test_ordered_json_model(self):
 202         mock_responses = {
 203             'arvados.humans.get': (
 204                 None,
 205                 json.dumps(collections.OrderedDict(
 206                     (c, int(c, 16)) for c in string.hexdigits
 207                 )).encode(),
 208             ),
 209         }
 210         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 211         api = arvados.api('v1',
 212                           requestBuilder=req_builder, model=OrderedJsonModel())
 213         result = api.humans().get(uuid='test').execute()
 214         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 215
 216     def test_api_is_threadsafe(self):
 217         api_kwargs = {
 218             'host': os.environ['ARVADOS_API_HOST'],
 219             'token': os.environ['ARVADOS_API_TOKEN'],
 220             'insecure': True,
 221         }
 222         config_kwargs = {'apiconfig': os.environ}
 223         for api_constructor, kwargs in [
 224                 (arvados.api, {}),
 225                 (arvados.api, api_kwargs),
 226                 (arvados.api_from_config, {}),
 227                 (arvados.api_from_config, config_kwargs),
 228         ]:
 229             sub_kwargs = "kwargs" if kwargs else "no kwargs"
 230             with self.subTest(f"{api_constructor.__name__} with {sub_kwargs}"):
 231                 api_client = api_constructor('v1', **kwargs)
 232                 self.assertTrue(hasattr(api_client, 'localapi'),
 233                                 f"client missing localapi method")
 234                 self.assertTrue(hasattr(api_client, 'keep'),
 235                                 f"client missing keep attribute")
 236
 237     def test_api_host_constructor(self):
 238         cache = True
 239         insecure = True
 240         client = arvados.api(
 241             'v1',
 242             cache,
 243             os.environ['ARVADOS_API_HOST'],
 244             os.environ['ARVADOS_API_TOKEN'],
 245             insecure,
 246         )
 247         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 248                          "client constructed with incorrect token")
 249
 250     def test_api_url_constructor(self):
 251         client = arvados.api(
 252             'v1',
 253             discoveryServiceUrl=self._discoveryServiceUrl(),
 254             token=os.environ['ARVADOS_API_TOKEN'],
 255             insecure=True,
 256         )
 257         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 258                          "client constructed with incorrect token")
 259
 260     def test_api_bad_args(self):
 261         all_kwargs = {
 262             'host': os.environ['ARVADOS_API_HOST'],
 263             'token': os.environ['ARVADOS_API_TOKEN'],
 264             'discoveryServiceUrl': self._discoveryServiceUrl(),
 265         }
 266         for use_keys in [
 267                 # Passing only a single key is missing required info
 268                 *([key] for key in all_kwargs.keys()),
 269                 # Passing all keys is a conflict
 270                 list(all_kwargs.keys()),
 271         ]:
 272             kwargs = {key: all_kwargs[key] for key in use_keys}
 273             kwargs_list = ', '.join(use_keys)
 274             with self.subTest(f"calling arvados.api with {kwargs_list} fails"), \
 275                  self.assertRaises(ValueError):
 276                 arvados.api('v1', insecure=True, **kwargs)
 277
 278     def test_api_bad_url(self):
 279         for bad_kwargs in [
 280                 {'discoveryServiceUrl': self._discoveryServiceUrl() + '/BadTestURL'},
 281                 {'version': 'BadTestVersion', 'host': os.environ['ARVADOS_API_HOST']},
 282         ]:
 283             bad_key = next(iter(bad_kwargs))
 284             with self.subTest(f"api fails with bad {bad_key}"), \
 285                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 286                 arvados.api(**bad_kwargs, token='test_api_bad_url', insecure=True)
 287
 288     def test_normalize_api_good_args(self):
 289         for version, discoveryServiceUrl, host in [
 290                 ('Test1', None, os.environ['ARVADOS_API_HOST']),
 291                 (None, self._discoveryServiceUrl(), None)
 292         ]:
 293             argname = 'discoveryServiceUrl' if host is None else 'host'
 294             with self.subTest(f"normalize_api_kwargs with {argname}"):
 295                 actual = normalize_api_kwargs(
 296                     version,
 297                     discoveryServiceUrl,
 298                     host,
 299                     os.environ['ARVADOS_API_TOKEN'],
 300                     insecure=True,
 301                 )
 302                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 303                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 304                 self.assertEqual(actual['version'], version or 'v1')
 305                 self.assertTrue(actual['insecure'])
 306                 self.assertNotIn('host', actual)
 307
 308     def test_normalize_api_bad_args(self):
 309         all_args = (
 310             self._discoveryServiceUrl(),
 311             os.environ['ARVADOS_API_HOST'],
 312             os.environ['ARVADOS_API_TOKEN'],
 313         )
 314         for arg_index, arg_value in enumerate(all_args):
 315             args = [None] * len(all_args)
 316             args[arg_index] = arg_value
 317             with self.subTest(f"normalize_api_kwargs with only arg #{arg_index + 1}"), \
 318                  self.assertRaises(ValueError):
 319                 normalize_api_kwargs('v1', *args)
 320         with self.subTest("normalize_api_kwargs with discoveryServiceUrl and host"), \
 321              self.assertRaises(ValueError):
 322             normalize_api_kwargs('v1', *all_args)
 323
 324     def test_api_from_config_default(self):
 325         client = arvados.api_from_config('v1')
 326         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 327                          "client constructed with incorrect token")
 328
 329     def test_api_from_config_explicit(self):
 330         config = self._config_from_environ()
 331         client = arvados.api_from_config('v1', config)
 332         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 333                          "client constructed with incorrect token")
 334
 335     def test_api_from_bad_config(self):
 336         base_config = self._config_from_environ()
 337         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 338             with self.subTest(f"api_from_config without {del_key} fails"), \
 339                  self.assertRaises(ValueError):
 340                 config = dict(base_config)
 341                 del config[del_key]
 342                 arvados.api_from_config('v1', config)
 343
 344     def test_api_kwargs_from_good_config(self):
 345         for config in [None, self._config_from_environ()]:
 346             conf_type = 'default' if config is None else 'passed'
 347             with self.subTest(f"api_kwargs_from_config with {conf_type} config"):
 348                 version = 'Test1' if config else None
 349                 actual = api_kwargs_from_config(version, config)
 350                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 351                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 352                 self.assertEqual(actual['version'], version or 'v1')
 353                 self.assertTrue(actual['insecure'])
 354                 self.assertNotIn('host', actual)
 355
 356     def test_api_kwargs_from_bad_config(self):
 357         base_config = self._config_from_environ()
 358         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 359             with self.subTest(f"api_kwargs_from_config without {del_key} fails"), \
 360                  self.assertRaises(ValueError):
 361                 config = dict(base_config)
 362                 del config[del_key]
 363                 api_kwargs_from_config('v1', config)
 364
 365     def test_api_client_constructor(self):
 366         client = api_client(
 367             'v1',
 368             self._discoveryServiceUrl(),
 369             os.environ['ARVADOS_API_TOKEN'],
 370             insecure=True,
 371         )
 372         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 373                          "client constructed with incorrect token")
 374         self.assertFalse(
 375             hasattr(client, 'localapi'),
 376             "client has localapi method when it should not be thread-safe",
 377         )
 378
 379     def test_api_client_bad_url(self):
 380         all_args = ('v1', self._discoveryServiceUrl(), 'test_api_client_bad_url')
 381         for arg_index, arg_value in [
 382                 (0, 'BadTestVersion'),
 383                 (1, all_args[1] + '/BadTestURL'),
 384         ]:
 385             with self.subTest(f"api_client fails with {arg_index}={arg_value!r}"), \
 386                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 387                 args = list(all_args)
 388                 args[arg_index] = arg_value
 389                 api_client(*args, insecure=True)
 390
 391
 392 class ConstructNumRetriesTestCase(unittest.TestCase):
 393     @staticmethod
 394     def _fake_retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs):
 395         return http.request(uri, method, *args, **kwargs)
 396
 397     @contextlib.contextmanager
 398     def patch_retry(self):
 399         # We have this dedicated context manager that goes through `sys.modules`
 400         # instead of just using `mock.patch` because of the unfortunate
 401         # `arvados.api` name collision.
 402         orig_func = sys.modules['arvados.api']._orig_retry_request
 403         expect_name = 'googleapiclient.http._retry_request'
 404         self.assertEqual(
 405             '{0.__module__}.{0.__name__}'.format(orig_func), expect_name,
 406             f"test setup problem: {expect_name} not at arvados.api._orig_retry_request",
 407         )
 408         retry_mock = mock.Mock(wraps=self._fake_retry_request)
 409         sys.modules['arvados.api']._orig_retry_request = retry_mock
 410         try:
 411             yield retry_mock
 412         finally:
 413             sys.modules['arvados.api']._orig_retry_request = orig_func
 414
 415     def _iter_num_retries(self, retry_mock):
 416         for call in retry_mock.call_args_list:
 417             try:
 418                 yield call.args[1]
 419             except IndexError:
 420                 yield call.kwargs['num_retries']
 421
 422     def test_default_num_retries(self):
 423         with self.patch_retry() as retry_mock:
 424             client = arvados.api('v1')
 425         actual = set(self._iter_num_retries(retry_mock))
 426         self.assertEqual(len(actual), 1)
 427         self.assertTrue(actual.pop() > 6, "num_retries lower than expected")
 428
 429     def _test_calls(self, init_arg, call_args, expected):
 430         with self.patch_retry() as retry_mock:
 431             client = arvados.api('v1', num_retries=init_arg)
 432             for num_retries in call_args:
 433                 client.users().current().execute(num_retries=num_retries)
 434         actual = self._iter_num_retries(retry_mock)
 435         # The constructor makes two requests with its num_retries argument:
 436         # one for the discovery document, and one for the config.
 437         self.assertEqual(next(actual, None), init_arg)
 438         self.assertEqual(next(actual, None), init_arg)
 439         self.assertEqual(list(actual), expected)
 440
 441     def test_discovery_num_retries(self):
 442         for num_retries in [0, 5, 55]:
 443             with self.subTest(f"num_retries={num_retries}"):
 444                 self._test_calls(num_retries, [], [])
 445
 446     def test_num_retries_called_le_init(self):
 447         for n in [6, 10]:
 448             with self.subTest(f"init_arg={n}"):
 449                 call_args = [n - 4, n - 2, n]
 450                 expected = [n] * 3
 451                 self._test_calls(n, call_args, expected)
 452
 453     def test_num_retries_called_ge_init(self):
 454         for n in [0, 10]:
 455             with self.subTest(f"init_arg={n}"):
 456                 call_args = [n, n + 4, n + 8]
 457                 self._test_calls(n, call_args, call_args)
 458
 459     def test_num_retries_called_mixed(self):
 460         self._test_calls(5, [2, 6, 4, 8], [5, 6, 5, 8])
 461
 462
 463 class PreCloseSocketTestCase(unittest.TestCase):
 464     def setUp(self):
 465         self.api = arvados.api('v1')
 466         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 467                         "test doesn't know how to intercept HTTP requests")
 468         self.mock_response = {'user': 'person'}
 469         self.request_success = (fake_httplib2_response(200),
 470                                 json.dumps(self.mock_response))
 471         self.api._http.orig_http_request = mock.MagicMock()
 472         # All requests succeed by default. Tests override as needed.
 473         self.api._http.orig_http_request.return_value = self.request_success
 474
 475     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 476     def test_close_old_connections_non_retryable(self, sleep):
 477         self._test_connection_close(expect=1)
 478
 479     @mock.patch('time.time', side_effect=itertools.count())
 480     def test_no_close_fresh_connections_non_retryable(self, sleep):
 481         self._test_connection_close(expect=0)
 482
 483     @mock.patch('time.time', side_effect=itertools.count())
 484     def test_override_max_idle_time(self, sleep):
 485         self.api._http._max_keepalive_idle = 0
 486         self._test_connection_close(expect=1)
 487
 488     def _test_connection_close(self, expect=0):
 489         # Do two POST requests. The second one must close all
 490         # connections +expect+ times.
 491         self.api.users().create(body={}).execute()
 492         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 493         self.api._http.connections = mock_conns.copy()
 494         self.api.users().create(body={}).execute()
 495         for c in mock_conns.values():
 496             self.assertEqual(c.close.call_count, expect)
 497
 498
 499 if __name__ == '__main__':
 500     unittest.main()