sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import str
   7 from builtins import range
   8 import arvados
   9 import collections
  10 import contextlib
  11 import httplib2
  12 import itertools
  13 import json
  14 import logging
  15 import mimetypes
  16 import os
  17 import socket
  18 import string
  19 import sys
  20 import unittest
  21 import urllib.parse as urlparse
  22
  23 import mock
  24 from . import run_test_server
  25
  26 from apiclient import errors as apiclient_errors
  27 from apiclient import http as apiclient_http
  28 from arvados.api import (
  29     api_client,
  30     normalize_api_kwargs,
  31     api_kwargs_from_config,
  32     OrderedJsonModel,
  33     _googleapiclient_log_lock,
  34 )
  35 from .arvados_testutil import fake_httplib2_response, mock_api_responses, queue_with
  36 import httplib2.error
  37
  38 if not mimetypes.inited:
  39     mimetypes.init()
  40
  41 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  42     MAIN_SERVER = {}
  43     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  44     RETRIED_4XX = frozenset([408, 409, 423])
  45
  46     def api_error_response(self, code, *errors):
  47         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  48                 json.dumps({'errors': errors,
  49                             'error_token': '1234567890+12345678'}).encode())
  50
  51     def _config_from_environ(self):
  52         return {
  53             key: value
  54             for key, value in os.environ.items()
  55             if key.startswith('ARVADOS_API_')
  56         }
  57
  58     def _discoveryServiceUrl(
  59             self,
  60             host=None,
  61             path='/discovery/v1/apis/{api}/{apiVersion}/rest',
  62             scheme='https',
  63     ):
  64         if host is None:
  65             host = os.environ['ARVADOS_API_HOST']
  66         return urlparse.urlunsplit((scheme, host, path, None, None))
  67
  68     def test_new_api_objects_with_cache(self):
  69         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  70         self.assertIsNot(*clients)
  71
  72     def test_empty_list(self):
  73         answer = arvados.api('v1').humans().list(
  74             filters=[['uuid', '=', None]]).execute()
  75         self.assertEqual(answer['items_available'], len(answer['items']))
  76
  77     def test_nonempty_list(self):
  78         answer = arvados.api('v1').collections().list().execute()
  79         self.assertNotEqual(0, answer['items_available'])
  80         self.assertNotEqual(0, len(answer['items']))
  81
  82     def test_timestamp_inequality_filter(self):
  83         api = arvados.api('v1')
  84         new_item = api.specimens().create(body={}).execute()
  85         for operator, should_include in [
  86                 ['<', False], ['>', False],
  87                 ['<=', True], ['>=', True], ['=', True]]:
  88             response = api.specimens().list(filters=[
  89                 ['created_at', operator, new_item['created_at']],
  90                 # Also filter by uuid to ensure (if it matches) it's on page 0
  91                 ['uuid', '=', new_item['uuid']]]).execute()
  92             uuids = [item['uuid'] for item in response['items']]
  93             did_include = new_item['uuid'] in uuids
  94             self.assertEqual(
  95                 did_include, should_include,
  96                 "'%s %s' filter should%s have matched '%s'" % (
  97                     operator, new_item['created_at'],
  98                     ('' if should_include else ' not'),
  99                     new_item['created_at']))
 100
 101     def test_exceptions_include_errors(self):
 102         mock_responses = {
 103             'arvados.humans.get': self.api_error_response(
 104                 422, "Bad UUID format", "Bad output format"),
 105             }
 106         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 107         api = arvados.api('v1', requestBuilder=req_builder)
 108         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 109             api.humans().get(uuid='xyz-xyz-abcdef').execute()
 110         err_s = str(err_ctx.exception)
 111         for msg in ["Bad UUID format", "Bad output format"]:
 112             self.assertIn(msg, err_s)
 113
 114     @mock.patch('time.sleep')
 115     def test_exceptions_include_request_id(self, sleep):
 116         api = arvados.api('v1')
 117         api.request_id='fake-request-id'
 118         api._http.orig_http_request = mock.MagicMock()
 119         api._http.orig_http_request.side_effect = socket.error('mock error')
 120         caught = None
 121         try:
 122             api.users().current().execute()
 123         except Exception as e:
 124             caught = e
 125         self.assertRegex(str(caught), r'fake-request-id')
 126
 127     def test_exceptions_without_errors_have_basic_info(self):
 128         mock_responses = {
 129             'arvados.humans.delete': (
 130                 fake_httplib2_response(500, **self.ERROR_HEADERS),
 131                 b"")
 132             }
 133         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 134         api = arvados.api('v1', requestBuilder=req_builder)
 135         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 136             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
 137         self.assertIn("500", str(err_ctx.exception))
 138
 139     def test_request_too_large(self):
 140         api = arvados.api('v1')
 141         maxsize = api._rootDesc.get('maxRequestSize', 0)
 142         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 143             text = "X" * maxsize
 144             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 145
 146     def test_default_request_timeout(self):
 147         api = arvados.api('v1')
 148         self.assertEqual(api._http.timeout, 300,
 149             "Default timeout value should be 300")
 150
 151     def test_custom_request_timeout(self):
 152         api = arvados.api('v1', timeout=1234)
 153         self.assertEqual(api._http.timeout, 1234,
 154             "Requested timeout value was 1234")
 155
 156     def test_4xx_retried(self):
 157         client = arvados.api('v1')
 158         for code in self.RETRIED_4XX:
 159             name = f'retried #{code}'
 160             with self.subTest(name), mock.patch('time.sleep'):
 161                 expected = {'username': name}
 162                 with mock_api_responses(
 163                         client,
 164                         json.dumps(expected),
 165                         [code, code, 200],
 166                         self.ERROR_HEADERS,
 167                         'orig_http_request',
 168                 ):
 169                     actual = client.users().current().execute()
 170                 self.assertEqual(actual, expected)
 171
 172     def test_4xx_not_retried(self):
 173         client = arvados.api('v1', num_retries=3)
 174         # Note that googleapiclient does retry 403 *if* the response JSON
 175         # includes flags that say the request was denied by rate limiting.
 176         # An empty JSON response like we use here should not be retried.
 177         for code in [400, 401, 403, 404, 422]:
 178             with self.subTest(f'error {code}'), mock.patch('time.sleep'):
 179                 with mock_api_responses(
 180                         client,
 181                         b'{}',
 182                         [code, 200],
 183                         self.ERROR_HEADERS,
 184                         'orig_http_request',
 185                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
 186                     client.users().current().execute()
 187                 response = exc_check.exception.args[0]
 188                 self.assertEqual(response.status, code)
 189                 self.assertEqual(response.get('status'), str(code))
 190
 191     def test_4xx_raised_after_retry_exhaustion(self):
 192         client = arvados.api('v1', num_retries=1)
 193         for code in self.RETRIED_4XX:
 194             with self.subTest(f'failed {code}'), mock.patch('time.sleep'):
 195                 with mock_api_responses(
 196                         client,
 197                         b'{}',
 198                         [code, code, code, 200],
 199                         self.ERROR_HEADERS,
 200                         'orig_http_request',
 201                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
 202                     client.users().current().execute()
 203                 response = exc_check.exception.args[0]
 204                 self.assertEqual(response.status, code)
 205                 self.assertEqual(response.get('status'), str(code))
 206
 207     def test_ordered_json_model(self):
 208         mock_responses = {
 209             'arvados.humans.get': (
 210                 None,
 211                 json.dumps(collections.OrderedDict(
 212                     (c, int(c, 16)) for c in string.hexdigits
 213                 )).encode(),
 214             ),
 215         }
 216         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 217         api = arvados.api('v1',
 218                           requestBuilder=req_builder, model=OrderedJsonModel())
 219         result = api.humans().get(uuid='test').execute()
 220         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 221
 222     def test_api_is_threadsafe(self):
 223         api_kwargs = {
 224             'host': os.environ['ARVADOS_API_HOST'],
 225             'token': os.environ['ARVADOS_API_TOKEN'],
 226             'insecure': True,
 227         }
 228         config_kwargs = {'apiconfig': os.environ}
 229         for api_constructor, kwargs in [
 230                 (arvados.api, {}),
 231                 (arvados.api, api_kwargs),
 232                 (arvados.api_from_config, {}),
 233                 (arvados.api_from_config, config_kwargs),
 234         ]:
 235             sub_kwargs = "kwargs" if kwargs else "no kwargs"
 236             with self.subTest(f"{api_constructor.__name__} with {sub_kwargs}"):
 237                 api_client = api_constructor('v1', **kwargs)
 238                 self.assertTrue(hasattr(api_client, 'localapi'),
 239                                 f"client missing localapi method")
 240                 self.assertTrue(hasattr(api_client, 'keep'),
 241                                 f"client missing keep attribute")
 242
 243     def test_api_host_constructor(self):
 244         cache = True
 245         insecure = True
 246         client = arvados.api(
 247             'v1',
 248             cache,
 249             os.environ['ARVADOS_API_HOST'],
 250             os.environ['ARVADOS_API_TOKEN'],
 251             insecure,
 252         )
 253         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 254                          "client constructed with incorrect token")
 255
 256     def test_api_url_constructor(self):
 257         client = arvados.api(
 258             'v1',
 259             discoveryServiceUrl=self._discoveryServiceUrl(),
 260             token=os.environ['ARVADOS_API_TOKEN'],
 261             insecure=True,
 262         )
 263         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 264                          "client constructed with incorrect token")
 265
 266     def test_api_bad_args(self):
 267         all_kwargs = {
 268             'host': os.environ['ARVADOS_API_HOST'],
 269             'token': os.environ['ARVADOS_API_TOKEN'],
 270             'discoveryServiceUrl': self._discoveryServiceUrl(),
 271         }
 272         for use_keys in [
 273                 # Passing only a single key is missing required info
 274                 *([key] for key in all_kwargs.keys()),
 275                 # Passing all keys is a conflict
 276                 list(all_kwargs.keys()),
 277         ]:
 278             kwargs = {key: all_kwargs[key] for key in use_keys}
 279             kwargs_list = ', '.join(use_keys)
 280             with self.subTest(f"calling arvados.api with {kwargs_list} fails"), \
 281                  self.assertRaises(ValueError):
 282                 arvados.api('v1', insecure=True, **kwargs)
 283
 284     def test_api_bad_url(self):
 285         for bad_kwargs in [
 286                 {'discoveryServiceUrl': self._discoveryServiceUrl() + '/BadTestURL'},
 287                 {'version': 'BadTestVersion', 'host': os.environ['ARVADOS_API_HOST']},
 288         ]:
 289             bad_key = next(iter(bad_kwargs))
 290             with self.subTest(f"api fails with bad {bad_key}"), \
 291                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 292                 arvados.api(**bad_kwargs, token='test_api_bad_url', insecure=True)
 293
 294     def test_normalize_api_good_args(self):
 295         for version, discoveryServiceUrl, host in [
 296                 ('Test1', None, os.environ['ARVADOS_API_HOST']),
 297                 (None, self._discoveryServiceUrl(), None)
 298         ]:
 299             argname = 'discoveryServiceUrl' if host is None else 'host'
 300             with self.subTest(f"normalize_api_kwargs with {argname}"):
 301                 actual = normalize_api_kwargs(
 302                     version,
 303                     discoveryServiceUrl,
 304                     host,
 305                     os.environ['ARVADOS_API_TOKEN'],
 306                     insecure=True,
 307                 )
 308                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 309                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 310                 self.assertEqual(actual['version'], version or 'v1')
 311                 self.assertTrue(actual['insecure'])
 312                 self.assertNotIn('host', actual)
 313
 314     def test_normalize_api_bad_args(self):
 315         all_args = (
 316             self._discoveryServiceUrl(),
 317             os.environ['ARVADOS_API_HOST'],
 318             os.environ['ARVADOS_API_TOKEN'],
 319         )
 320         for arg_index, arg_value in enumerate(all_args):
 321             args = [None] * len(all_args)
 322             args[arg_index] = arg_value
 323             with self.subTest(f"normalize_api_kwargs with only arg #{arg_index + 1}"), \
 324                  self.assertRaises(ValueError):
 325                 normalize_api_kwargs('v1', *args)
 326         with self.subTest("normalize_api_kwargs with discoveryServiceUrl and host"), \
 327              self.assertRaises(ValueError):
 328             normalize_api_kwargs('v1', *all_args)
 329
 330     def test_api_from_config_default(self):
 331         client = arvados.api_from_config('v1')
 332         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 333                          "client constructed with incorrect token")
 334
 335     def test_api_from_config_explicit(self):
 336         config = self._config_from_environ()
 337         client = arvados.api_from_config('v1', config)
 338         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 339                          "client constructed with incorrect token")
 340
 341     def test_api_from_bad_config(self):
 342         base_config = self._config_from_environ()
 343         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 344             with self.subTest(f"api_from_config without {del_key} fails"), \
 345                  self.assertRaises(ValueError):
 346                 config = dict(base_config)
 347                 del config[del_key]
 348                 arvados.api_from_config('v1', config)
 349
 350     def test_api_kwargs_from_good_config(self):
 351         for config in [None, self._config_from_environ()]:
 352             conf_type = 'default' if config is None else 'passed'
 353             with self.subTest(f"api_kwargs_from_config with {conf_type} config"):
 354                 version = 'Test1' if config else None
 355                 actual = api_kwargs_from_config(version, config)
 356                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 357                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 358                 self.assertEqual(actual['version'], version or 'v1')
 359                 self.assertTrue(actual['insecure'])
 360                 self.assertNotIn('host', actual)
 361
 362     def test_api_kwargs_from_bad_config(self):
 363         base_config = self._config_from_environ()
 364         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 365             with self.subTest(f"api_kwargs_from_config without {del_key} fails"), \
 366                  self.assertRaises(ValueError):
 367                 config = dict(base_config)
 368                 del config[del_key]
 369                 api_kwargs_from_config('v1', config)
 370
 371     def test_api_client_constructor(self):
 372         client = api_client(
 373             'v1',
 374             self._discoveryServiceUrl(),
 375             os.environ['ARVADOS_API_TOKEN'],
 376             insecure=True,
 377         )
 378         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 379                          "client constructed with incorrect token")
 380         self.assertFalse(
 381             hasattr(client, 'localapi'),
 382             "client has localapi method when it should not be thread-safe",
 383         )
 384
 385     def test_api_client_bad_url(self):
 386         all_args = ('v1', self._discoveryServiceUrl(), 'test_api_client_bad_url')
 387         for arg_index, arg_value in [
 388                 (0, 'BadTestVersion'),
 389                 (1, all_args[1] + '/BadTestURL'),
 390         ]:
 391             with self.subTest(f"api_client fails with {arg_index}={arg_value!r}"), \
 392                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 393                 args = list(all_args)
 394                 args[arg_index] = arg_value
 395                 api_client(*args, insecure=True)
 396
 397     def test_initial_retry_logs(self):
 398         try:
 399             _googleapiclient_log_lock.release()
 400         except RuntimeError:
 401             # Lock was never acquired - that's the state we want anyway
 402             pass
 403         real_logger = logging.getLogger('googleapiclient.http')
 404         mock_logger = mock.Mock(wraps=real_logger)
 405         mock_logger.handlers = logging.getLogger('googleapiclient').handlers
 406         mock_logger.level = logging.NOTSET
 407         with mock.patch('logging.getLogger', return_value=mock_logger), \
 408              mock.patch('time.sleep'), \
 409              self.assertLogs(real_logger, 'INFO') as actual_logs:
 410             try:
 411                 api_client('v1', 'https://test.invalid/', 'NoToken', num_retries=1)
 412             except httplib2.error.ServerNotFoundError:
 413                 pass
 414         mock_logger.addFilter.assert_called()
 415         mock_logger.addHandler.assert_called()
 416         mock_logger.setLevel.assert_called()
 417         mock_logger.removeHandler.assert_called()
 418         mock_logger.removeFilter.assert_called()
 419         self.assertRegex(actual_logs.output[0], r'^INFO:googleapiclient\.http:Sleeping \d')
 420
 421     def test_configured_logger_untouched(self):
 422         real_logger = logging.getLogger('googleapiclient.http')
 423         mock_logger = mock.Mock(wraps=real_logger)
 424         mock_logger.handlers = logging.getLogger().handlers
 425         with mock.patch('logging.getLogger', return_value=mock_logger), \
 426              mock.patch('time.sleep'):
 427             try:
 428                 api_client('v1', 'https://test.invalid/', 'NoToken', num_retries=1)
 429             except httplib2.error.ServerNotFoundError:
 430                 pass
 431         mock_logger.addFilter.assert_not_called()
 432         mock_logger.addHandler.assert_not_called()
 433         mock_logger.setLevel.assert_not_called()
 434         mock_logger.removeHandler.assert_not_called()
 435         mock_logger.removeFilter.assert_not_called()
 436
 437
 438 class ConstructNumRetriesTestCase(unittest.TestCase):
 439     @staticmethod
 440     def _fake_retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs):
 441         return http.request(uri, method, *args, **kwargs)
 442
 443     @contextlib.contextmanager
 444     def patch_retry(self):
 445         # We have this dedicated context manager that goes through `sys.modules`
 446         # instead of just using `mock.patch` because of the unfortunate
 447         # `arvados.api` name collision.
 448         orig_func = sys.modules['arvados.api']._orig_retry_request
 449         expect_name = 'googleapiclient.http._retry_request'
 450         self.assertEqual(
 451             '{0.__module__}.{0.__name__}'.format(orig_func), expect_name,
 452             f"test setup problem: {expect_name} not at arvados.api._orig_retry_request",
 453         )
 454         retry_mock = mock.Mock(wraps=self._fake_retry_request)
 455         sys.modules['arvados.api']._orig_retry_request = retry_mock
 456         try:
 457             yield retry_mock
 458         finally:
 459             sys.modules['arvados.api']._orig_retry_request = orig_func
 460
 461     def _iter_num_retries(self, retry_mock):
 462         for call in retry_mock.call_args_list:
 463             try:
 464                 yield call.args[1]
 465             except IndexError:
 466                 yield call.kwargs['num_retries']
 467
 468     def test_default_num_retries(self):
 469         with self.patch_retry() as retry_mock:
 470             client = arvados.api('v1')
 471         actual = set(self._iter_num_retries(retry_mock))
 472         self.assertEqual(len(actual), 1)
 473         self.assertTrue(actual.pop() > 6, "num_retries lower than expected")
 474
 475     def _test_calls(self, init_arg, call_args, expected):
 476         with self.patch_retry() as retry_mock:
 477             client = arvados.api('v1', num_retries=init_arg)
 478             for num_retries in call_args:
 479                 client.users().current().execute(num_retries=num_retries)
 480         actual = self._iter_num_retries(retry_mock)
 481         # The constructor makes two requests with its num_retries argument:
 482         # one for the discovery document, and one for the config.
 483         self.assertEqual(next(actual, None), init_arg)
 484         self.assertEqual(next(actual, None), init_arg)
 485         self.assertEqual(list(actual), expected)
 486
 487     def test_discovery_num_retries(self):
 488         for num_retries in [0, 5, 55]:
 489             with self.subTest(f"num_retries={num_retries}"):
 490                 self._test_calls(num_retries, [], [])
 491
 492     def test_num_retries_called_le_init(self):
 493         for n in [6, 10]:
 494             with self.subTest(f"init_arg={n}"):
 495                 call_args = [n - 4, n - 2, n]
 496                 expected = [n] * 3
 497                 self._test_calls(n, call_args, expected)
 498
 499     def test_num_retries_called_ge_init(self):
 500         for n in [0, 10]:
 501             with self.subTest(f"init_arg={n}"):
 502                 call_args = [n, n + 4, n + 8]
 503                 self._test_calls(n, call_args, call_args)
 504
 505     def test_num_retries_called_mixed(self):
 506         self._test_calls(5, [2, 6, 4, 8], [5, 6, 5, 8])
 507
 508
 509 class PreCloseSocketTestCase(unittest.TestCase):
 510     def setUp(self):
 511         self.api = arvados.api('v1')
 512         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 513                         "test doesn't know how to intercept HTTP requests")
 514         self.mock_response = {'user': 'person'}
 515         self.request_success = (fake_httplib2_response(200),
 516                                 json.dumps(self.mock_response))
 517         self.api._http.orig_http_request = mock.MagicMock()
 518         # All requests succeed by default. Tests override as needed.
 519         self.api._http.orig_http_request.return_value = self.request_success
 520
 521     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 522     def test_close_old_connections_non_retryable(self, sleep):
 523         self._test_connection_close(expect=1)
 524
 525     @mock.patch('time.time', side_effect=itertools.count())
 526     def test_no_close_fresh_connections_non_retryable(self, sleep):
 527         self._test_connection_close(expect=0)
 528
 529     @mock.patch('time.time', side_effect=itertools.count())
 530     def test_override_max_idle_time(self, sleep):
 531         self.api._http._max_keepalive_idle = 0
 532         self._test_connection_close(expect=1)
 533
 534     def _test_connection_close(self, expect=0):
 535         # Do two POST requests. The second one must close all
 536         # connections +expect+ times.
 537         self.api.users().create(body={}).execute()
 538         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 539         self.api._http.connections = mock_conns.copy()
 540         self.api.users().create(body={}).execute()
 541         for c in mock_conns.values():
 542             self.assertEqual(c.close.call_count, expect)
 543
 544
 545 if __name__ == '__main__':
 546     unittest.main()