sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados
   6 import collections
   7 import contextlib
   8 import httplib2
   9 import itertools
  10 import json
  11 import logging
  12 import mimetypes
  13 import os
  14 import socket
  15 import string
  16 import sys
  17 import unittest
  18 import urllib.parse as urlparse
  19
  20 from unittest import mock
  21 from . import run_test_server
  22
  23 from apiclient import errors as apiclient_errors
  24 from apiclient import http as apiclient_http
  25 from arvados.api import (
  26     api_client,
  27     normalize_api_kwargs,
  28     api_kwargs_from_config,
  29     _googleapiclient_log_lock,
  30 )
  31 from .arvados_testutil import fake_httplib2_response, mock_api_responses, queue_with
  32 import httplib2.error
  33
  34 if not mimetypes.inited:
  35     mimetypes.init()
  36
  37 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  38     MAIN_SERVER = {}
  39     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  40     RETRIED_4XX = frozenset([408, 409, 423])
  41
  42     def api_error_response(self, code, *errors):
  43         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  44                 json.dumps({'errors': errors,
  45                             'error_token': '1234567890+12345678'}).encode())
  46
  47     def _config_from_environ(self):
  48         return {
  49             key: value
  50             for key, value in os.environ.items()
  51             if key.startswith('ARVADOS_API_')
  52         }
  53
  54     def _discoveryServiceUrl(
  55             self,
  56             host=None,
  57             path='/discovery/v1/apis/{api}/{apiVersion}/rest',
  58             scheme='https',
  59     ):
  60         if host is None:
  61             host = os.environ['ARVADOS_API_HOST']
  62         return urlparse.urlunsplit((scheme, host, path, None, None))
  63
  64     def test_new_api_objects_with_cache(self):
  65         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  66         self.assertIsNot(*clients)
  67
  68     def test_empty_list(self):
  69         answer = arvados.api('v1').collections().list(
  70             filters=[['uuid', '=', 'abcdef']]).execute()
  71         self.assertEqual(answer['items_available'], len(answer['items']))
  72
  73     def test_nonempty_list(self):
  74         answer = arvados.api('v1').collections().list().execute()
  75         self.assertNotEqual(0, answer['items_available'])
  76         self.assertNotEqual(0, len(answer['items']))
  77
  78     def test_timestamp_inequality_filter(self):
  79         api = arvados.api('v1')
  80         new_item = api.collections().create(body={}).execute()
  81         for operator, should_include in [
  82                 ['<', False], ['>', False],
  83                 ['<=', True], ['>=', True], ['=', True]]:
  84             response = api.collections().list(filters=[
  85                 ['created_at', operator, new_item['created_at']],
  86                 # Also filter by uuid to ensure (if it matches) it's on page 0
  87                 ['uuid', '=', new_item['uuid']]]).execute()
  88             uuids = [item['uuid'] for item in response['items']]
  89             did_include = new_item['uuid'] in uuids
  90             self.assertEqual(
  91                 did_include, should_include,
  92                 "'%s %s' filter should%s have matched '%s'" % (
  93                     operator, new_item['created_at'],
  94                     ('' if should_include else ' not'),
  95                     new_item['created_at']))
  96
  97     def test_exceptions_include_errors(self):
  98         mock_responses = {
  99             'arvados.collections.get': self.api_error_response(
 100                 422, "Bad UUID format", "Bad output format"),
 101             }
 102         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 103         api = arvados.api('v1', requestBuilder=req_builder)
 104         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 105             api.collections().get(uuid='xyz-xyz-abcdef').execute()
 106         err_s = str(err_ctx.exception)
 107         for msg in ["Bad UUID format", "Bad output format"]:
 108             self.assertIn(msg, err_s)
 109
 110     @mock.patch('time.sleep')
 111     def test_exceptions_include_request_id(self, sleep):
 112         api = arvados.api('v1')
 113         api.request_id='fake-request-id'
 114         api._http.orig_http_request = mock.MagicMock()
 115         api._http.orig_http_request.side_effect = socket.error('mock error')
 116         caught = None
 117         try:
 118             api.users().current().execute()
 119         except Exception as e:
 120             caught = e
 121         self.assertRegex(str(caught), r'fake-request-id')
 122
 123     def test_exceptions_without_errors_have_basic_info(self):
 124         mock_responses = {
 125             'arvados.collections.delete': (
 126                 fake_httplib2_response(500, **self.ERROR_HEADERS),
 127                 b"")
 128             }
 129         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 130         api = arvados.api('v1', requestBuilder=req_builder)
 131         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 132             api.collections().delete(uuid='xyz-xyz-abcdef').execute()
 133         self.assertIn("500", str(err_ctx.exception))
 134
 135     def test_request_too_large(self):
 136         api = arvados.api('v1')
 137         maxsize = api._rootDesc.get('maxRequestSize', 0)
 138         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 139             text = "X" * maxsize
 140             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 141
 142     def test_default_request_timeout(self):
 143         api = arvados.api('v1')
 144         self.assertEqual(api._http.timeout, 300,
 145             "Default timeout value should be 300")
 146
 147     def test_custom_request_timeout(self):
 148         api = arvados.api('v1', timeout=1234)
 149         self.assertEqual(api._http.timeout, 1234,
 150             "Requested timeout value was 1234")
 151
 152     def test_4xx_retried(self):
 153         client = arvados.api('v1')
 154         for code in self.RETRIED_4XX:
 155             name = f'retried #{code}'
 156             with self.subTest(name), mock.patch('time.sleep'):
 157                 expected = {'username': name}
 158                 with mock_api_responses(
 159                         client,
 160                         json.dumps(expected),
 161                         [code, code, 200],
 162                         self.ERROR_HEADERS,
 163                         'orig_http_request',
 164                 ):
 165                     actual = client.users().current().execute()
 166                 self.assertEqual(actual, expected)
 167
 168     def test_4xx_not_retried(self):
 169         client = arvados.api('v1', num_retries=3)
 170         # Note that googleapiclient does retry 403 *if* the response JSON
 171         # includes flags that say the request was denied by rate limiting.
 172         # An empty JSON response like we use here should not be retried.
 173         for code in [400, 401, 403, 404, 422]:
 174             with self.subTest(f'error {code}'), mock.patch('time.sleep'):
 175                 with mock_api_responses(
 176                         client,
 177                         b'{}',
 178                         [code, 200],
 179                         self.ERROR_HEADERS,
 180                         'orig_http_request',
 181                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
 182                     client.users().current().execute()
 183                 response = exc_check.exception.args[0]
 184                 self.assertEqual(response.status, code)
 185                 self.assertEqual(response.get('status'), str(code))
 186
 187     def test_4xx_raised_after_retry_exhaustion(self):
 188         client = arvados.api('v1', num_retries=1)
 189         for code in self.RETRIED_4XX:
 190             with self.subTest(f'failed {code}'), mock.patch('time.sleep'):
 191                 with mock_api_responses(
 192                         client,
 193                         b'{}',
 194                         [code, code, code, 200],
 195                         self.ERROR_HEADERS,
 196                         'orig_http_request',
 197                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
 198                     client.users().current().execute()
 199                 response = exc_check.exception.args[0]
 200                 self.assertEqual(response.status, code)
 201                 self.assertEqual(response.get('status'), str(code))
 202
 203     def test_api_is_threadsafe(self):
 204         api_kwargs = {
 205             'host': os.environ['ARVADOS_API_HOST'],
 206             'token': os.environ['ARVADOS_API_TOKEN'],
 207             'insecure': True,
 208         }
 209         config_kwargs = {'apiconfig': os.environ}
 210         for api_constructor, kwargs in [
 211                 (arvados.api, {}),
 212                 (arvados.api, api_kwargs),
 213                 (arvados.api_from_config, {}),
 214                 (arvados.api_from_config, config_kwargs),
 215         ]:
 216             sub_kwargs = "kwargs" if kwargs else "no kwargs"
 217             with self.subTest(f"{api_constructor.__name__} with {sub_kwargs}"):
 218                 api_client = api_constructor('v1', **kwargs)
 219                 self.assertTrue(hasattr(api_client, 'localapi'),
 220                                 f"client missing localapi method")
 221                 self.assertTrue(hasattr(api_client, 'keep'),
 222                                 f"client missing keep attribute")
 223
 224     def test_api_host_constructor(self):
 225         cache = True
 226         insecure = True
 227         client = arvados.api(
 228             'v1',
 229             cache,
 230             os.environ['ARVADOS_API_HOST'],
 231             os.environ['ARVADOS_API_TOKEN'],
 232             insecure,
 233         )
 234         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 235                          "client constructed with incorrect token")
 236
 237     def test_api_url_constructor(self):
 238         client = arvados.api(
 239             'v1',
 240             discoveryServiceUrl=self._discoveryServiceUrl(),
 241             token=os.environ['ARVADOS_API_TOKEN'],
 242             insecure=True,
 243         )
 244         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 245                          "client constructed with incorrect token")
 246
 247     def test_api_bad_args(self):
 248         all_kwargs = {
 249             'host': os.environ['ARVADOS_API_HOST'],
 250             'token': os.environ['ARVADOS_API_TOKEN'],
 251             'discoveryServiceUrl': self._discoveryServiceUrl(),
 252         }
 253         for use_keys in [
 254                 # Passing only a single key is missing required info
 255                 *([key] for key in all_kwargs.keys()),
 256                 # Passing all keys is a conflict
 257                 list(all_kwargs.keys()),
 258         ]:
 259             kwargs = {key: all_kwargs[key] for key in use_keys}
 260             kwargs_list = ', '.join(use_keys)
 261             with self.subTest(f"calling arvados.api with {kwargs_list} fails"), \
 262                  self.assertRaises(ValueError):
 263                 arvados.api('v1', insecure=True, **kwargs)
 264
 265     def test_api_bad_url(self):
 266         for bad_kwargs in [
 267                 {'discoveryServiceUrl': self._discoveryServiceUrl() + '/BadTestURL'},
 268                 {'version': 'BadTestVersion', 'host': os.environ['ARVADOS_API_HOST']},
 269         ]:
 270             bad_key = next(iter(bad_kwargs))
 271             with self.subTest(f"api fails with bad {bad_key}"), \
 272                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 273                 arvados.api(**bad_kwargs, token='test_api_bad_url', insecure=True)
 274
 275     def test_normalize_api_good_args(self):
 276         for version, discoveryServiceUrl, host in [
 277                 ('Test1', None, os.environ['ARVADOS_API_HOST']),
 278                 (None, self._discoveryServiceUrl(), None)
 279         ]:
 280             argname = 'discoveryServiceUrl' if host is None else 'host'
 281             with self.subTest(f"normalize_api_kwargs with {argname}"):
 282                 actual = normalize_api_kwargs(
 283                     version,
 284                     discoveryServiceUrl,
 285                     host,
 286                     os.environ['ARVADOS_API_TOKEN'],
 287                     insecure=True,
 288                 )
 289                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 290                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 291                 self.assertEqual(actual['version'], version or 'v1')
 292                 self.assertTrue(actual['insecure'])
 293                 self.assertNotIn('host', actual)
 294
 295     def test_normalize_api_bad_args(self):
 296         all_args = (
 297             self._discoveryServiceUrl(),
 298             os.environ['ARVADOS_API_HOST'],
 299             os.environ['ARVADOS_API_TOKEN'],
 300         )
 301         for arg_index, arg_value in enumerate(all_args):
 302             args = [None] * len(all_args)
 303             args[arg_index] = arg_value
 304             with self.subTest(f"normalize_api_kwargs with only arg #{arg_index + 1}"), \
 305                  self.assertRaises(ValueError):
 306                 normalize_api_kwargs('v1', *args)
 307         with self.subTest("normalize_api_kwargs with discoveryServiceUrl and host"), \
 308              self.assertRaises(ValueError):
 309             normalize_api_kwargs('v1', *all_args)
 310
 311     def test_api_from_config_default(self):
 312         client = arvados.api_from_config('v1')
 313         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 314                          "client constructed with incorrect token")
 315
 316     def test_api_from_config_explicit(self):
 317         config = self._config_from_environ()
 318         client = arvados.api_from_config('v1', config)
 319         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 320                          "client constructed with incorrect token")
 321
 322     def test_api_from_bad_config(self):
 323         base_config = self._config_from_environ()
 324         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 325             with self.subTest(f"api_from_config without {del_key} fails"), \
 326                  self.assertRaises(ValueError):
 327                 config = dict(base_config)
 328                 del config[del_key]
 329                 arvados.api_from_config('v1', config)
 330
 331     def test_api_kwargs_from_good_config(self):
 332         for config in [None, self._config_from_environ()]:
 333             conf_type = 'default' if config is None else 'passed'
 334             with self.subTest(f"api_kwargs_from_config with {conf_type} config"):
 335                 version = 'Test1' if config else None
 336                 actual = api_kwargs_from_config(version, config)
 337                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 338                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 339                 self.assertEqual(actual['version'], version or 'v1')
 340                 self.assertTrue(actual['insecure'])
 341                 self.assertNotIn('host', actual)
 342
 343     def test_api_kwargs_from_bad_config(self):
 344         base_config = self._config_from_environ()
 345         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 346             with self.subTest(f"api_kwargs_from_config without {del_key} fails"), \
 347                  self.assertRaises(ValueError):
 348                 config = dict(base_config)
 349                 del config[del_key]
 350                 api_kwargs_from_config('v1', config)
 351
 352     def test_api_client_constructor(self):
 353         client = api_client(
 354             'v1',
 355             self._discoveryServiceUrl(),
 356             os.environ['ARVADOS_API_TOKEN'],
 357             insecure=True,
 358         )
 359         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 360                          "client constructed with incorrect token")
 361         self.assertFalse(
 362             hasattr(client, 'localapi'),
 363             "client has localapi method when it should not be thread-safe",
 364         )
 365
 366     def test_api_client_bad_url(self):
 367         all_args = ('v1', self._discoveryServiceUrl(), 'test_api_client_bad_url')
 368         for arg_index, arg_value in [
 369                 (0, 'BadTestVersion'),
 370                 (1, all_args[1] + '/BadTestURL'),
 371         ]:
 372             with self.subTest(f"api_client fails with {arg_index}={arg_value!r}"), \
 373                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 374                 args = list(all_args)
 375                 args[arg_index] = arg_value
 376                 api_client(*args, insecure=True)
 377
 378     def test_initial_retry_logs(self):
 379         try:
 380             _googleapiclient_log_lock.release()
 381         except RuntimeError:
 382             # Lock was never acquired - that's the state we want anyway
 383             pass
 384         real_logger = logging.getLogger('googleapiclient.http')
 385         mock_logger = mock.Mock(wraps=real_logger)
 386         mock_logger.handlers = logging.getLogger('googleapiclient').handlers
 387         mock_logger.level = logging.NOTSET
 388         with mock.patch('logging.getLogger', return_value=mock_logger), \
 389              mock.patch('time.sleep'), \
 390              self.assertLogs(real_logger, 'INFO') as actual_logs:
 391             try:
 392                 api_client('v1', 'https://test.invalid/', 'NoToken', num_retries=1)
 393             except httplib2.error.ServerNotFoundError:
 394                 pass
 395         mock_logger.addFilter.assert_called()
 396         mock_logger.addHandler.assert_called()
 397         mock_logger.setLevel.assert_called()
 398         mock_logger.removeHandler.assert_called()
 399         mock_logger.removeFilter.assert_called()
 400         self.assertRegex(actual_logs.output[0], r'^INFO:googleapiclient\.http:Sleeping \d')
 401
 402     def test_configured_logger_untouched(self):
 403         real_logger = logging.getLogger('googleapiclient.http')
 404         mock_logger = mock.Mock(wraps=real_logger)
 405         mock_logger.handlers = logging.getLogger().handlers
 406         with mock.patch('logging.getLogger', return_value=mock_logger), \
 407              mock.patch('time.sleep'):
 408             try:
 409                 api_client('v1', 'https://test.invalid/', 'NoToken', num_retries=1)
 410             except httplib2.error.ServerNotFoundError:
 411                 pass
 412         mock_logger.addFilter.assert_not_called()
 413         mock_logger.addHandler.assert_not_called()
 414         mock_logger.setLevel.assert_not_called()
 415         mock_logger.removeHandler.assert_not_called()
 416         mock_logger.removeFilter.assert_not_called()
 417
 418
 419 class ConstructNumRetriesTestCase(unittest.TestCase):
 420     @staticmethod
 421     def _fake_retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs):
 422         return http.request(uri, method, *args, **kwargs)
 423
 424     @contextlib.contextmanager
 425     def patch_retry(self):
 426         # We have this dedicated context manager that goes through `sys.modules`
 427         # instead of just using `mock.patch` because of the unfortunate
 428         # `arvados.api` name collision.
 429         orig_func = sys.modules['arvados.api']._orig_retry_request
 430         expect_name = 'googleapiclient.http._retry_request'
 431         self.assertEqual(
 432             '{0.__module__}.{0.__name__}'.format(orig_func), expect_name,
 433             f"test setup problem: {expect_name} not at arvados.api._orig_retry_request",
 434         )
 435         retry_mock = mock.Mock(wraps=self._fake_retry_request)
 436         sys.modules['arvados.api']._orig_retry_request = retry_mock
 437         try:
 438             yield retry_mock
 439         finally:
 440             sys.modules['arvados.api']._orig_retry_request = orig_func
 441
 442     def _iter_num_retries(self, retry_mock):
 443         for call in retry_mock.call_args_list:
 444             try:
 445                 yield call.args[1]
 446             except IndexError:
 447                 yield call.kwargs['num_retries']
 448
 449     def test_default_num_retries(self):
 450         with self.patch_retry() as retry_mock:
 451             client = arvados.api('v1')
 452         actual = set(self._iter_num_retries(retry_mock))
 453         self.assertEqual(len(actual), 1)
 454         self.assertTrue(actual.pop() > 6, "num_retries lower than expected")
 455
 456     def _test_calls(self, init_arg, call_args, expected):
 457         with self.patch_retry() as retry_mock:
 458             client = arvados.api('v1', num_retries=init_arg)
 459             for num_retries in call_args:
 460                 client.users().current().execute(num_retries=num_retries)
 461         actual = self._iter_num_retries(retry_mock)
 462         # The constructor makes two requests with its num_retries argument:
 463         # one for the discovery document, and one for the config.
 464         self.assertEqual(next(actual, None), init_arg)
 465         self.assertEqual(next(actual, None), init_arg)
 466         self.assertEqual(list(actual), expected)
 467
 468     def test_discovery_num_retries(self):
 469         for num_retries in [0, 5, 55]:
 470             with self.subTest(f"num_retries={num_retries}"):
 471                 self._test_calls(num_retries, [], [])
 472
 473     def test_num_retries_called_le_init(self):
 474         for n in [6, 10]:
 475             with self.subTest(f"init_arg={n}"):
 476                 call_args = [n - 4, n - 2, n]
 477                 expected = [n] * 3
 478                 self._test_calls(n, call_args, expected)
 479
 480     def test_num_retries_called_ge_init(self):
 481         for n in [0, 10]:
 482             with self.subTest(f"init_arg={n}"):
 483                 call_args = [n, n + 4, n + 8]
 484                 self._test_calls(n, call_args, call_args)
 485
 486     def test_num_retries_called_mixed(self):
 487         self._test_calls(5, [2, 6, 4, 8], [5, 6, 5, 8])
 488
 489
 490 class PreCloseSocketTestCase(unittest.TestCase):
 491     def setUp(self):
 492         self.api = arvados.api('v1')
 493         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 494                         "test doesn't know how to intercept HTTP requests")
 495         self.mock_response = {'user': 'person'}
 496         self.request_success = (fake_httplib2_response(200),
 497                                 json.dumps(self.mock_response))
 498         self.api._http.orig_http_request = mock.MagicMock()
 499         # All requests succeed by default. Tests override as needed.
 500         self.api._http.orig_http_request.return_value = self.request_success
 501
 502     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 503     def test_close_old_connections_non_retryable(self, sleep):
 504         self._test_connection_close(expect=1)
 505
 506     @mock.patch('time.time', side_effect=itertools.count())
 507     def test_no_close_fresh_connections_non_retryable(self, sleep):
 508         self._test_connection_close(expect=0)
 509
 510     @mock.patch('time.time', side_effect=itertools.count())
 511     def test_override_max_idle_time(self, sleep):
 512         self.api._http._max_keepalive_idle = 0
 513         self._test_connection_close(expect=1)
 514
 515     def _test_connection_close(self, expect=0):
 516         # Do two POST requests. The second one must close all
 517         # connections +expect+ times.
 518         self.api.users().create(body={}).execute()
 519         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 520         self.api._http.connections = mock_conns.copy()
 521         self.api.users().create(body={}).execute()
 522         for c in mock_conns.values():
 523             self.assertEqual(c.close.call_count, expect)
 524
 525
 526 if __name__ == '__main__':
 527     unittest.main()