sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados
   6 import collections
   7 import contextlib
   8 import httplib2
   9 import itertools
  10 import json
  11 import logging
  12 import mimetypes
  13 import os
  14 import socket
  15 import string
  16 import sys
  17 import unittest
  18 import urllib.parse as urlparse
  19
  20 from unittest import mock
  21 from . import run_test_server
  22
  23 from apiclient import errors as apiclient_errors
  24 from apiclient import http as apiclient_http
  25 from arvados.api import (
  26     api_client,
  27     normalize_api_kwargs,
  28     api_kwargs_from_config,
  29     OrderedJsonModel,
  30     _googleapiclient_log_lock,
  31 )
  32 from .arvados_testutil import fake_httplib2_response, mock_api_responses, queue_with
  33 import httplib2.error
  34
  35 if not mimetypes.inited:
  36     mimetypes.init()
  37
  38 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  39     MAIN_SERVER = {}
  40     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  41     RETRIED_4XX = frozenset([408, 409, 423])
  42
  43     def api_error_response(self, code, *errors):
  44         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  45                 json.dumps({'errors': errors,
  46                             'error_token': '1234567890+12345678'}).encode())
  47
  48     def _config_from_environ(self):
  49         return {
  50             key: value
  51             for key, value in os.environ.items()
  52             if key.startswith('ARVADOS_API_')
  53         }
  54
  55     def _discoveryServiceUrl(
  56             self,
  57             host=None,
  58             path='/discovery/v1/apis/{api}/{apiVersion}/rest',
  59             scheme='https',
  60     ):
  61         if host is None:
  62             host = os.environ['ARVADOS_API_HOST']
  63         return urlparse.urlunsplit((scheme, host, path, None, None))
  64
  65     def test_new_api_objects_with_cache(self):
  66         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  67         self.assertIsNot(*clients)
  68
  69     def test_empty_list(self):
  70         answer = arvados.api('v1').collections().list(
  71             filters=[['uuid', '=', 'abcdef']]).execute()
  72         self.assertEqual(answer['items_available'], len(answer['items']))
  73
  74     def test_nonempty_list(self):
  75         answer = arvados.api('v1').collections().list().execute()
  76         self.assertNotEqual(0, answer['items_available'])
  77         self.assertNotEqual(0, len(answer['items']))
  78
  79     def test_timestamp_inequality_filter(self):
  80         api = arvados.api('v1')
  81         new_item = api.collections().create(body={}).execute()
  82         for operator, should_include in [
  83                 ['<', False], ['>', False],
  84                 ['<=', True], ['>=', True], ['=', True]]:
  85             response = api.collections().list(filters=[
  86                 ['created_at', operator, new_item['created_at']],
  87                 # Also filter by uuid to ensure (if it matches) it's on page 0
  88                 ['uuid', '=', new_item['uuid']]]).execute()
  89             uuids = [item['uuid'] for item in response['items']]
  90             did_include = new_item['uuid'] in uuids
  91             self.assertEqual(
  92                 did_include, should_include,
  93                 "'%s %s' filter should%s have matched '%s'" % (
  94                     operator, new_item['created_at'],
  95                     ('' if should_include else ' not'),
  96                     new_item['created_at']))
  97
  98     def test_exceptions_include_errors(self):
  99         mock_responses = {
 100             'arvados.collections.get': self.api_error_response(
 101                 422, "Bad UUID format", "Bad output format"),
 102             }
 103         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 104         api = arvados.api('v1', requestBuilder=req_builder)
 105         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 106             api.collections().get(uuid='xyz-xyz-abcdef').execute()
 107         err_s = str(err_ctx.exception)
 108         for msg in ["Bad UUID format", "Bad output format"]:
 109             self.assertIn(msg, err_s)
 110
 111     @mock.patch('time.sleep')
 112     def test_exceptions_include_request_id(self, sleep):
 113         api = arvados.api('v1')
 114         api.request_id='fake-request-id'
 115         api._http.orig_http_request = mock.MagicMock()
 116         api._http.orig_http_request.side_effect = socket.error('mock error')
 117         caught = None
 118         try:
 119             api.users().current().execute()
 120         except Exception as e:
 121             caught = e
 122         self.assertRegex(str(caught), r'fake-request-id')
 123
 124     def test_exceptions_without_errors_have_basic_info(self):
 125         mock_responses = {
 126             'arvados.collections.delete': (
 127                 fake_httplib2_response(500, **self.ERROR_HEADERS),
 128                 b"")
 129             }
 130         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 131         api = arvados.api('v1', requestBuilder=req_builder)
 132         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 133             api.collections().delete(uuid='xyz-xyz-abcdef').execute()
 134         self.assertIn("500", str(err_ctx.exception))
 135
 136     def test_request_too_large(self):
 137         api = arvados.api('v1')
 138         maxsize = api._rootDesc.get('maxRequestSize', 0)
 139         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 140             text = "X" * maxsize
 141             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 142
 143     def test_default_request_timeout(self):
 144         api = arvados.api('v1')
 145         self.assertEqual(api._http.timeout, 300,
 146             "Default timeout value should be 300")
 147
 148     def test_custom_request_timeout(self):
 149         api = arvados.api('v1', timeout=1234)
 150         self.assertEqual(api._http.timeout, 1234,
 151             "Requested timeout value was 1234")
 152
 153     def test_4xx_retried(self):
 154         client = arvados.api('v1')
 155         for code in self.RETRIED_4XX:
 156             name = f'retried #{code}'
 157             with self.subTest(name), mock.patch('time.sleep'):
 158                 expected = {'username': name}
 159                 with mock_api_responses(
 160                         client,
 161                         json.dumps(expected),
 162                         [code, code, 200],
 163                         self.ERROR_HEADERS,
 164                         'orig_http_request',
 165                 ):
 166                     actual = client.users().current().execute()
 167                 self.assertEqual(actual, expected)
 168
 169     def test_4xx_not_retried(self):
 170         client = arvados.api('v1', num_retries=3)
 171         # Note that googleapiclient does retry 403 *if* the response JSON
 172         # includes flags that say the request was denied by rate limiting.
 173         # An empty JSON response like we use here should not be retried.
 174         for code in [400, 401, 403, 404, 422]:
 175             with self.subTest(f'error {code}'), mock.patch('time.sleep'):
 176                 with mock_api_responses(
 177                         client,
 178                         b'{}',
 179                         [code, 200],
 180                         self.ERROR_HEADERS,
 181                         'orig_http_request',
 182                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
 183                     client.users().current().execute()
 184                 response = exc_check.exception.args[0]
 185                 self.assertEqual(response.status, code)
 186                 self.assertEqual(response.get('status'), str(code))
 187
 188     def test_4xx_raised_after_retry_exhaustion(self):
 189         client = arvados.api('v1', num_retries=1)
 190         for code in self.RETRIED_4XX:
 191             with self.subTest(f'failed {code}'), mock.patch('time.sleep'):
 192                 with mock_api_responses(
 193                         client,
 194                         b'{}',
 195                         [code, code, code, 200],
 196                         self.ERROR_HEADERS,
 197                         'orig_http_request',
 198                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
 199                     client.users().current().execute()
 200                 response = exc_check.exception.args[0]
 201                 self.assertEqual(response.status, code)
 202                 self.assertEqual(response.get('status'), str(code))
 203
 204     def test_ordered_json_model(self):
 205         mock_responses = {
 206             'arvados.collections.get': (
 207                 None,
 208                 json.dumps(collections.OrderedDict(
 209                     (c, int(c, 16)) for c in string.hexdigits
 210                 )).encode(),
 211             ),
 212         }
 213         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 214         api = arvados.api('v1',
 215                           requestBuilder=req_builder, model=OrderedJsonModel())
 216         result = api.collections().get(uuid='test').execute()
 217         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 218
 219     def test_api_is_threadsafe(self):
 220         api_kwargs = {
 221             'host': os.environ['ARVADOS_API_HOST'],
 222             'token': os.environ['ARVADOS_API_TOKEN'],
 223             'insecure': True,
 224         }
 225         config_kwargs = {'apiconfig': os.environ}
 226         for api_constructor, kwargs in [
 227                 (arvados.api, {}),
 228                 (arvados.api, api_kwargs),
 229                 (arvados.api_from_config, {}),
 230                 (arvados.api_from_config, config_kwargs),
 231         ]:
 232             sub_kwargs = "kwargs" if kwargs else "no kwargs"
 233             with self.subTest(f"{api_constructor.__name__} with {sub_kwargs}"):
 234                 api_client = api_constructor('v1', **kwargs)
 235                 self.assertTrue(hasattr(api_client, 'localapi'),
 236                                 f"client missing localapi method")
 237                 self.assertTrue(hasattr(api_client, 'keep'),
 238                                 f"client missing keep attribute")
 239
 240     def test_api_host_constructor(self):
 241         cache = True
 242         insecure = True
 243         client = arvados.api(
 244             'v1',
 245             cache,
 246             os.environ['ARVADOS_API_HOST'],
 247             os.environ['ARVADOS_API_TOKEN'],
 248             insecure,
 249         )
 250         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 251                          "client constructed with incorrect token")
 252
 253     def test_api_url_constructor(self):
 254         client = arvados.api(
 255             'v1',
 256             discoveryServiceUrl=self._discoveryServiceUrl(),
 257             token=os.environ['ARVADOS_API_TOKEN'],
 258             insecure=True,
 259         )
 260         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 261                          "client constructed with incorrect token")
 262
 263     def test_api_bad_args(self):
 264         all_kwargs = {
 265             'host': os.environ['ARVADOS_API_HOST'],
 266             'token': os.environ['ARVADOS_API_TOKEN'],
 267             'discoveryServiceUrl': self._discoveryServiceUrl(),
 268         }
 269         for use_keys in [
 270                 # Passing only a single key is missing required info
 271                 *([key] for key in all_kwargs.keys()),
 272                 # Passing all keys is a conflict
 273                 list(all_kwargs.keys()),
 274         ]:
 275             kwargs = {key: all_kwargs[key] for key in use_keys}
 276             kwargs_list = ', '.join(use_keys)
 277             with self.subTest(f"calling arvados.api with {kwargs_list} fails"), \
 278                  self.assertRaises(ValueError):
 279                 arvados.api('v1', insecure=True, **kwargs)
 280
 281     def test_api_bad_url(self):
 282         for bad_kwargs in [
 283                 {'discoveryServiceUrl': self._discoveryServiceUrl() + '/BadTestURL'},
 284                 {'version': 'BadTestVersion', 'host': os.environ['ARVADOS_API_HOST']},
 285         ]:
 286             bad_key = next(iter(bad_kwargs))
 287             with self.subTest(f"api fails with bad {bad_key}"), \
 288                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 289                 arvados.api(**bad_kwargs, token='test_api_bad_url', insecure=True)
 290
 291     def test_normalize_api_good_args(self):
 292         for version, discoveryServiceUrl, host in [
 293                 ('Test1', None, os.environ['ARVADOS_API_HOST']),
 294                 (None, self._discoveryServiceUrl(), None)
 295         ]:
 296             argname = 'discoveryServiceUrl' if host is None else 'host'
 297             with self.subTest(f"normalize_api_kwargs with {argname}"):
 298                 actual = normalize_api_kwargs(
 299                     version,
 300                     discoveryServiceUrl,
 301                     host,
 302                     os.environ['ARVADOS_API_TOKEN'],
 303                     insecure=True,
 304                 )
 305                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 306                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 307                 self.assertEqual(actual['version'], version or 'v1')
 308                 self.assertTrue(actual['insecure'])
 309                 self.assertNotIn('host', actual)
 310
 311     def test_normalize_api_bad_args(self):
 312         all_args = (
 313             self._discoveryServiceUrl(),
 314             os.environ['ARVADOS_API_HOST'],
 315             os.environ['ARVADOS_API_TOKEN'],
 316         )
 317         for arg_index, arg_value in enumerate(all_args):
 318             args = [None] * len(all_args)
 319             args[arg_index] = arg_value
 320             with self.subTest(f"normalize_api_kwargs with only arg #{arg_index + 1}"), \
 321                  self.assertRaises(ValueError):
 322                 normalize_api_kwargs('v1', *args)
 323         with self.subTest("normalize_api_kwargs with discoveryServiceUrl and host"), \
 324              self.assertRaises(ValueError):
 325             normalize_api_kwargs('v1', *all_args)
 326
 327     def test_api_from_config_default(self):
 328         client = arvados.api_from_config('v1')
 329         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 330                          "client constructed with incorrect token")
 331
 332     def test_api_from_config_explicit(self):
 333         config = self._config_from_environ()
 334         client = arvados.api_from_config('v1', config)
 335         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 336                          "client constructed with incorrect token")
 337
 338     def test_api_from_bad_config(self):
 339         base_config = self._config_from_environ()
 340         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 341             with self.subTest(f"api_from_config without {del_key} fails"), \
 342                  self.assertRaises(ValueError):
 343                 config = dict(base_config)
 344                 del config[del_key]
 345                 arvados.api_from_config('v1', config)
 346
 347     def test_api_kwargs_from_good_config(self):
 348         for config in [None, self._config_from_environ()]:
 349             conf_type = 'default' if config is None else 'passed'
 350             with self.subTest(f"api_kwargs_from_config with {conf_type} config"):
 351                 version = 'Test1' if config else None
 352                 actual = api_kwargs_from_config(version, config)
 353                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 354                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 355                 self.assertEqual(actual['version'], version or 'v1')
 356                 self.assertTrue(actual['insecure'])
 357                 self.assertNotIn('host', actual)
 358
 359     def test_api_kwargs_from_bad_config(self):
 360         base_config = self._config_from_environ()
 361         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 362             with self.subTest(f"api_kwargs_from_config without {del_key} fails"), \
 363                  self.assertRaises(ValueError):
 364                 config = dict(base_config)
 365                 del config[del_key]
 366                 api_kwargs_from_config('v1', config)
 367
 368     def test_api_client_constructor(self):
 369         client = api_client(
 370             'v1',
 371             self._discoveryServiceUrl(),
 372             os.environ['ARVADOS_API_TOKEN'],
 373             insecure=True,
 374         )
 375         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 376                          "client constructed with incorrect token")
 377         self.assertFalse(
 378             hasattr(client, 'localapi'),
 379             "client has localapi method when it should not be thread-safe",
 380         )
 381
 382     def test_api_client_bad_url(self):
 383         all_args = ('v1', self._discoveryServiceUrl(), 'test_api_client_bad_url')
 384         for arg_index, arg_value in [
 385                 (0, 'BadTestVersion'),
 386                 (1, all_args[1] + '/BadTestURL'),
 387         ]:
 388             with self.subTest(f"api_client fails with {arg_index}={arg_value!r}"), \
 389                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 390                 args = list(all_args)
 391                 args[arg_index] = arg_value
 392                 api_client(*args, insecure=True)
 393
 394     def test_initial_retry_logs(self):
 395         try:
 396             _googleapiclient_log_lock.release()
 397         except RuntimeError:
 398             # Lock was never acquired - that's the state we want anyway
 399             pass
 400         real_logger = logging.getLogger('googleapiclient.http')
 401         mock_logger = mock.Mock(wraps=real_logger)
 402         mock_logger.handlers = logging.getLogger('googleapiclient').handlers
 403         mock_logger.level = logging.NOTSET
 404         with mock.patch('logging.getLogger', return_value=mock_logger), \
 405              mock.patch('time.sleep'), \
 406              self.assertLogs(real_logger, 'INFO') as actual_logs:
 407             try:
 408                 api_client('v1', 'https://test.invalid/', 'NoToken', num_retries=1)
 409             except httplib2.error.ServerNotFoundError:
 410                 pass
 411         mock_logger.addFilter.assert_called()
 412         mock_logger.addHandler.assert_called()
 413         mock_logger.setLevel.assert_called()
 414         mock_logger.removeHandler.assert_called()
 415         mock_logger.removeFilter.assert_called()
 416         self.assertRegex(actual_logs.output[0], r'^INFO:googleapiclient\.http:Sleeping \d')
 417
 418     def test_configured_logger_untouched(self):
 419         real_logger = logging.getLogger('googleapiclient.http')
 420         mock_logger = mock.Mock(wraps=real_logger)
 421         mock_logger.handlers = logging.getLogger().handlers
 422         with mock.patch('logging.getLogger', return_value=mock_logger), \
 423              mock.patch('time.sleep'):
 424             try:
 425                 api_client('v1', 'https://test.invalid/', 'NoToken', num_retries=1)
 426             except httplib2.error.ServerNotFoundError:
 427                 pass
 428         mock_logger.addFilter.assert_not_called()
 429         mock_logger.addHandler.assert_not_called()
 430         mock_logger.setLevel.assert_not_called()
 431         mock_logger.removeHandler.assert_not_called()
 432         mock_logger.removeFilter.assert_not_called()
 433
 434
 435 class ConstructNumRetriesTestCase(unittest.TestCase):
 436     @staticmethod
 437     def _fake_retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs):
 438         return http.request(uri, method, *args, **kwargs)
 439
 440     @contextlib.contextmanager
 441     def patch_retry(self):
 442         # We have this dedicated context manager that goes through `sys.modules`
 443         # instead of just using `mock.patch` because of the unfortunate
 444         # `arvados.api` name collision.
 445         orig_func = sys.modules['arvados.api']._orig_retry_request
 446         expect_name = 'googleapiclient.http._retry_request'
 447         self.assertEqual(
 448             '{0.__module__}.{0.__name__}'.format(orig_func), expect_name,
 449             f"test setup problem: {expect_name} not at arvados.api._orig_retry_request",
 450         )
 451         retry_mock = mock.Mock(wraps=self._fake_retry_request)
 452         sys.modules['arvados.api']._orig_retry_request = retry_mock
 453         try:
 454             yield retry_mock
 455         finally:
 456             sys.modules['arvados.api']._orig_retry_request = orig_func
 457
 458     def _iter_num_retries(self, retry_mock):
 459         for call in retry_mock.call_args_list:
 460             try:
 461                 yield call.args[1]
 462             except IndexError:
 463                 yield call.kwargs['num_retries']
 464
 465     def test_default_num_retries(self):
 466         with self.patch_retry() as retry_mock:
 467             client = arvados.api('v1')
 468         actual = set(self._iter_num_retries(retry_mock))
 469         self.assertEqual(len(actual), 1)
 470         self.assertTrue(actual.pop() > 6, "num_retries lower than expected")
 471
 472     def _test_calls(self, init_arg, call_args, expected):
 473         with self.patch_retry() as retry_mock:
 474             client = arvados.api('v1', num_retries=init_arg)
 475             for num_retries in call_args:
 476                 client.users().current().execute(num_retries=num_retries)
 477         actual = self._iter_num_retries(retry_mock)
 478         # The constructor makes two requests with its num_retries argument:
 479         # one for the discovery document, and one for the config.
 480         self.assertEqual(next(actual, None), init_arg)
 481         self.assertEqual(next(actual, None), init_arg)
 482         self.assertEqual(list(actual), expected)
 483
 484     def test_discovery_num_retries(self):
 485         for num_retries in [0, 5, 55]:
 486             with self.subTest(f"num_retries={num_retries}"):
 487                 self._test_calls(num_retries, [], [])
 488
 489     def test_num_retries_called_le_init(self):
 490         for n in [6, 10]:
 491             with self.subTest(f"init_arg={n}"):
 492                 call_args = [n - 4, n - 2, n]
 493                 expected = [n] * 3
 494                 self._test_calls(n, call_args, expected)
 495
 496     def test_num_retries_called_ge_init(self):
 497         for n in [0, 10]:
 498             with self.subTest(f"init_arg={n}"):
 499                 call_args = [n, n + 4, n + 8]
 500                 self._test_calls(n, call_args, call_args)
 501
 502     def test_num_retries_called_mixed(self):
 503         self._test_calls(5, [2, 6, 4, 8], [5, 6, 5, 8])
 504
 505
 506 class PreCloseSocketTestCase(unittest.TestCase):
 507     def setUp(self):
 508         self.api = arvados.api('v1')
 509         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 510                         "test doesn't know how to intercept HTTP requests")
 511         self.mock_response = {'user': 'person'}
 512         self.request_success = (fake_httplib2_response(200),
 513                                 json.dumps(self.mock_response))
 514         self.api._http.orig_http_request = mock.MagicMock()
 515         # All requests succeed by default. Tests override as needed.
 516         self.api._http.orig_http_request.return_value = self.request_success
 517
 518     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 519     def test_close_old_connections_non_retryable(self, sleep):
 520         self._test_connection_close(expect=1)
 521
 522     @mock.patch('time.time', side_effect=itertools.count())
 523     def test_no_close_fresh_connections_non_retryable(self, sleep):
 524         self._test_connection_close(expect=0)
 525
 526     @mock.patch('time.time', side_effect=itertools.count())
 527     def test_override_max_idle_time(self, sleep):
 528         self.api._http._max_keepalive_idle = 0
 529         self._test_connection_close(expect=1)
 530
 531     def _test_connection_close(self, expect=0):
 532         # Do two POST requests. The second one must close all
 533         # connections +expect+ times.
 534         self.api.users().create(body={}).execute()
 535         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 536         self.api._http.connections = mock_conns.copy()
 537         self.api.users().create(body={}).execute()
 538         for c in mock_conns.values():
 539             self.assertEqual(c.close.call_count, expect)
 540
 541
 542 if __name__ == '__main__':
 543     unittest.main()