sdk/python/tests/test_api.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import str
   7 from builtins import range
   8 import arvados
   9 import collections
  10 import httplib2
  11 import itertools
  12 import json
  13 import mimetypes
  14 import os
  15 import socket
  16 import string
  17 import unittest
  18 import urllib.parse as urlparse
  19
  20 import mock
  21 from . import run_test_server
  22
  23 from apiclient import errors as apiclient_errors
  24 from apiclient import http as apiclient_http
  25 from arvados.api import (
  26     api_client,
  27     normalize_api_kwargs,
  28     api_kwargs_from_config,
  29     OrderedJsonModel,
  30 )
  31 from .arvados_testutil import fake_httplib2_response, queue_with
  32
  33 if not mimetypes.inited:
  34     mimetypes.init()
  35
  36 class ArvadosApiTest(run_test_server.TestCaseWithServers):
  37     MAIN_SERVER = {}
  38     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
  39
  40     def api_error_response(self, code, *errors):
  41         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
  42                 json.dumps({'errors': errors,
  43                             'error_token': '1234567890+12345678'}).encode())
  44
  45     def _config_from_environ(self):
  46         return {
  47             key: value
  48             for key, value in os.environ.items()
  49             if key.startswith('ARVADOS_API_')
  50         }
  51
  52     def _discoveryServiceUrl(
  53             self,
  54             host=None,
  55             path='/discovery/v1/apis/{api}/{apiVersion}/rest',
  56             scheme='https',
  57     ):
  58         if host is None:
  59             host = os.environ['ARVADOS_API_HOST']
  60         return urlparse.urlunsplit((scheme, host, path, None, None))
  61
  62     def test_new_api_objects_with_cache(self):
  63         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
  64         self.assertIsNot(*clients)
  65
  66     def test_empty_list(self):
  67         answer = arvados.api('v1').humans().list(
  68             filters=[['uuid', '=', None]]).execute()
  69         self.assertEqual(answer['items_available'], len(answer['items']))
  70
  71     def test_nonempty_list(self):
  72         answer = arvados.api('v1').collections().list().execute()
  73         self.assertNotEqual(0, answer['items_available'])
  74         self.assertNotEqual(0, len(answer['items']))
  75
  76     def test_timestamp_inequality_filter(self):
  77         api = arvados.api('v1')
  78         new_item = api.specimens().create(body={}).execute()
  79         for operator, should_include in [
  80                 ['<', False], ['>', False],
  81                 ['<=', True], ['>=', True], ['=', True]]:
  82             response = api.specimens().list(filters=[
  83                 ['created_at', operator, new_item['created_at']],
  84                 # Also filter by uuid to ensure (if it matches) it's on page 0
  85                 ['uuid', '=', new_item['uuid']]]).execute()
  86             uuids = [item['uuid'] for item in response['items']]
  87             did_include = new_item['uuid'] in uuids
  88             self.assertEqual(
  89                 did_include, should_include,
  90                 "'%s %s' filter should%s have matched '%s'" % (
  91                     operator, new_item['created_at'],
  92                     ('' if should_include else ' not'),
  93                     new_item['created_at']))
  94
  95     def test_exceptions_include_errors(self):
  96         mock_responses = {
  97             'arvados.humans.get': self.api_error_response(
  98                 422, "Bad UUID format", "Bad output format"),
  99             }
 100         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 101         api = arvados.api('v1', requestBuilder=req_builder)
 102         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 103             api.humans().get(uuid='xyz-xyz-abcdef').execute()
 104         err_s = str(err_ctx.exception)
 105         for msg in ["Bad UUID format", "Bad output format"]:
 106             self.assertIn(msg, err_s)
 107
 108     @mock.patch('time.sleep')
 109     def test_exceptions_include_request_id(self, sleep):
 110         api = arvados.api('v1')
 111         api.request_id='fake-request-id'
 112         api._http.orig_http_request = mock.MagicMock()
 113         api._http.orig_http_request.side_effect = socket.error('mock error')
 114         caught = None
 115         try:
 116             api.users().current().execute()
 117         except Exception as e:
 118             caught = e
 119         self.assertRegex(str(caught), r'fake-request-id')
 120
 121     def test_exceptions_without_errors_have_basic_info(self):
 122         mock_responses = {
 123             'arvados.humans.delete': (
 124                 fake_httplib2_response(500, **self.ERROR_HEADERS),
 125                 b"")
 126             }
 127         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 128         api = arvados.api('v1', requestBuilder=req_builder)
 129         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
 130             api.humans().delete(uuid='xyz-xyz-abcdef').execute()
 131         self.assertIn("500", str(err_ctx.exception))
 132
 133     def test_request_too_large(self):
 134         api = arvados.api('v1')
 135         maxsize = api._rootDesc.get('maxRequestSize', 0)
 136         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
 137             text = "X" * maxsize
 138             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
 139
 140     def test_default_request_timeout(self):
 141         api = arvados.api('v1')
 142         self.assertEqual(api._http.timeout, 300,
 143             "Default timeout value should be 300")
 144
 145     def test_custom_request_timeout(self):
 146         api = arvados.api('v1', timeout=1234)
 147         self.assertEqual(api._http.timeout, 1234,
 148             "Requested timeout value was 1234")
 149
 150     def test_ordered_json_model(self):
 151         mock_responses = {
 152             'arvados.humans.get': (
 153                 None,
 154                 json.dumps(collections.OrderedDict(
 155                     (c, int(c, 16)) for c in string.hexdigits
 156                 )).encode(),
 157             ),
 158         }
 159         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
 160         api = arvados.api('v1',
 161                           requestBuilder=req_builder, model=OrderedJsonModel())
 162         result = api.humans().get(uuid='test').execute()
 163         self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
 164
 165     def test_api_is_threadsafe(self):
 166         api_kwargs = {
 167             'host': os.environ['ARVADOS_API_HOST'],
 168             'token': os.environ['ARVADOS_API_TOKEN'],
 169             'insecure': True,
 170         }
 171         config_kwargs = {'apiconfig': os.environ}
 172         for api_constructor, kwargs in [
 173                 (arvados.api, {}),
 174                 (arvados.api, api_kwargs),
 175                 (arvados.api_from_config, {}),
 176                 (arvados.api_from_config, config_kwargs),
 177         ]:
 178             sub_kwargs = "kwargs" if kwargs else "no kwargs"
 179             with self.subTest(f"{api_constructor.__name__} with {sub_kwargs}"):
 180                 api_client = api_constructor('v1', **kwargs)
 181                 self.assertTrue(hasattr(api_client, 'localapi'),
 182                                 f"client missing localapi method")
 183                 self.assertTrue(hasattr(api_client, 'keep'),
 184                                 f"client missing keep attribute")
 185
 186     def test_api_host_constructor(self):
 187         cache = True
 188         insecure = True
 189         client = arvados.api(
 190             'v1',
 191             cache,
 192             os.environ['ARVADOS_API_HOST'],
 193             os.environ['ARVADOS_API_TOKEN'],
 194             insecure,
 195         )
 196         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 197                          "client constructed with incorrect token")
 198
 199     def test_api_url_constructor(self):
 200         client = arvados.api(
 201             'v1',
 202             discoveryServiceUrl=self._discoveryServiceUrl(),
 203             token=os.environ['ARVADOS_API_TOKEN'],
 204             insecure=True,
 205         )
 206         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 207                          "client constructed with incorrect token")
 208
 209     def test_api_bad_args(self):
 210         all_kwargs = {
 211             'host': os.environ['ARVADOS_API_HOST'],
 212             'token': os.environ['ARVADOS_API_TOKEN'],
 213             'discoveryServiceUrl': self._discoveryServiceUrl(),
 214         }
 215         for use_keys in [
 216                 # Passing only a single key is missing required info
 217                 *([key] for key in all_kwargs.keys()),
 218                 # Passing all keys is a conflict
 219                 list(all_kwargs.keys()),
 220         ]:
 221             kwargs = {key: all_kwargs[key] for key in use_keys}
 222             kwargs_list = ', '.join(use_keys)
 223             with self.subTest(f"calling arvados.api with {kwargs_list} fails"), \
 224                  self.assertRaises(ValueError):
 225                 arvados.api('v1', insecure=True, **kwargs)
 226
 227     def test_api_bad_url(self):
 228         for bad_kwargs in [
 229                 {'discoveryServiceUrl': self._discoveryServiceUrl() + '/BadTestURL'},
 230                 {'version': 'BadTestVersion', 'host': os.environ['ARVADOS_API_HOST']},
 231         ]:
 232             bad_key = next(iter(bad_kwargs))
 233             with self.subTest(f"api fails with bad {bad_key}"), \
 234                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 235                 arvados.api(**bad_kwargs, token='test_api_bad_url', insecure=True)
 236
 237     def test_normalize_api_good_args(self):
 238         for version, discoveryServiceUrl, host in [
 239                 ('Test1', None, os.environ['ARVADOS_API_HOST']),
 240                 (None, self._discoveryServiceUrl(), None)
 241         ]:
 242             argname = 'discoveryServiceUrl' if host is None else 'host'
 243             with self.subTest(f"normalize_api_kwargs with {argname}"):
 244                 actual = normalize_api_kwargs(
 245                     version,
 246                     discoveryServiceUrl,
 247                     host,
 248                     os.environ['ARVADOS_API_TOKEN'],
 249                     insecure=True,
 250                 )
 251                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 252                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 253                 self.assertEqual(actual['version'], version or 'v1')
 254                 self.assertTrue(actual['insecure'])
 255                 self.assertNotIn('host', actual)
 256
 257     def test_normalize_api_bad_args(self):
 258         all_args = (
 259             self._discoveryServiceUrl(),
 260             os.environ['ARVADOS_API_HOST'],
 261             os.environ['ARVADOS_API_TOKEN'],
 262         )
 263         for arg_index, arg_value in enumerate(all_args):
 264             args = [None] * len(all_args)
 265             args[arg_index] = arg_value
 266             with self.subTest(f"normalize_api_kwargs with only arg #{arg_index + 1}"), \
 267                  self.assertRaises(ValueError):
 268                 normalize_api_kwargs('v1', *args)
 269         with self.subTest("normalize_api_kwargs with discoveryServiceUrl and host"), \
 270              self.assertRaises(ValueError):
 271             normalize_api_kwargs('v1', *all_args)
 272
 273     def test_api_from_config_default(self):
 274         client = arvados.api_from_config('v1')
 275         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 276                          "client constructed with incorrect token")
 277
 278     def test_api_from_config_explicit(self):
 279         config = self._config_from_environ()
 280         client = arvados.api_from_config('v1', config)
 281         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 282                          "client constructed with incorrect token")
 283
 284     def test_api_from_bad_config(self):
 285         base_config = self._config_from_environ()
 286         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 287             with self.subTest(f"api_from_config without {del_key} fails"), \
 288                  self.assertRaises(ValueError):
 289                 config = dict(base_config)
 290                 del config[del_key]
 291                 arvados.api_from_config('v1', config)
 292
 293     def test_api_kwargs_from_good_config(self):
 294         for config in [None, self._config_from_environ()]:
 295             conf_type = 'default' if config is None else 'passed'
 296             with self.subTest(f"api_kwargs_from_config with {conf_type} config"):
 297                 version = 'Test1' if config else None
 298                 actual = api_kwargs_from_config(version, config)
 299                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
 300                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
 301                 self.assertEqual(actual['version'], version or 'v1')
 302                 self.assertTrue(actual['insecure'])
 303                 self.assertNotIn('host', actual)
 304
 305     def test_api_kwargs_from_bad_config(self):
 306         base_config = self._config_from_environ()
 307         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
 308             with self.subTest(f"api_kwargs_from_config without {del_key} fails"), \
 309                  self.assertRaises(ValueError):
 310                 config = dict(base_config)
 311                 del config[del_key]
 312                 api_kwargs_from_config('v1', config)
 313
 314     def test_api_client_constructor(self):
 315         client = api_client(
 316             'v1',
 317             self._discoveryServiceUrl(),
 318             os.environ['ARVADOS_API_TOKEN'],
 319             insecure=True,
 320         )
 321         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
 322                          "client constructed with incorrect token")
 323         self.assertFalse(
 324             hasattr(client, 'localapi'),
 325             "client has localapi method when it should not be thread-safe",
 326         )
 327
 328     def test_api_client_bad_url(self):
 329         all_args = ('v1', self._discoveryServiceUrl(), 'test_api_client_bad_url')
 330         for arg_index, arg_value in [
 331                 (0, 'BadTestVersion'),
 332                 (1, all_args[1] + '/BadTestURL'),
 333         ]:
 334             with self.subTest(f"api_client fails with {arg_index}={arg_value!r}"), \
 335                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
 336                 args = list(all_args)
 337                 args[arg_index] = arg_value
 338                 api_client(*args, insecure=True)
 339
 340
 341 class PreCloseSocketTestCase(unittest.TestCase):
 342     def setUp(self):
 343         self.api = arvados.api('v1')
 344         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
 345                         "test doesn't know how to intercept HTTP requests")
 346         self.mock_response = {'user': 'person'}
 347         self.request_success = (fake_httplib2_response(200),
 348                                 json.dumps(self.mock_response))
 349         self.api._http.orig_http_request = mock.MagicMock()
 350         # All requests succeed by default. Tests override as needed.
 351         self.api._http.orig_http_request.return_value = self.request_success
 352
 353     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
 354     def test_close_old_connections_non_retryable(self, sleep):
 355         self._test_connection_close(expect=1)
 356
 357     @mock.patch('time.time', side_effect=itertools.count())
 358     def test_no_close_fresh_connections_non_retryable(self, sleep):
 359         self._test_connection_close(expect=0)
 360
 361     @mock.patch('time.time', side_effect=itertools.count())
 362     def test_override_max_idle_time(self, sleep):
 363         self.api._http._max_keepalive_idle = 0
 364         self._test_connection_close(expect=1)
 365
 366     def _test_connection_close(self, expect=0):
 367         # Do two POST requests. The second one must close all
 368         # connections +expect+ times.
 369         self.api.users().create(body={}).execute()
 370         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
 371         self.api._http.connections = mock_conns.copy()
 372         self.api.users().create(body={}).execute()
 373         for c in mock_conns.values():
 374             self.assertEqual(c.close.call_count, expect)
 375
 376
 377 if __name__ == '__main__':
 378     unittest.main()