Merge branch '21944-work-launch-description' refs #21944
[arvados.git] / sdk / python / tests / test_api.py
1 # Copyright (C) The Arvados Authors. All rights reserved.
2 #
3 # SPDX-License-Identifier: Apache-2.0
4
5 import arvados
6 import collections
7 import contextlib
8 import httplib2
9 import itertools
10 import json
11 import logging
12 import mimetypes
13 import os
14 import socket
15 import string
16 import sys
17 import unittest
18 import urllib.parse as urlparse
19
20 from unittest import mock
21 from . import run_test_server
22
23 from apiclient import errors as apiclient_errors
24 from apiclient import http as apiclient_http
25 from arvados.api import (
26     ThreadSafeAPIClient,
27     api_client,
28     normalize_api_kwargs,
29     api_kwargs_from_config,
30     _googleapiclient_log_lock,
31 )
32 from .arvados_testutil import fake_httplib2_response, mock_api_responses, queue_with
33
34 import googleapiclient
35 import httplib2.error
36
37 if not mimetypes.inited:
38     mimetypes.init()
39
40 class ArvadosApiTest(run_test_server.TestCaseWithServers):
41     MAIN_SERVER = {}
42     ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
43     RETRIED_4XX = frozenset([408, 409, 423])
44
45     def api_error_response(self, code, *errors):
46         return (fake_httplib2_response(code, **self.ERROR_HEADERS),
47                 json.dumps({'errors': errors,
48                             'error_token': '1234567890+12345678'}).encode())
49
50     def _config_from_environ(self):
51         return {
52             key: value
53             for key, value in os.environ.items()
54             if key.startswith('ARVADOS_API_')
55         }
56
57     def _discoveryServiceUrl(
58             self,
59             host=None,
60             path='/discovery/v1/apis/{api}/{apiVersion}/rest',
61             scheme='https',
62     ):
63         if host is None:
64             host = os.environ['ARVADOS_API_HOST']
65         return urlparse.urlunsplit((scheme, host, path, None, None))
66
67     def test_new_api_objects_with_cache(self):
68         clients = [arvados.api('v1', cache=True) for index in [0, 1]]
69         self.assertIsNot(*clients)
70
71     def test_empty_list(self):
72         answer = arvados.api('v1').collections().list(
73             filters=[['uuid', '=', 'abcdef']]).execute()
74         self.assertEqual(answer['items_available'], len(answer['items']))
75
76     def test_nonempty_list(self):
77         answer = arvados.api('v1').collections().list().execute()
78         self.assertNotEqual(0, answer['items_available'])
79         self.assertNotEqual(0, len(answer['items']))
80
81     def test_timestamp_inequality_filter(self):
82         api = arvados.api('v1')
83         new_item = api.collections().create(body={}).execute()
84         for operator, should_include in [
85                 ['<', False], ['>', False],
86                 ['<=', True], ['>=', True], ['=', True]]:
87             response = api.collections().list(filters=[
88                 ['created_at', operator, new_item['created_at']],
89                 # Also filter by uuid to ensure (if it matches) it's on page 0
90                 ['uuid', '=', new_item['uuid']]]).execute()
91             uuids = [item['uuid'] for item in response['items']]
92             did_include = new_item['uuid'] in uuids
93             self.assertEqual(
94                 did_include, should_include,
95                 "'%s %s' filter should%s have matched '%s'" % (
96                     operator, new_item['created_at'],
97                     ('' if should_include else ' not'),
98                     new_item['created_at']))
99
100     def test_exceptions_include_errors(self):
101         mock_responses = {
102             'arvados.collections.get': self.api_error_response(
103                 422, "Bad UUID format", "Bad output format"),
104             }
105         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
106         api = arvados.api('v1', requestBuilder=req_builder)
107         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
108             api.collections().get(uuid='xyz-xyz-abcdef').execute()
109         err_s = str(err_ctx.exception)
110         for msg in ["Bad UUID format", "Bad output format"]:
111             self.assertIn(msg, err_s)
112
113     @mock.patch('time.sleep')
114     def test_exceptions_include_request_id(self, sleep):
115         api = arvados.api('v1')
116         api.request_id='fake-request-id'
117         api._http.orig_http_request = mock.MagicMock()
118         api._http.orig_http_request.side_effect = socket.error('mock error')
119         caught = None
120         try:
121             api.users().current().execute()
122         except Exception as e:
123             caught = e
124         self.assertRegex(str(caught), r'fake-request-id')
125
126     def test_exceptions_without_errors_have_basic_info(self):
127         mock_responses = {
128             'arvados.collections.delete': (
129                 fake_httplib2_response(500, **self.ERROR_HEADERS),
130                 b"")
131             }
132         req_builder = apiclient_http.RequestMockBuilder(mock_responses)
133         api = arvados.api('v1', requestBuilder=req_builder)
134         with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
135             api.collections().delete(uuid='xyz-xyz-abcdef').execute()
136         self.assertIn("500", str(err_ctx.exception))
137
138     def test_request_too_large(self):
139         api = arvados.api('v1')
140         maxsize = api._rootDesc.get('maxRequestSize', 0)
141         with self.assertRaises(apiclient_errors.MediaUploadSizeError):
142             text = "X" * maxsize
143             arvados.api('v1').collections().create(body={"manifest_text": text}).execute()
144
145     def test_default_request_timeout(self):
146         api = arvados.api('v1')
147         self.assertEqual(api._http.timeout, 300,
148             "Default timeout value should be 300")
149
150     def test_custom_request_timeout(self):
151         api = arvados.api('v1', timeout=1234)
152         self.assertEqual(api._http.timeout, 1234,
153             "Requested timeout value was 1234")
154
155     def test_4xx_retried(self):
156         client = arvados.api('v1')
157         for code in self.RETRIED_4XX:
158             name = f'retried #{code}'
159             with self.subTest(name), mock.patch('time.sleep'):
160                 expected = {'username': name}
161                 with mock_api_responses(
162                         client,
163                         json.dumps(expected),
164                         [code, code, 200],
165                         self.ERROR_HEADERS,
166                         'orig_http_request',
167                 ):
168                     actual = client.users().current().execute()
169                 self.assertEqual(actual, expected)
170
171     def test_4xx_not_retried(self):
172         client = arvados.api('v1', num_retries=3)
173         # Note that googleapiclient does retry 403 *if* the response JSON
174         # includes flags that say the request was denied by rate limiting.
175         # An empty JSON response like we use here should not be retried.
176         for code in [400, 401, 403, 404, 422]:
177             with self.subTest(f'error {code}'), mock.patch('time.sleep'):
178                 with mock_api_responses(
179                         client,
180                         b'{}',
181                         [code, 200],
182                         self.ERROR_HEADERS,
183                         'orig_http_request',
184                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
185                     client.users().current().execute()
186                 response = exc_check.exception.args[0]
187                 self.assertEqual(response.status, code)
188                 self.assertEqual(response.get('status'), str(code))
189
190     def test_4xx_raised_after_retry_exhaustion(self):
191         client = arvados.api('v1', num_retries=1)
192         for code in self.RETRIED_4XX:
193             with self.subTest(f'failed {code}'), mock.patch('time.sleep'):
194                 with mock_api_responses(
195                         client,
196                         b'{}',
197                         [code, code, code, 200],
198                         self.ERROR_HEADERS,
199                         'orig_http_request',
200                 ), self.assertRaises(arvados.errors.ApiError) as exc_check:
201                     client.users().current().execute()
202                 response = exc_check.exception.args[0]
203                 self.assertEqual(response.status, code)
204                 self.assertEqual(response.get('status'), str(code))
205
206     def test_api_is_threadsafe(self):
207         api_kwargs = {
208             'host': os.environ['ARVADOS_API_HOST'],
209             'token': os.environ['ARVADOS_API_TOKEN'],
210             'insecure': True,
211         }
212         config_kwargs = {'apiconfig': os.environ}
213         for api_constructor, kwargs in [
214                 (arvados.api, {}),
215                 (arvados.api, api_kwargs),
216                 (arvados.api_from_config, {}),
217                 (arvados.api_from_config, config_kwargs),
218         ]:
219             sub_kwargs = "kwargs" if kwargs else "no kwargs"
220             with self.subTest(f"{api_constructor.__name__} with {sub_kwargs}"):
221                 api_client = api_constructor('v1', **kwargs)
222                 self.assertTrue(hasattr(api_client, 'localapi'),
223                                 f"client missing localapi method")
224                 self.assertTrue(hasattr(api_client, 'keep'),
225                                 f"client missing keep attribute")
226
227     def test_api_host_constructor(self):
228         cache = True
229         insecure = True
230         client = arvados.api(
231             'v1',
232             cache,
233             os.environ['ARVADOS_API_HOST'],
234             os.environ['ARVADOS_API_TOKEN'],
235             insecure,
236         )
237         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
238                          "client constructed with incorrect token")
239
240     def test_api_url_constructor(self):
241         client = arvados.api(
242             'v1',
243             discoveryServiceUrl=self._discoveryServiceUrl(),
244             token=os.environ['ARVADOS_API_TOKEN'],
245             insecure=True,
246         )
247         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
248                          "client constructed with incorrect token")
249
250     def test_api_bad_args(self):
251         all_kwargs = {
252             'host': os.environ['ARVADOS_API_HOST'],
253             'token': os.environ['ARVADOS_API_TOKEN'],
254             'discoveryServiceUrl': self._discoveryServiceUrl(),
255         }
256         for use_keys in [
257                 # Passing only a single key is missing required info
258                 *([key] for key in all_kwargs.keys()),
259                 # Passing all keys is a conflict
260                 list(all_kwargs.keys()),
261         ]:
262             kwargs = {key: all_kwargs[key] for key in use_keys}
263             kwargs_list = ', '.join(use_keys)
264             with self.subTest(f"calling arvados.api with {kwargs_list} fails"), \
265                  self.assertRaises(ValueError):
266                 arvados.api('v1', insecure=True, **kwargs)
267
268     def test_api_bad_url(self):
269         for bad_kwargs in [
270                 {'discoveryServiceUrl': self._discoveryServiceUrl() + '/BadTestURL'},
271                 {'version': 'BadTestVersion', 'host': os.environ['ARVADOS_API_HOST']},
272         ]:
273             bad_key = next(iter(bad_kwargs))
274             with self.subTest(f"api fails with bad {bad_key}"), \
275                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
276                 arvados.api(**bad_kwargs, token='test_api_bad_url', insecure=True)
277
278     def test_normalize_api_good_args(self):
279         for version, discoveryServiceUrl, host in [
280                 ('Test1', None, os.environ['ARVADOS_API_HOST']),
281                 (None, self._discoveryServiceUrl(), None)
282         ]:
283             argname = 'discoveryServiceUrl' if host is None else 'host'
284             with self.subTest(f"normalize_api_kwargs with {argname}"):
285                 actual = normalize_api_kwargs(
286                     version,
287                     discoveryServiceUrl,
288                     host,
289                     os.environ['ARVADOS_API_TOKEN'],
290                     insecure=True,
291                 )
292                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
293                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
294                 self.assertEqual(actual['version'], version or 'v1')
295                 self.assertTrue(actual['insecure'])
296                 self.assertNotIn('host', actual)
297
298     def test_normalize_api_bad_args(self):
299         all_args = (
300             self._discoveryServiceUrl(),
301             os.environ['ARVADOS_API_HOST'],
302             os.environ['ARVADOS_API_TOKEN'],
303         )
304         for arg_index, arg_value in enumerate(all_args):
305             args = [None] * len(all_args)
306             args[arg_index] = arg_value
307             with self.subTest(f"normalize_api_kwargs with only arg #{arg_index + 1}"), \
308                  self.assertRaises(ValueError):
309                 normalize_api_kwargs('v1', *args)
310         with self.subTest("normalize_api_kwargs with discoveryServiceUrl and host"), \
311              self.assertRaises(ValueError):
312             normalize_api_kwargs('v1', *all_args)
313
314     def test_api_from_config_default(self):
315         client = arvados.api_from_config('v1')
316         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
317                          "client constructed with incorrect token")
318
319     def test_api_from_config_explicit(self):
320         config = self._config_from_environ()
321         client = arvados.api_from_config('v1', config)
322         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
323                          "client constructed with incorrect token")
324
325     def test_api_from_bad_config(self):
326         base_config = self._config_from_environ()
327         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
328             with self.subTest(f"api_from_config without {del_key} fails"), \
329                  self.assertRaises(ValueError):
330                 config = dict(base_config)
331                 del config[del_key]
332                 arvados.api_from_config('v1', config)
333
334     def test_api_kwargs_from_good_config(self):
335         for config in [None, self._config_from_environ()]:
336             conf_type = 'default' if config is None else 'passed'
337             with self.subTest(f"api_kwargs_from_config with {conf_type} config"):
338                 version = 'Test1' if config else None
339                 actual = api_kwargs_from_config(version, config)
340                 self.assertEqual(actual['discoveryServiceUrl'], self._discoveryServiceUrl())
341                 self.assertEqual(actual['token'], os.environ['ARVADOS_API_TOKEN'])
342                 self.assertEqual(actual['version'], version or 'v1')
343                 self.assertTrue(actual['insecure'])
344                 self.assertNotIn('host', actual)
345
346     def test_api_kwargs_from_bad_config(self):
347         base_config = self._config_from_environ()
348         for del_key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
349             with self.subTest(f"api_kwargs_from_config without {del_key} fails"), \
350                  self.assertRaises(ValueError):
351                 config = dict(base_config)
352                 del config[del_key]
353                 api_kwargs_from_config('v1', config)
354
355     def test_api_client_constructor(self):
356         client = api_client(
357             'v1',
358             self._discoveryServiceUrl(),
359             os.environ['ARVADOS_API_TOKEN'],
360             insecure=True,
361         )
362         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'],
363                          "client constructed with incorrect token")
364         self.assertFalse(
365             hasattr(client, 'localapi'),
366             "client has localapi method when it should not be thread-safe",
367         )
368
369     def test_api_client_bad_url(self):
370         all_args = ('v1', self._discoveryServiceUrl(), 'test_api_client_bad_url')
371         for arg_index, arg_value in [
372                 (0, 'BadTestVersion'),
373                 (1, all_args[1] + '/BadTestURL'),
374         ]:
375             with self.subTest(f"api_client fails with {arg_index}={arg_value!r}"), \
376                  self.assertRaises(apiclient_errors.UnknownApiNameOrVersion):
377                 args = list(all_args)
378                 args[arg_index] = arg_value
379                 api_client(*args, insecure=True)
380
381     def test_initial_retry_logs(self):
382         try:
383             _googleapiclient_log_lock.release()
384         except RuntimeError:
385             # Lock was never acquired - that's the state we want anyway
386             pass
387         real_logger = logging.getLogger('googleapiclient.http')
388         mock_logger = mock.Mock(wraps=real_logger)
389         mock_logger.handlers = logging.getLogger('googleapiclient').handlers
390         mock_logger.level = logging.NOTSET
391         with mock.patch('logging.getLogger', return_value=mock_logger), \
392              mock.patch('time.sleep'), \
393              self.assertLogs(real_logger, 'INFO') as actual_logs:
394             try:
395                 api_client('v1', 'https://test.invalid/', 'NoToken', num_retries=1)
396             except httplib2.error.ServerNotFoundError:
397                 pass
398         mock_logger.addFilter.assert_called()
399         mock_logger.addHandler.assert_called()
400         mock_logger.setLevel.assert_called()
401         mock_logger.removeHandler.assert_called()
402         mock_logger.removeFilter.assert_called()
403         self.assertRegex(actual_logs.output[0], r'^INFO:googleapiclient\.http:Sleeping \d')
404
405     def test_configured_logger_untouched(self):
406         real_logger = logging.getLogger('googleapiclient.http')
407         mock_logger = mock.Mock(wraps=real_logger)
408         mock_logger.handlers = logging.getLogger().handlers
409         with mock.patch('logging.getLogger', return_value=mock_logger), \
410              mock.patch('time.sleep'):
411             try:
412                 api_client('v1', 'https://test.invalid/', 'NoToken', num_retries=1)
413             except httplib2.error.ServerNotFoundError:
414                 pass
415         mock_logger.addFilter.assert_not_called()
416         mock_logger.addHandler.assert_not_called()
417         mock_logger.setLevel.assert_not_called()
418         mock_logger.removeHandler.assert_not_called()
419         mock_logger.removeFilter.assert_not_called()
420
421
422 class ConstructNumRetriesTestCase(unittest.TestCase):
423     @staticmethod
424     def _fake_retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs):
425         return http.request(uri, method, *args, **kwargs)
426
427     @contextlib.contextmanager
428     def patch_retry(self):
429         # We have this dedicated context manager that goes through `sys.modules`
430         # instead of just using `mock.patch` because of the unfortunate
431         # `arvados.api` name collision.
432         orig_func = sys.modules['arvados.api']._orig_retry_request
433         expect_name = 'googleapiclient.http._retry_request'
434         self.assertEqual(
435             '{0.__module__}.{0.__name__}'.format(orig_func), expect_name,
436             f"test setup problem: {expect_name} not at arvados.api._orig_retry_request",
437         )
438         retry_mock = mock.Mock(wraps=self._fake_retry_request)
439         sys.modules['arvados.api']._orig_retry_request = retry_mock
440         try:
441             yield retry_mock
442         finally:
443             sys.modules['arvados.api']._orig_retry_request = orig_func
444
445     def _iter_num_retries(self, retry_mock):
446         for call in retry_mock.call_args_list:
447             try:
448                 yield call.args[1]
449             except IndexError:
450                 yield call.kwargs['num_retries']
451
452     def test_default_num_retries(self):
453         with self.patch_retry() as retry_mock:
454             client = arvados.api('v1')
455         actual = set(self._iter_num_retries(retry_mock))
456         self.assertEqual(len(actual), 1)
457         self.assertTrue(actual.pop() > 6, "num_retries lower than expected")
458
459     def _test_calls(self, init_arg, call_args, expected):
460         with self.patch_retry() as retry_mock:
461             client = arvados.api('v1', num_retries=init_arg)
462             for num_retries in call_args:
463                 client.users().current().execute(num_retries=num_retries)
464         actual = self._iter_num_retries(retry_mock)
465         # The constructor makes two requests with its num_retries argument:
466         # one for the discovery document, and one for the config.
467         self.assertEqual(next(actual, None), init_arg)
468         self.assertEqual(next(actual, None), init_arg)
469         self.assertEqual(list(actual), expected)
470
471     def test_discovery_num_retries(self):
472         for num_retries in [0, 5, 55]:
473             with self.subTest(f"num_retries={num_retries}"):
474                 self._test_calls(num_retries, [], [])
475
476     def test_num_retries_called_le_init(self):
477         for n in [6, 10]:
478             with self.subTest(f"init_arg={n}"):
479                 call_args = [n - 4, n - 2, n]
480                 expected = [n] * 3
481                 self._test_calls(n, call_args, expected)
482
483     def test_num_retries_called_ge_init(self):
484         for n in [0, 10]:
485             with self.subTest(f"init_arg={n}"):
486                 call_args = [n, n + 4, n + 8]
487                 self._test_calls(n, call_args, call_args)
488
489     def test_num_retries_called_mixed(self):
490         self._test_calls(5, [2, 6, 4, 8], [5, 6, 5, 8])
491
492
493 class PreCloseSocketTestCase(unittest.TestCase):
494     def setUp(self):
495         self.api = arvados.api('v1')
496         self.assertTrue(hasattr(self.api._http, 'orig_http_request'),
497                         "test doesn't know how to intercept HTTP requests")
498         self.mock_response = {'user': 'person'}
499         self.request_success = (fake_httplib2_response(200),
500                                 json.dumps(self.mock_response))
501         self.api._http.orig_http_request = mock.MagicMock()
502         # All requests succeed by default. Tests override as needed.
503         self.api._http.orig_http_request.return_value = self.request_success
504
505     @mock.patch('time.time', side_effect=[i*2**20 for i in range(99)])
506     def test_close_old_connections_non_retryable(self, sleep):
507         self._test_connection_close(expect=1)
508
509     @mock.patch('time.time', side_effect=itertools.count())
510     def test_no_close_fresh_connections_non_retryable(self, sleep):
511         self._test_connection_close(expect=0)
512
513     @mock.patch('time.time', side_effect=itertools.count())
514     def test_override_max_idle_time(self, sleep):
515         self.api._http._max_keepalive_idle = 0
516         self._test_connection_close(expect=1)
517
518     def _test_connection_close(self, expect=0):
519         # Do two POST requests. The second one must close all
520         # connections +expect+ times.
521         self.api.users().create(body={}).execute()
522         mock_conns = {str(i): mock.MagicMock() for i in range(2)}
523         self.api._http.connections = mock_conns.copy()
524         self.api.users().create(body={}).execute()
525         for c in mock_conns.values():
526             self.assertEqual(c.close.call_count, expect)
527
528
529 class ThreadSafeAPIClientTestCase(run_test_server.TestCaseWithServers):
530     MAIN_SERVER = {}
531
532     def test_constructor(self):
533         env_mapping = {
534             key: value
535             for key, value in os.environ.items()
536             if key.startswith('ARVADOS_API_')
537         }
538         extra_params = {
539             'timeout': 299,
540         }
541         base_params = {
542             key[12:].lower(): value
543             for key, value in env_mapping.items()
544         }
545         try:
546             base_params['insecure'] = base_params.pop('host_insecure')
547         except KeyError:
548             pass
549         expected_keep_params = {}
550         for config, params, subtest in [
551                 (None, {}, "default arguments"),
552                 (None, extra_params, "extra params"),
553                 (env_mapping, {}, "explicit config"),
554                 (env_mapping, extra_params, "explicit config and params"),
555                 ({}, base_params, "params only"),
556         ]:
557             with self.subTest(f"test constructor with {subtest}"):
558                 expected_timeout = params.get('timeout', 300)
559                 expected_params = dict(params)
560                 keep_params = dict(expected_keep_params)
561                 client = ThreadSafeAPIClient(config, keep_params, params, 'v1')
562                 self.assertTrue(hasattr(client, 'localapi'), "client missing localapi method")
563                 self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'])
564                 self.assertEqual(client._http.timeout, expected_timeout)
565                 self.assertEqual(params, expected_params,
566                                  "api_params was modified in-place")
567                 self.assertEqual(keep_params, expected_keep_params,
568                                  "keep_params was modified in-place")
569
570     def test_constructor_no_args(self):
571         client = ThreadSafeAPIClient()
572         self.assertTrue(hasattr(client, 'localapi'), "client missing localapi method")
573         self.assertEqual(client.api_token, os.environ['ARVADOS_API_TOKEN'])
574         self.assertTrue(client.insecure)
575
576     def test_constructor_bad_version(self):
577         with self.assertRaises(googleapiclient.errors.UnknownApiNameOrVersion):
578             ThreadSafeAPIClient(version='BadTestVersion')
579
580     def test_pre_v3_0_name(self):
581         from arvados.safeapi import ThreadSafeApiCache
582         self.assertIs(ThreadSafeApiCache, ThreadSafeAPIClient)
583
584
585 if __name__ == '__main__':
586     unittest.main()