X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a8378b8deaa2bbf9d2c154d9d9bb072538c288cc..b6d7efab2c4bffa3fabd55b166e44cca8ac1391f:/sdk/python/arvados/api.py diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py index d1263e24f2..ae687c50bd 100644 --- a/sdk/python/arvados/api.py +++ b/sdk/python/arvados/api.py @@ -1,5 +1,13 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +from builtins import range import collections -import httplib +import http.client import httplib2 import json import logging @@ -12,10 +20,10 @@ import types import apiclient from apiclient import discovery as apiclient_discovery from apiclient import errors as apiclient_errors -import config -import errors -import util -import cache +from . import config +from . import errors +from . import util +from . import cache _logger = logging.getLogger('arvados.api') @@ -44,21 +52,20 @@ class OrderedJsonModel(apiclient.model.JsonModel): return body -def _intercept_http_request(self, uri, **kwargs): +def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs): if (self.max_request_size and kwargs.get('body') and self.max_request_size < len(kwargs['body'])): raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size)) - if 'headers' not in kwargs: - kwargs['headers'] = {} - if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true": - kwargs['headers']['X-External-Client'] = '1' + headers['X-External-Client'] = '1' - kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token + headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token + if not headers.get('X-Request-Id'): + headers['X-Request-Id'] = self._request_id() - retryable = kwargs.get('method', 'GET') in [ + retryable = method in [ 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT'] retry_count = self._retry_count if retryable else 0 @@ -67,7 +74,7 @@ def _intercept_http_request(self, uri, **kwargs): # High probability of failure due to connection atrophy. Make # sure this request [re]opens a new connection by closing and # forgetting all cached connections first. - for conn in self.connections.itervalues(): + for conn in self.connections.values(): conn.close() self.connections.clear() @@ -75,8 +82,8 @@ def _intercept_http_request(self, uri, **kwargs): for _ in range(retry_count): self._last_request_time = time.time() try: - return self.orig_http_request(uri, **kwargs) - except httplib.HTTPException: + return self.orig_http_request(uri, method, headers=headers, **kwargs) + except http.client.HTTPException: _logger.debug("Retrying API request in %d s after HTTP error", delay, exc_info=True) except socket.error: @@ -87,13 +94,17 @@ def _intercept_http_request(self, uri, **kwargs): # httplib2 reopens connections when needed. _logger.debug("Retrying API request in %d s after socket error", delay, exc_info=True) - for conn in self.connections.itervalues(): + for conn in self.connections.values(): conn.close() + except httplib2.SSLHandshakeError as e: + # Intercept and re-raise with a better error message. + raise httplib2.SSLHandshakeError("Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e)) + time.sleep(delay) delay = delay * self._retry_delay_backoff self._last_request_time = time.time() - return self.orig_http_request(uri, **kwargs) + return self.orig_http_request(uri, method, headers=headers, **kwargs) def _patch_http_request(http, api_token): http.arvados_api_token = api_token @@ -105,6 +116,7 @@ def _patch_http_request(http, api_token): http._retry_delay_initial = RETRY_DELAY_INITIAL http._retry_delay_backoff = RETRY_DELAY_BACKOFF http._retry_count = RETRY_COUNT + http._request_id = util.new_request_id return http # Monkey patch discovery._cast() so objects and arrays get serialized @@ -113,6 +125,7 @@ _cast_orig = apiclient_discovery._cast def _cast_objects_too(value, schema_type): global _cast_orig if (type(value) != type('') and + type(value) != type(b'') and (schema_type == 'object' or schema_type == 'array')): return json.dumps(value) else: @@ -136,10 +149,11 @@ def http_cache(data_type): try: util.mkdir_dash_p(path) except OSError: - path = None + return None return cache.SafeHTTPCache(path, max_age=60*60*24*2) -def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs): +def api(version=None, cache=True, host=None, token=None, insecure=False, + request_id=None, timeout=5*60, **kwargs): """Return an apiclient Resources object for an Arvados instance. :version: @@ -159,6 +173,15 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg :insecure: If True, ignore SSL certificate validation errors. + :timeout: + A timeout value for http requests. + + :request_id: + Default X-Request-Id header value for outgoing requests that + don't already provide one. If None or omitted, generate a random + ID. When retrying failed requests, the same ID is used on all + attempts. + Additional keyword arguments will be passed directly to `apiclient_discovery.build` if a new Resource object is created. If the `discoveryServiceUrl` or `http` keyword arguments are @@ -183,7 +206,8 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg elif host and token: pass elif not host and not token: - return api_from_config(version=version, cache=cache, **kwargs) + return api_from_config( + version=version, cache=cache, request_id=request_id, **kwargs) else: # Caller provided one but not the other if not host: @@ -204,13 +228,19 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg http_kwargs['disable_ssl_certificate_validation'] = True kwargs['http'] = httplib2.Http(**http_kwargs) + if kwargs['http'].timeout is None: + kwargs['http'].timeout = timeout + kwargs['http'] = _patch_http_request(kwargs['http'], token) svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs) svc.api_token = token svc.insecure = insecure + svc.request_id = request_id + svc.config = lambda: util.get_config_once(svc) kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0) kwargs['http'].cache = None + kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id() return svc def api_from_config(version=None, apiconfig=None, **kwargs): @@ -235,9 +265,12 @@ def api_from_config(version=None, apiconfig=None, **kwargs): if apiconfig is None: apiconfig = config.settings() + errors = [] for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']: if x not in apiconfig: - raise ValueError("%s is not set. Aborting." % x) + errors.append(x) + if errors: + raise ValueError(" and ".join(errors)+" not set.\nPlease set in %s or export environment variable." % config.default_config_file) host = apiconfig.get('ARVADOS_API_HOST') token = apiconfig.get('ARVADOS_API_TOKEN') insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig)