X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/055b9792b7692d6c42f4e13d38dc6cd008396a6f..d49403ac749fffe24ac6357597e6014bb65efa30:/sdk/python/arvados/api.py diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py index 86d24dfc06..db1d0f4e12 100644 --- a/sdk/python/arvados/api.py +++ b/sdk/python/arvados/api.py @@ -14,6 +14,7 @@ import logging import os import re import socket +import ssl import sys import time import types @@ -57,58 +58,67 @@ class OrderedJsonModel(apiclient.model.JsonModel): def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs): - if (self.max_request_size and - kwargs.get('body') and - self.max_request_size < len(kwargs['body'])): - raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size)) - - if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true": - headers['X-External-Client'] = '1' - - headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token if not headers.get('X-Request-Id'): headers['X-Request-Id'] = self._request_id() + try: + if (self.max_request_size and + kwargs.get('body') and + self.max_request_size < len(kwargs['body'])): + raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size)) - retryable = method in [ - 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT'] - retry_count = self._retry_count if retryable else 0 - - if (not retryable and - time.time() - self._last_request_time > self._max_keepalive_idle): - # High probability of failure due to connection atrophy. Make - # sure this request [re]opens a new connection by closing and - # forgetting all cached connections first. - for conn in self.connections.values(): - conn.close() - self.connections.clear() - - delay = self._retry_delay_initial - for _ in range(retry_count): - self._last_request_time = time.time() - try: - return self.orig_http_request(uri, method, headers=headers, **kwargs) - except http.client.HTTPException: - _logger.debug("Retrying API request in %d s after HTTP error", - delay, exc_info=True) - except socket.error: - # This is the one case where httplib2 doesn't close the - # underlying connection first. Close all open - # connections, expecting this object only has the one - # connection to the API server. This is safe because - # httplib2 reopens connections when needed. - _logger.debug("Retrying API request in %d s after socket error", - delay, exc_info=True) + if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true": + headers['X-External-Client'] = '1' + + headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token + + retryable = method in [ + 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT'] + retry_count = self._retry_count if retryable else 0 + + if (not retryable and + time.time() - self._last_request_time > self._max_keepalive_idle): + # High probability of failure due to connection atrophy. Make + # sure this request [re]opens a new connection by closing and + # forgetting all cached connections first. for conn in self.connections.values(): conn.close() - except httplib2.SSLHandshakeError as e: - # Intercept and re-raise with a better error message. - raise httplib2.SSLHandshakeError("Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e)) - - time.sleep(delay) - delay = delay * self._retry_delay_backoff + self.connections.clear() + + delay = self._retry_delay_initial + for _ in range(retry_count): + self._last_request_time = time.time() + try: + return self.orig_http_request(uri, method, headers=headers, **kwargs) + except http.client.HTTPException: + _logger.debug("[%s] Retrying API request in %d s after HTTP error", + headers['X-Request-Id'], delay, exc_info=True) + except ssl.SSLCertVerificationError as e: + raise ssl.SSLCertVerificationError(e.args[0], "Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e)) from None + except socket.error: + # This is the one case where httplib2 doesn't close the + # underlying connection first. Close all open + # connections, expecting this object only has the one + # connection to the API server. This is safe because + # httplib2 reopens connections when needed. + _logger.debug("[%s] Retrying API request in %d s after socket error", + headers['X-Request-Id'], delay, exc_info=True) + for conn in self.connections.values(): + conn.close() + + time.sleep(delay) + delay = delay * self._retry_delay_backoff - self._last_request_time = time.time() - return self.orig_http_request(uri, method, headers=headers, **kwargs) + self._last_request_time = time.time() + return self.orig_http_request(uri, method, headers=headers, **kwargs) + except Exception as e: + # Prepend "[request_id] " to the error message, which we + # assume is the first string argument passed to the exception + # constructor. + for i in range(len(e.args or ())): + if type(e.args[i]) == type(""): + e.args = e.args[:i] + ("[{}] {}".format(headers['X-Request-Id'], e.args[i]),) + e.args[i+1:] + raise type(e)(*e.args) + raise def _patch_http_request(http, api_token): http.arvados_api_token = api_token @@ -123,6 +133,10 @@ def _patch_http_request(http, api_token): http._request_id = util.new_request_id return http +def _close_connections(self): + for conn in self._http.connections.values(): + conn.close() + # Monkey patch discovery._cast() so objects and arrays get serialized # with json.dumps() instead of str(). _cast_orig = apiclient_discovery._cast @@ -243,6 +257,8 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, svc.insecure = insecure svc.request_id = request_id svc.config = lambda: util.get_config_once(svc) + svc.vocabulary = lambda: util.get_vocabulary_once(svc) + svc.close_connections = types.MethodType(_close_connections, svc) kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0) kwargs['http'].cache = None kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id()