X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/78ddad37d72c6c3a728530dc6932fb91f7d81b87..3b12ef6b6d7ff6852f6109ab71dbec382322a686:/sdk/python/arvados/api.py diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py index 48e73f3149..6581a8e9ac 100644 --- a/sdk/python/arvados/api.py +++ b/sdk/python/arvados/api.py @@ -1,20 +1,33 @@ +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +from builtins import range import collections +import http.client import httplib2 import json import logging import os import re +import socket +import time import types import apiclient from apiclient import discovery as apiclient_discovery from apiclient import errors as apiclient_errors -import config -import errors -import util +from . import config +from . import errors +from . import util +from . import cache _logger = logging.getLogger('arvados.api') +MAX_IDLE_CONNECTION_DURATION = 30 +RETRY_DELAY_INITIAL = 2 +RETRY_DELAY_BACKOFF = 2 +RETRY_COUNT = 2 + class OrderedJsonModel(apiclient.model.JsonModel): """Model class for JSON that preserves the contents' order. @@ -35,9 +48,7 @@ class OrderedJsonModel(apiclient.model.JsonModel): return body -def _intercept_http_request(self, uri, **kwargs): - from httplib import BadStatusLine - +def _intercept_http_request(self, uri, method="GET", **kwargs): if (self.max_request_size and kwargs.get('body') and self.max_request_size < len(kwargs['body'])): @@ -50,22 +61,54 @@ def _intercept_http_request(self, uri, **kwargs): kwargs['headers']['X-External-Client'] = '1' kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token - try: - return self.orig_http_request(uri, **kwargs) - except BadStatusLine: - # This is how httplib tells us that it tried to reuse an - # existing connection but it was already closed by the - # server. In that case, yes, we would like to retry. - # Unfortunately, we are not absolutely certain that the - # previous call did not succeed, so this is slightly - # risky. - return self.orig_http_request(uri, **kwargs) + + retryable = method in [ + 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT'] + retry_count = self._retry_count if retryable else 0 + + if (not retryable and + time.time() - self._last_request_time > self._max_keepalive_idle): + # High probability of failure due to connection atrophy. Make + # sure this request [re]opens a new connection by closing and + # forgetting all cached connections first. + for conn in self.connections.values(): + conn.close() + self.connections.clear() + + delay = self._retry_delay_initial + for _ in range(retry_count): + self._last_request_time = time.time() + try: + return self.orig_http_request(uri, method, **kwargs) + except http.client.HTTPException: + _logger.debug("Retrying API request in %d s after HTTP error", + delay, exc_info=True) + except socket.error: + # This is the one case where httplib2 doesn't close the + # underlying connection first. Close all open + # connections, expecting this object only has the one + # connection to the API server. This is safe because + # httplib2 reopens connections when needed. + _logger.debug("Retrying API request in %d s after socket error", + delay, exc_info=True) + for conn in self.connections.values(): + conn.close() + time.sleep(delay) + delay = delay * self._retry_delay_backoff + + self._last_request_time = time.time() + return self.orig_http_request(uri, method, **kwargs) def _patch_http_request(http, api_token): http.arvados_api_token = api_token http.max_request_size = 0 http.orig_http_request = http.request http.request = types.MethodType(_intercept_http_request, http) + http._last_request_time = 0 + http._max_keepalive_idle = MAX_IDLE_CONNECTION_DURATION + http._retry_delay_initial = RETRY_DELAY_INITIAL + http._retry_delay_backoff = RETRY_DELAY_BACKOFF + http._retry_count = RETRY_COUNT return http # Monkey patch discovery._cast() so objects and arrays get serialized @@ -74,6 +117,7 @@ _cast_orig = apiclient_discovery._cast def _cast_objects_too(value, schema_type): global _cast_orig if (type(value) != type('') and + type(value) != type(b'') and (schema_type == 'object' or schema_type == 'array')): return json.dumps(value) else: @@ -90,12 +134,15 @@ def _new_http_error(cls, *args, **kwargs): apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error) def http_cache(data_type): - path = os.environ['HOME'] + '/.cache/arvados/' + data_type + homedir = os.environ.get('HOME') + if not homedir or len(homedir) == 0: + return None + path = homedir + '/.cache/arvados/' + data_type try: util.mkdir_dash_p(path) except OSError: - path = None - return path + return None + return cache.SafeHTTPCache(path, max_age=60*60*24*2) def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs): """Return an apiclient Resources object for an Arvados instance. @@ -155,11 +202,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,)) if 'http' not in kwargs: - http_kwargs = {} - # Prefer system's CA certificates (if available) over httplib2's. - certs_path = '/etc/ssl/certs/ca-certificates.crt' - if os.path.exists(certs_path): - http_kwargs['ca_certs'] = certs_path + http_kwargs = {'ca_certs': util.ca_certs_path()} if cache: http_kwargs['cache'] = http_cache('discovery') if insecure: @@ -168,8 +211,9 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg kwargs['http'] = _patch_http_request(kwargs['http'], token) - svc = apiclient_discovery.build('arvados', version, **kwargs) + svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs) svc.api_token = token + svc.insecure = insecure kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0) kwargs['http'].cache = None return svc