X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/01463a6fad519def1d3558ddb9c518a605fc9cc2..5fbc413ccdabc9c7551f13e5a76573d72c438251:/sdk/python/arvados/api.py diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py index 3df24306df..3968f0185e 100644 --- a/sdk/python/arvados/api.py +++ b/sdk/python/arvados/api.py @@ -6,48 +6,50 @@ import re import types import apiclient -import apiclient.discovery -import apiclient.errors +from apiclient import discovery as apiclient_discovery +from apiclient import errors as apiclient_errors import config import errors import util _logger = logging.getLogger('arvados.api') -conncache = {} - -class CredentialsFromToken(object): - def __init__(self, api_token): - self.api_token = api_token - - @staticmethod - def http_request(self, uri, **kwargs): - from httplib import BadStatusLine - if 'headers' not in kwargs: - kwargs['headers'] = {} - - if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true": - kwargs['headers']['X-External-Client'] = '1' - - kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token - try: - return self.orig_http_request(uri, **kwargs) - except BadStatusLine: - # This is how httplib tells us that it tried to reuse an - # existing connection but it was already closed by the - # server. In that case, yes, we would like to retry. - # Unfortunately, we are not absolutely certain that the - # previous call did not succeed, so this is slightly - # risky. - return self.orig_http_request(uri, **kwargs) - def authorize(self, http): - http.arvados_api_token = self.api_token - http.orig_http_request = http.request - http.request = types.MethodType(self.http_request, http) - return http + +def _intercept_http_request(self, uri, **kwargs): + from httplib import BadStatusLine + + if (self.max_request_size and + kwargs.get('body') and + self.max_request_size < len(kwargs['body'])): + raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size)) + + if 'headers' not in kwargs: + kwargs['headers'] = {} + + if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true": + kwargs['headers']['X-External-Client'] = '1' + + kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token + try: + return self.orig_http_request(uri, **kwargs) + except BadStatusLine: + # This is how httplib tells us that it tried to reuse an + # existing connection but it was already closed by the + # server. In that case, yes, we would like to retry. + # Unfortunately, we are not absolutely certain that the + # previous call did not succeed, so this is slightly + # risky. + return self.orig_http_request(uri, **kwargs) + +def _patch_http_request(http, api_token): + http.arvados_api_token = api_token + http.max_request_size = 0 + http.orig_http_request = http.request + http.request = types.MethodType(_intercept_http_request, http) + return http # Monkey patch discovery._cast() so objects and arrays get serialized # with json.dumps() instead of str(). -_cast_orig = apiclient.discovery._cast +_cast_orig = apiclient_discovery._cast def _cast_objects_too(value, schema_type): global _cast_orig if (type(value) != type('') and @@ -55,16 +57,16 @@ def _cast_objects_too(value, schema_type): return json.dumps(value) else: return _cast_orig(value, schema_type) -apiclient.discovery._cast = _cast_objects_too +apiclient_discovery._cast = _cast_objects_too # Convert apiclient's HttpErrors into our own API error subclass for better # error reporting. -# Reassigning apiclient.errors.HttpError is not sufficient because most of the +# Reassigning apiclient_errors.HttpError is not sufficient because most of the # apiclient submodules import the class into their own namespace. def _new_http_error(cls, *args, **kwargs): - return super(apiclient.errors.HttpError, cls).__new__( + return super(apiclient_errors.HttpError, cls).__new__( errors.ApiError, *args, **kwargs) -apiclient.errors.HttpError.__new__ = staticmethod(_new_http_error) +apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error) def http_cache(data_type): path = os.environ['HOME'] + '/.cache/arvados/' + data_type @@ -77,20 +79,25 @@ def http_cache(data_type): def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs): """Return an apiclient Resources object for an Arvados instance. - Arguments: - * version: A string naming the version of the Arvados API to use (for + :version: + A string naming the version of the Arvados API to use (for example, 'v1'). - * cache: If True (default), return an existing Resources object if - one already exists with the same endpoint and credentials. If - False, create a new one, and do not keep it in the cache (i.e., - do not return it from subsequent api(cache=True) calls with - matching endpoint and credentials). - * host: The Arvados API server host (and optional :port) to connect to. - * token: The authentication token to send with each API call. - * insecure: If True, ignore SSL certificate validation errors. + + :cache: + Use a cache (~/.cache/arvados/discovery) for the discovery + document. + + :host: + The Arvados API server host (and optional :port) to connect to. + + :token: + The authentication token to send with each API call. + + :insecure: + If True, ignore SSL certificate validation errors. Additional keyword arguments will be passed directly to - `apiclient.discovery.build` if a new Resource object is created. + `apiclient_discovery.build` if a new Resource object is created. If the `discoveryServiceUrl` or `http` keyword arguments are missing, this function will set default values for them, based on the current Arvados configuration settings. @@ -99,7 +106,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg if not version: version = 'v1' - logging.info("Using default API version. " + + _logger.info("Using default API version. " + "Call arvados.api('%s') instead." % version) if 'discoveryServiceUrl' in kwargs: @@ -113,14 +120,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg elif host and token: pass elif not host and not token: - # Load from user configuration or environment - for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']: - if x not in config.settings(): - raise ValueError("%s is not set. Aborting." % x) - host = config.get('ARVADOS_API_HOST') - token = config.get('ARVADOS_API_TOKEN') - insecure = (config.get('ARVADOS_API_HOST_INSECURE', '').lower() in - ('yes', 'true', '1')) + return api_from_config(version=version, cache=cache, **kwargs) else: # Caller provided one but not the other if not host: @@ -133,12 +133,6 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg kwargs['discoveryServiceUrl'] = ( 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,)) - if cache: - connprofile = (version, host, token, insecure) - svc = conncache.get(connprofile) - if svc: - return svc - if 'http' not in kwargs: http_kwargs = {} # Prefer system's CA certificates (if available) over httplib2's. @@ -151,11 +145,41 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg http_kwargs['disable_ssl_certificate_validation'] = True kwargs['http'] = httplib2.Http(**http_kwargs) - credentials = CredentialsFromToken(api_token=token) - kwargs['http'] = credentials.authorize(kwargs['http']) + kwargs['http'] = _patch_http_request(kwargs['http'], token) - svc = apiclient.discovery.build('arvados', version, **kwargs) + svc = apiclient_discovery.build('arvados', version, **kwargs) + svc.api_token = token + kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0) kwargs['http'].cache = None - if cache: - conncache[connprofile] = svc return svc + +def api_from_config(version=None, apiconfig=None, **kwargs): + """Return an apiclient Resources object enabling access to an Arvados server + instance. + + :version: + A string naming the version of the Arvados REST API to use (for + example, 'v1'). + + :apiconfig: + If provided, this should be a dict-like object (must support the get() + method) with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and + optionally ARVADOS_API_HOST_INSECURE. If not provided, use + arvados.config (which gets these parameters from the environment by + default.) + + Other keyword arguments such as `cache` will be passed along `api()` + + """ + # Load from user configuration or environment + if apiconfig is None: + apiconfig = config.settings() + + for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']: + if x not in apiconfig: + raise ValueError("%s is not set. Aborting." % x) + host = apiconfig.get('ARVADOS_API_HOST') + token = apiconfig.get('ARVADOS_API_TOKEN') + insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig) + + return api(version=version, host=host, token=token, insecure=insecure, **kwargs)