X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/09c19c4c60d6b9353f98202ac7b1782e762eaf54..2668643b7570db96651466250e7a496184f6ef0a:/sdk/python/arvados/api.py diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py index 88596211d4..19154f3e8b 100644 --- a/sdk/python/arvados/api.py +++ b/sdk/python/arvados/api.py @@ -1,6 +1,13 @@ # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 +"""Arvados API client + +The code in this module builds Arvados API client objects you can use to submit +Arvados API requests. This includes extending the underlying HTTP client with +niceties such as caching, X-Request-Id header for tracking, and more. The main +client constructors are `api` and `api_from_config`. +""" from __future__ import absolute_import from future import standard_library @@ -41,7 +48,7 @@ class OrderedJsonModel(apiclient.model.JsonModel): """Model class for JSON that preserves the contents' order. API clients that care about preserving the order of fields in API - server responses can use this model to do so, like this:: + server responses can use this model to do so, like this: from arvados.api import OrderedJsonModel client = arvados.api('v1', ..., model=OrderedJsonModel()) @@ -66,9 +73,6 @@ def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs): self.max_request_size < len(kwargs['body'])): raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size)) - if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true": - headers['X-External-Client'] = '1' - headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token retryable = method in [ @@ -133,6 +137,10 @@ def _patch_http_request(http, api_token): http._request_id = util.new_request_id return http +def _close_connections(self): + for conn in self._http.connections.values(): + conn.close() + # Monkey patch discovery._cast() so objects and arrays get serialized # with json.dumps() instead of str(). _cast_orig = apiclient_discovery._cast @@ -166,128 +174,274 @@ def http_cache(data_type): return None return cache.SafeHTTPCache(path, max_age=60*60*24*2) -def api(version=None, cache=True, host=None, token=None, insecure=False, - request_id=None, timeout=5*60, **kwargs): - """Return an apiclient Resources object for an Arvados instance. +def api_client( + version, + discoveryServiceUrl, + token, + *, + cache=True, + http=None, + insecure=False, + request_id=None, + timeout=5*60, + **kwargs, +): + """Build an Arvados API client + + This function returns a `googleapiclient.discovery.Resource` object + constructed from the given arguments. This is a relatively low-level + interface that requires all the necessary inputs as arguments. Most + users will prefer to use `api` which can accept more flexible inputs. + + Arguments: + + version: str + : A string naming the version of the Arvados API to use. + + discoveryServiceUrl: str + : The URL used to discover APIs passed directly to + `googleapiclient.discovery.build`. + + token: str + : The authentication token to send with each API call. + + Keyword-only arguments: + + cache: bool + : If true, loads the API discovery document from, or saves it to, a cache + on disk (located at `~/.cache/arvados/discovery`). + + http: httplib2.Http | None + : The HTTP client object the API client object will use to make requests. + If not provided, this function will build its own to use. Either way, the + object will be patched as part of the build process. + + insecure: bool + : If true, ignore SSL certificate validation errors. Default `False`. + + request_id: str | None + : Default `X-Request-Id` header value for outgoing requests that + don't already provide one. If `None` or omitted, generate a random + ID. When retrying failed requests, the same ID is used on all + attempts. - :version: - A string naming the version of the Arvados API to use (for - example, 'v1'). + timeout: int + : A timeout value for HTTP requests in seconds. Default 300 (5 minutes). + + Additional keyword arguments will be passed directly to + `googleapiclient.discovery.build`. + """ + if http is None: + http = httplib2.Http( + ca_certs=util.ca_certs_path(), + cache=http_cache('discovery') if cache else None, + disable_ssl_certificate_validation=bool(insecure), + ) + if http.timeout is None: + http.timeout = timeout + http = _patch_http_request(http, token) + + svc = apiclient_discovery.build( + 'arvados', version, + cache_discovery=False, + discoveryServiceUrl=discoveryServiceUrl, + http=http, + **kwargs, + ) + svc.api_token = token + svc.insecure = insecure + svc.request_id = request_id + svc.config = lambda: util.get_config_once(svc) + svc.vocabulary = lambda: util.get_vocabulary_once(svc) + svc.close_connections = types.MethodType(_close_connections, svc) + http.max_request_size = svc._rootDesc.get('maxRequestSize', 0) + http.cache = None + http._request_id = lambda: svc.request_id or util.new_request_id() + return svc - :cache: - Use a cache (~/.cache/arvados/discovery) for the discovery - document. +def normalize_api_kwargs( + version=None, + discoveryServiceUrl=None, + host=None, + token=None, + **kwargs, +): + """Validate kwargs from `api` and build kwargs for `api_client` - :host: - The Arvados API server host (and optional :port) to connect to. + This method takes high-level keyword arguments passed to the `api` + constructor and normalizes them into a new dictionary that can be passed + as keyword arguments to `api_client`. It raises `ValueError` if required + arguments are missing or conflict. - :token: - The authentication token to send with each API call. + Arguments: - :insecure: - If True, ignore SSL certificate validation errors. + version: str | None + : A string naming the version of the Arvados API to use. If not specified, + the code will log a warning and fall back to 'v1'. - :timeout: - A timeout value for http requests. + discoveryServiceUrl: str | None + : The URL used to discover APIs passed directly to + `googleapiclient.discovery.build`. It is an error to pass both + `discoveryServiceUrl` and `host`. - :request_id: - Default X-Request-Id header value for outgoing requests that - don't already provide one. If None or omitted, generate a random - ID. When retrying failed requests, the same ID is used on all - attempts. + host: str | None + : The hostname and optional port number of the Arvados API server. Used to + build `discoveryServiceUrl`. It is an error to pass both + `discoveryServiceUrl` and `host`. - Additional keyword arguments will be passed directly to - `apiclient_discovery.build` if a new Resource object is created. - If the `discoveryServiceUrl` or `http` keyword arguments are - missing, this function will set default values for them, based on - the current Arvados configuration settings. + token: str + : The authentication token to send with each API call. + Additional keyword arguments will be included in the return value. """ - + if discoveryServiceUrl and host: + raise ValueError("both discoveryServiceUrl and host provided") + elif discoveryServiceUrl: + url_src = "discoveryServiceUrl" + elif host: + url_src = "host argument" + discoveryServiceUrl = 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,) + elif token: + # This specific error message gets priority for backwards compatibility. + raise ValueError("token argument provided, but host missing.") + else: + raise ValueError("neither discoveryServiceUrl nor host provided") + if not token: + raise ValueError("%s provided, but token missing" % (url_src,)) if not version: version = 'v1' - _logger.info("Using default API version. " + - "Call arvados.api('%s') instead." % - version) - if 'discoveryServiceUrl' in kwargs: - if host: - raise ValueError("both discoveryServiceUrl and host provided") - # Here we can't use a token from environment, config file, - # etc. Those probably have nothing to do with the host - # provided by the caller. - if not token: - raise ValueError("discoveryServiceUrl provided, but token missing") - elif host and token: - pass - elif not host and not token: - return api_from_config( - version=version, cache=cache, timeout=timeout, - request_id=request_id, **kwargs) + _logger.info( + "Using default API version. Call arvados.api(%r) instead.", + version, + ) + return { + 'discoveryServiceUrl': discoveryServiceUrl, + 'token': token, + 'version': version, + **kwargs, + } + +def api_kwargs_from_config(version=None, apiconfig=None, **kwargs): + """Build `api_client` keyword arguments from configuration + + This function accepts a mapping with Arvados configuration settings like + `ARVADOS_API_HOST` and converts them into a mapping of keyword arguments + that can be passed to `api_client`. If `ARVADOS_API_HOST` or + `ARVADOS_API_TOKEN` are not configured, it raises `ValueError`. + + Arguments: + + version: str | None + : A string naming the version of the Arvados API to use. If not specified, + the code will log a warning and fall back to 'v1'. + + apiconfig: Mapping[str, str] | None + : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and + optionally `ARVADOS_API_HOST_INSECURE`. If not provided, calls + `arvados.config.settings` to get these parameters from user configuration. + + Additional keyword arguments will be included in the return value. + """ + if apiconfig is None: + apiconfig = config.settings() + missing = " and ".join( + key + for key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN'] + if key not in apiconfig + ) + if missing: + raise ValueError( + "%s not set.\nPlease set in %s or export environment variable." % + (missing, config.default_config_file), + ) + return normalize_api_kwargs( + version, + None, + apiconfig['ARVADOS_API_HOST'], + apiconfig['ARVADOS_API_TOKEN'], + insecure=config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig), + **kwargs, + ) + +def api(version=None, cache=True, host=None, token=None, insecure=False, + request_id=None, timeout=5*60, *, + discoveryServiceUrl=None, **kwargs): + """Dynamically build an Arvados API client + + This function provides a high-level "do what I mean" interface to build an + Arvados API client object. You can call it with no arguments to build a + client from user configuration; pass `host` and `token` arguments just + like you would write in user configuration; or pass additional arguments + for lower-level control over the client. + + This function returns a `arvados.safeapi.ThreadSafeApiCache`, an + API-compatible wrapper around `googleapiclient.discovery.Resource`. If + you're handling concurrency yourself and/or your application is very + performance-sensitive, consider calling `api_client` directly. + + Arguments: + + version: str | None + : A string naming the version of the Arvados API to use. If not specified, + the code will log a warning and fall back to 'v1'. + + host: str | None + : The hostname and optional port number of the Arvados API server. + + token: str | None + : The authentication token to send with each API call. + + discoveryServiceUrl: str | None + : The URL used to discover APIs passed directly to + `googleapiclient.discovery.build`. + + If `host`, `token`, and `discoveryServiceUrl` are all omitted, `host` and + `token` will be loaded from the user's configuration. Otherwise, you must + pass `token` and one of `host` or `discoveryServiceUrl`. It is an error to + pass both `host` and `discoveryServiceUrl`. + + Other arguments are passed directly to `api_client`. See that function's + docstring for more information about their meaning. + """ + kwargs.update( + cache=cache, + insecure=insecure, + request_id=request_id, + timeout=timeout, + ) + if discoveryServiceUrl or host or token: + kwargs.update(normalize_api_kwargs(version, discoveryServiceUrl, host, token)) else: - # Caller provided one but not the other - if not host: - raise ValueError("token argument provided, but host missing.") - else: - raise ValueError("host argument provided, but token missing.") - - if host: - # Caller wants us to build the discoveryServiceUrl - kwargs['discoveryServiceUrl'] = ( - 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,)) - - if 'http' not in kwargs: - http_kwargs = {'ca_certs': util.ca_certs_path()} - if cache: - http_kwargs['cache'] = http_cache('discovery') - if insecure: - http_kwargs['disable_ssl_certificate_validation'] = True - kwargs['http'] = httplib2.Http(**http_kwargs) - - if kwargs['http'].timeout is None: - kwargs['http'].timeout = timeout - - kwargs['http'] = _patch_http_request(kwargs['http'], token) - - svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs) - svc.api_token = token - svc.insecure = insecure - svc.request_id = request_id - svc.config = lambda: util.get_config_once(svc) - kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0) - kwargs['http'].cache = None - kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id() - return svc + kwargs.update(api_kwargs_from_config(version)) + version = kwargs.pop('version') + # We do the import here to avoid a circular import at the top level. + from .safeapi import ThreadSafeApiCache + return ThreadSafeApiCache({}, {}, kwargs, version) def api_from_config(version=None, apiconfig=None, **kwargs): - """Return an apiclient Resources object enabling access to an Arvados server - instance. + """Build an Arvados API client from a configuration mapping - :version: - A string naming the version of the Arvados REST API to use (for - example, 'v1'). + This function builds an Arvados API client from a mapping with user + configuration. It accepts that mapping as an argument, so you can use a + configuration that's different from what the user has set up. - :apiconfig: - If provided, this should be a dict-like object (must support the get() - method) with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and - optionally ARVADOS_API_HOST_INSECURE. If not provided, use - arvados.config (which gets these parameters from the environment by - default.) + This function returns a `arvados.safeapi.ThreadSafeApiCache`, an + API-compatible wrapper around `googleapiclient.discovery.Resource`. If + you're handling concurrency yourself and/or your application is very + performance-sensitive, consider calling `api_client` directly. - Other keyword arguments such as `cache` will be passed along `api()` + Arguments: - """ - # Load from user configuration or environment - if apiconfig is None: - apiconfig = config.settings() + version: str | None + : A string naming the version of the Arvados API to use. If not specified, + the code will log a warning and fall back to 'v1'. - errors = [] - for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']: - if x not in apiconfig: - errors.append(x) - if errors: - raise ValueError(" and ".join(errors)+" not set.\nPlease set in %s or export environment variable." % config.default_config_file) - host = apiconfig.get('ARVADOS_API_HOST') - token = apiconfig.get('ARVADOS_API_TOKEN') - insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig) - - return api(version=version, host=host, token=token, insecure=insecure, **kwargs) + apiconfig: Mapping[str, str] | None + : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and + optionally `ARVADOS_API_HOST_INSECURE`. If not provided, calls + `arvados.config.settings` to get these parameters from user configuration. + + Other arguments are passed directly to `api_client`. See that function's + docstring for more information about their meaning. + """ + return api(**api_kwargs_from_config(version, apiconfig, **kwargs))