client constructors are `api` and `api_from_config`.
"""
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import range
import collections
-import http.client
import httplib2
import json
import logging
import os
+import pathlib
import re
import socket
import ssl
import sys
+import threading
import time
import types
+from typing import (
+ Any,
+ Dict,
+ List,
+ Mapping,
+ Optional,
+)
+
import apiclient
+import apiclient.http
from apiclient import discovery as apiclient_discovery
from apiclient import errors as apiclient_errors
from . import config
from . import errors
+from . import retry
from . import util
from . import cache
+from .logging import GoogleHTTPClientFilter, log_handler
_logger = logging.getLogger('arvados.api')
+_googleapiclient_log_lock = threading.Lock()
MAX_IDLE_CONNECTION_DURATION = 30
+"""
+Number of seconds that API client HTTP connections should be allowed to idle
+in keepalive state before they are forced closed. Client code can adjust this
+constant, and it will be used for all Arvados API clients constructed after
+that point.
+"""
-# These constants supported our own retry logic that we've since removed in
-# favor of using googleapiclient's num_retries. They're kept here purely for
-# API compatibility, but set to 0 to indicate no retries happen.
-RETRY_DELAY_INITIAL = 0
-RETRY_DELAY_BACKOFF = 0
-RETRY_COUNT = 0
+# An unused HTTP 5xx status code to request a retry internally.
+# See _intercept_http_request. This should not be user-visible.
+_RETRY_4XX_STATUS = 545
if sys.version_info >= (3,):
httplib2.SSLHandshakeError = None
-class OrderedJsonModel(apiclient.model.JsonModel):
- """Model class for JSON that preserves the contents' order.
-
- API clients that care about preserving the order of fields in API
- server responses can use this model to do so, like this:
-
- from arvados.api import OrderedJsonModel
- client = arvados.api('v1', ..., model=OrderedJsonModel())
- """
-
- def deserialize(self, content):
- # This is a very slightly modified version of the parent class'
- # implementation. Copyright (c) 2010 Google.
- content = content.decode('utf-8')
- body = json.loads(content, object_pairs_hook=collections.OrderedDict)
- if self._data_wrapper and isinstance(body, dict) and 'data' in body:
- body = body['data']
- return body
-
+_orig_retry_request = apiclient.http._retry_request
+def _retry_request(http, num_retries, *args, **kwargs):
+ try:
+ num_retries = max(num_retries, http.num_retries)
+ except AttributeError:
+ # `http` client object does not have a `num_retries` attribute.
+ # It apparently hasn't gone through _patch_http_request, possibly
+ # because this isn't an Arvados API client. Pass through to
+ # avoid interfering with other Google API clients.
+ return _orig_retry_request(http, num_retries, *args, **kwargs)
+ response, body = _orig_retry_request(http, num_retries, *args, **kwargs)
+ # If _intercept_http_request ran out of retries for a 4xx response,
+ # restore the original status code.
+ if response.status == _RETRY_4XX_STATUS:
+ response.status = int(response['status'])
+ return (response, body)
+apiclient.http._retry_request = _retry_request
def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs):
if not headers.get('X-Request-Id'):
self._last_request_time = time.time()
try:
- return self.orig_http_request(uri, method, headers=headers, **kwargs)
+ response, body = self.orig_http_request(uri, method, headers=headers, **kwargs)
except ssl.SSLCertVerificationError as e:
raise ssl.SSLCertVerificationError(e.args[0], "Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e)) from None
+ # googleapiclient only retries 403, 429, and 5xx status codes.
+ # If we got another 4xx status that we want to retry, convert it into
+ # 5xx so googleapiclient handles it the way we want.
+ if response.status in retry._HTTP_CAN_RETRY and response.status < 500:
+ response.status = _RETRY_4XX_STATUS
+ return (response, body)
except Exception as e:
# Prepend "[request_id] " to the error message, which we
# assume is the first string argument passed to the exception
raise type(e)(*e.args)
raise
-def _patch_http_request(http, api_token):
+def _patch_http_request(http, api_token, num_retries):
http.arvados_api_token = api_token
http.max_request_size = 0
+ http.num_retries = num_retries
http.orig_http_request = http.request
http.request = types.MethodType(_intercept_http_request, http)
http._last_request_time = 0
errors.ApiError, *args, **kwargs)
apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
-def http_cache(data_type):
- homedir = os.environ.get('HOME')
- if not homedir or len(homedir) == 0:
+def http_cache(data_type: str) -> cache.SafeHTTPCache:
+ """Set up an HTTP file cache
+
+ This function constructs and returns an `arvados.cache.SafeHTTPCache`
+ backed by the filesystem under `~/.cache/arvados/`, or `None` if the
+ directory cannot be set up. The return value can be passed to
+ `httplib2.Http` as the `cache` argument.
+
+ Arguments:
+
+ * data_type: str --- The name of the subdirectory under `~/.cache/arvados`
+ where data is cached.
+ """
+ try:
+ homedir = pathlib.Path.home()
+ except RuntimeError:
return None
- path = homedir + '/.cache/arvados/' + data_type
+ path = pathlib.Path(homedir, '.cache', 'arvados', data_type)
try:
- util.mkdir_dash_p(path)
+ path.mkdir(parents=True, exist_ok=True)
except OSError:
return None
- return cache.SafeHTTPCache(path, max_age=60*60*24*2)
+ return cache.SafeHTTPCache(str(path), max_age=60*60*24*2)
def api_client(
- version,
- discoveryServiceUrl,
- token,
+ version: str,
+ discoveryServiceUrl: str,
+ token: str,
*,
- cache=True,
- http=None,
- insecure=False,
- request_id=None,
- timeout=5*60,
- **kwargs,
-):
+ cache: bool=True,
+ http: Optional[httplib2.Http]=None,
+ insecure: bool=False,
+ num_retries: int=10,
+ request_id: Optional[str]=None,
+ timeout: int=5*60,
+ **kwargs: Any,
+) -> apiclient_discovery.Resource:
"""Build an Arvados API client
This function returns a `googleapiclient.discovery.Resource` object
Arguments:
- version: str
- : A string naming the version of the Arvados API to use.
+ * version: str --- A string naming the version of the Arvados API to use.
- discoveryServiceUrl: str
- : The URL used to discover APIs passed directly to
- `googleapiclient.discovery.build`.
+ * discoveryServiceUrl: str --- The URL used to discover APIs passed
+ directly to `googleapiclient.discovery.build`.
- token: str
- : The authentication token to send with each API call.
+ * token: str --- The authentication token to send with each API call.
Keyword-only arguments:
- cache: bool
- : If true, loads the API discovery document from, or saves it to, a cache
- on disk (located at `~/.cache/arvados/discovery`).
+ * cache: bool --- If true, loads the API discovery document from, or
+ saves it to, a cache on disk (located at
+ `~/.cache/arvados/discovery`).
- http: httplib2.Http | None
- : The HTTP client object the API client object will use to make requests.
- If not provided, this function will build its own to use. Either way, the
- object will be patched as part of the build process.
+ * http: httplib2.Http | None --- The HTTP client object the API client
+ object will use to make requests. If not provided, this function will
+ build its own to use. Either way, the object will be patched as part
+ of the build process.
- insecure: bool
- : If true, ignore SSL certificate validation errors. Default `False`.
+ * insecure: bool --- If true, ignore SSL certificate validation
+ errors. Default `False`.
- request_id: str | None
- : Default `X-Request-Id` header value for outgoing requests that
- don't already provide one. If `None` or omitted, generate a random
- ID. When retrying failed requests, the same ID is used on all
- attempts.
+ * num_retries: int --- The number of times to retry each API request if
+ it encounters a temporary failure. Default 10.
- timeout: int
- : A timeout value for HTTP requests in seconds. Default 300 (5 minutes).
+ * request_id: str | None --- Default `X-Request-Id` header value for
+ outgoing requests that don't already provide one. If `None` or
+ omitted, generate a random ID. When retrying failed requests, the same
+ ID is used on all attempts.
+
+ * timeout: int --- A timeout value for HTTP requests in seconds. Default
+ 300 (5 minutes).
Additional keyword arguments will be passed directly to
`googleapiclient.discovery.build`.
)
if http.timeout is None:
http.timeout = timeout
- http = _patch_http_request(http, token)
-
- svc = apiclient_discovery.build(
- 'arvados', version,
- cache_discovery=False,
- discoveryServiceUrl=discoveryServiceUrl,
- http=http,
- **kwargs,
+ http = _patch_http_request(http, token, num_retries)
+
+ # The first time a client is instantiated, temporarily route
+ # googleapiclient.http retry logs if they're not already. These are
+ # important because temporary problems fetching the discovery document
+ # can cause clients to appear to hang early. This can be removed after
+ # we have a more general story for handling googleapiclient logs (#20521).
+ client_logger = logging.getLogger('googleapiclient.http')
+ # "first time a client is instantiated" = thread that acquires this lock
+ # It is never released.
+ # googleapiclient sets up its own NullHandler so we detect if logging is
+ # configured by looking for a real handler anywhere in the hierarchy.
+ client_logger_unconfigured = _googleapiclient_log_lock.acquire(blocking=False) and all(
+ isinstance(handler, logging.NullHandler)
+ for logger_name in ['', 'googleapiclient', 'googleapiclient.http']
+ for handler in logging.getLogger(logger_name).handlers
)
+ if client_logger_unconfigured:
+ client_level = client_logger.level
+ client_filter = GoogleHTTPClientFilter()
+ client_logger.addFilter(client_filter)
+ client_logger.addHandler(log_handler)
+ if logging.NOTSET < client_level < client_filter.retry_levelno:
+ client_logger.setLevel(client_level)
+ else:
+ client_logger.setLevel(client_filter.retry_levelno)
+ try:
+ svc = apiclient_discovery.build(
+ 'arvados', version,
+ cache_discovery=False,
+ discoveryServiceUrl=discoveryServiceUrl,
+ http=http,
+ num_retries=num_retries,
+ **kwargs,
+ )
+ finally:
+ if client_logger_unconfigured:
+ client_logger.removeHandler(log_handler)
+ client_logger.removeFilter(client_filter)
+ client_logger.setLevel(client_level)
svc.api_token = token
svc.insecure = insecure
svc.request_id = request_id
return svc
def normalize_api_kwargs(
- version=None,
- discoveryServiceUrl=None,
- host=None,
- token=None,
- **kwargs,
-):
+ version: Optional[str]=None,
+ discoveryServiceUrl: Optional[str]=None,
+ host: Optional[str]=None,
+ token: Optional[str]=None,
+ **kwargs: Any,
+) -> Dict[str, Any]:
"""Validate kwargs from `api` and build kwargs for `api_client`
This method takes high-level keyword arguments passed to the `api`
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- discoveryServiceUrl: str | None
- : The URL used to discover APIs passed directly to
- `googleapiclient.discovery.build`. It is an error to pass both
- `discoveryServiceUrl` and `host`.
+ * discoveryServiceUrl: str | None --- The URL used to discover APIs
+ passed directly to `googleapiclient.discovery.build`. It is an error
+ to pass both `discoveryServiceUrl` and `host`.
- host: str | None
- : The hostname and optional port number of the Arvados API server. Used to
- build `discoveryServiceUrl`. It is an error to pass both
- `discoveryServiceUrl` and `host`.
+ * host: str | None --- The hostname and optional port number of the
+ Arvados API server. Used to build `discoveryServiceUrl`. It is an
+ error to pass both `discoveryServiceUrl` and `host`.
- token: str
- : The authentication token to send with each API call.
+ * token: str --- The authentication token to send with each API call.
Additional keyword arguments will be included in the return value.
"""
**kwargs,
}
-def api_kwargs_from_config(version=None, apiconfig=None, **kwargs):
+def api_kwargs_from_config(
+ version: Optional[str]=None,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ **kwargs: Any
+) -> Dict[str, Any]:
"""Build `api_client` keyword arguments from configuration
This function accepts a mapping with Arvados configuration settings like
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- apiconfig: Mapping[str, str] | None
- : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and
- optionally `ARVADOS_API_HOST_INSECURE`. If not provided, calls
- `arvados.config.settings` to get these parameters from user configuration.
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. If not provided, calls
+ `arvados.config.settings` to get these parameters from user
+ configuration.
Additional keyword arguments will be included in the return value.
"""
**kwargs,
)
-def api(version=None, cache=True, host=None, token=None, insecure=False,
- request_id=None, timeout=5*60, *,
- discoveryServiceUrl=None, **kwargs):
+def api(
+ version: Optional[str]=None,
+ cache: bool=True,
+ host: Optional[str]=None,
+ token: Optional[str]=None,
+ insecure: bool=False,
+ request_id: Optional[str]=None,
+ timeout: int=5*60,
+ *,
+ discoveryServiceUrl: Optional[str]=None,
+ **kwargs: Any,
+) -> 'arvados.safeapi.ThreadSafeApiCache':
"""Dynamically build an Arvados API client
This function provides a high-level "do what I mean" interface to build an
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- host: str | None
- : The hostname and optional port number of the Arvados API server.
+ * host: str | None --- The hostname and optional port number of the
+ Arvados API server.
- token: str | None
- : The authentication token to send with each API call.
+ * token: str | None --- The authentication token to send with each API
+ call.
- discoveryServiceUrl: str | None
- : The URL used to discover APIs passed directly to
- `googleapiclient.discovery.build`.
+ * discoveryServiceUrl: str | None --- The URL used to discover APIs
+ passed directly to `googleapiclient.discovery.build`.
If `host`, `token`, and `discoveryServiceUrl` are all omitted, `host` and
`token` will be loaded from the user's configuration. Otherwise, you must
from .safeapi import ThreadSafeApiCache
return ThreadSafeApiCache({}, {}, kwargs, version)
-def api_from_config(version=None, apiconfig=None, **kwargs):
+def api_from_config(
+ version: Optional[str]=None,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ **kwargs: Any
+) -> 'arvados.safeapi.ThreadSafeApiCache':
"""Build an Arvados API client from a configuration mapping
This function builds an Arvados API client from a mapping with user
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- apiconfig: Mapping[str, str] | None
- : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and
- optionally `ARVADOS_API_HOST_INSECURE`. If not provided, calls
- `arvados.config.settings` to get these parameters from user configuration.
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. If not provided, calls
+ `arvados.config.settings` to get these parameters from user
+ configuration.
Other arguments are passed directly to `api_client`. See that function's
docstring for more information about their meaning.
"""
return api(**api_kwargs_from_config(version, apiconfig, **kwargs))
+
+class OrderedJsonModel(apiclient.model.JsonModel):
+ """Model class for JSON that preserves the contents' order
+
+ .. WARNING:: Deprecated
+ This model is redundant now that Python dictionaries preserve insertion
+ ordering. Code that passes this model to API constructors can remove it.
+
+ In Python versions before 3.6, API clients that cared about preserving the
+ order of fields in API server responses could use this model to do so.
+ Typical usage looked like:
+
+ from arvados.api import OrderedJsonModel
+ client = arvados.api('v1', ..., model=OrderedJsonModel())
+ """
+ @util._deprecated(preferred="the default model and rely on Python's built-in dictionary ordering")
+ def __init__(self, data_wrapper=False):
+ return super().__init__(data_wrapper)
+
+
+RETRY_DELAY_INITIAL = 0
+"""
+.. WARNING:: Deprecated
+ This constant was used by retry code in previous versions of the Arvados SDK.
+ Changing the value has no effect anymore.
+ Prefer passing `num_retries` to an API client constructor instead.
+ Refer to the constructor docstrings for details.
+"""
+
+RETRY_DELAY_BACKOFF = 0
+"""
+.. WARNING:: Deprecated
+ This constant was used by retry code in previous versions of the Arvados SDK.
+ Changing the value has no effect anymore.
+ Prefer passing `num_retries` to an API client constructor instead.
+ Refer to the constructor docstrings for details.
+"""
+
+RETRY_COUNT = 0
+"""
+.. WARNING:: Deprecated
+ This constant was used by retry code in previous versions of the Arvados SDK.
+ Changing the value has no effect anymore.
+ Prefer passing `num_retries` to an API client constructor instead.
+ Refer to the constructor docstrings for details.
+"""
projects / arvados.git / blobdiff