"""
import collections
+import errno
+import hashlib
import httplib2
import json
import logging
import socket
import ssl
import sys
+import tempfile
import threading
import time
import types
from apiclient import errors as apiclient_errors
from . import config
from . import errors
+from . import keep
from . import retry
from . import util
-from . import cache
+from ._internal import basedirs
from .logging import GoogleHTTPClientFilter, log_handler
_logger = logging.getLogger('arvados.api')
self.max_request_size < len(kwargs['body'])):
raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
- headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
+ headers['Authorization'] = 'Bearer %s' % self.arvados_api_token
if (time.time() - self._last_request_time) > self._max_keepalive_idle:
# High probability of failure due to connection atrophy. Make
self._last_request_time = time.time()
try:
response, body = self.orig_http_request(uri, method, headers=headers, **kwargs)
- except ssl.SSLCertVerificationError as e:
- raise ssl.SSLCertVerificationError(e.args[0], "Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e)) from None
+ except ssl.CertificateError as e:
+ raise ssl.CertificateError(e.args[0], "Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e)) from None
# googleapiclient only retries 403, 429, and 5xx status codes.
# If we got another 4xx status that we want to retry, convert it into
# 5xx so googleapiclient handles it the way we want.
errors.ApiError, *args, **kwargs)
apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
-def http_cache(data_type: str) -> cache.SafeHTTPCache:
+class ThreadSafeHTTPCache:
+ """Thread-safe replacement for `httplib2.FileCache`
+
+ `arvados.api.http_cache` is the preferred way to construct this object.
+ Refer to that function's docstring for details.
+ """
+
+ def __init__(self, path=None, max_age=None):
+ self._dir = path
+ if max_age is not None:
+ try:
+ self._clean(threshold=time.time() - max_age)
+ except:
+ pass
+
+ def _clean(self, threshold=0):
+ for ent in os.listdir(self._dir):
+ fnm = os.path.join(self._dir, ent)
+ if os.path.isdir(fnm) or not fnm.endswith('.tmp'):
+ continue
+ stat = os.lstat(fnm)
+ if stat.st_mtime < threshold:
+ try:
+ os.unlink(fnm)
+ except OSError as err:
+ if err.errno != errno.ENOENT:
+ raise
+
+ def __str__(self):
+ return self._dir
+
+ def _filename(self, url):
+ return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp')
+
+ def get(self, url):
+ filename = self._filename(url)
+ try:
+ with open(filename, 'rb') as f:
+ return f.read()
+ except (IOError, OSError):
+ return None
+
+ def set(self, url, content):
+ try:
+ fd, tempname = tempfile.mkstemp(dir=self._dir)
+ except:
+ return None
+ try:
+ try:
+ f = os.fdopen(fd, 'wb')
+ except:
+ os.close(fd)
+ raise
+ try:
+ f.write(content)
+ finally:
+ f.close()
+ os.rename(tempname, self._filename(url))
+ tempname = None
+ finally:
+ if tempname:
+ os.unlink(tempname)
+
+ def delete(self, url):
+ try:
+ os.unlink(self._filename(url))
+ except OSError as err:
+ if err.errno != errno.ENOENT:
+ raise
+
+
+class ThreadSafeAPIClient(object):
+ """Thread-safe wrapper for an Arvados API client
+
+ This class takes all the arguments necessary to build a lower-level
+ Arvados API client `googleapiclient.discovery.Resource`, then
+ transparently builds and wraps a unique object per thread. This works
+ around the fact that the client's underlying HTTP client object is not
+ thread-safe.
+
+ Arguments:
+
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. If not provided, uses
+ `arvados.config.settings` to get these parameters from user
+ configuration. You can pass an empty mapping to build the client
+ solely from `api_params`.
+
+ * keep_params: Mapping[str, Any] --- Keyword arguments used to construct
+ an associated `arvados.keep.KeepClient`.
+
+ * api_params: Mapping[str, Any] --- Keyword arguments used to construct
+ each thread's API client. These have the same meaning as in the
+ `arvados.api.api` function.
+
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ `'v1'`.
+ """
+ def __init__(
+ self,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ keep_params: Optional[Mapping[str, Any]]={},
+ api_params: Optional[Mapping[str, Any]]={},
+ version: Optional[str]=None,
+ ) -> None:
+ if apiconfig or apiconfig is None:
+ self._api_kwargs = api_kwargs_from_config(version, apiconfig, **api_params)
+ else:
+ self._api_kwargs = normalize_api_kwargs(version, **api_params)
+ self.api_token = self._api_kwargs['token']
+ self.request_id = self._api_kwargs.get('request_id')
+ self.local = threading.local()
+ self.keep = keep.KeepClient(api_client=self, **keep_params)
+
+ def localapi(self) -> 'googleapiclient.discovery.Resource':
+ try:
+ client = self.local.api
+ except AttributeError:
+ client = api_client(**self._api_kwargs)
+ client._http._request_id = lambda: self.request_id or util.new_request_id()
+ self.local.api = client
+ return client
+
+ def __getattr__(self, name: str) -> Any:
+ # Proxy nonexistent attributes to the thread-local API client.
+ return getattr(self.localapi(), name)
+
+
+def http_cache(data_type: str) -> Optional[ThreadSafeHTTPCache]:
"""Set up an HTTP file cache
- This function constructs and returns an `arvados.cache.SafeHTTPCache`
- backed by the filesystem under `~/.cache/arvados/`, or `None` if the
- directory cannot be set up. The return value can be passed to
+ This function constructs and returns an `arvados.api.ThreadSafeHTTPCache`
+ backed by the filesystem under a cache directory from the environment, or
+ `None` if the directory cannot be set up. The return value can be passed to
`httplib2.Http` as the `cache` argument.
Arguments:
- * data_type: str --- The name of the subdirectory under `~/.cache/arvados`
+ * data_type: str --- The name of the subdirectory
where data is cached.
"""
try:
- homedir = pathlib.Path.home()
- except RuntimeError:
- return None
- path = pathlib.Path(homedir, '.cache', 'arvados', data_type)
- try:
- path.mkdir(parents=True, exist_ok=True)
- except OSError:
+ path = basedirs.BaseDirectories('CACHE').storage_path(data_type)
+ except (OSError, RuntimeError):
return None
- return cache.SafeHTTPCache(str(path), max_age=60*60*24*2)
+ else:
+ return ThreadSafeHTTPCache(str(path), max_age=60*60*24*2)
def api_client(
version: str,
Keyword-only arguments:
* cache: bool --- If true, loads the API discovery document from, or
- saves it to, a cache on disk (located at
- `~/.cache/arvados/discovery`).
+ saves it to, a cache on disk.
* http: httplib2.Http | None --- The HTTP client object the API client
object will use to make requests. If not provided, this function will
*,
discoveryServiceUrl: Optional[str]=None,
**kwargs: Any,
-) -> 'arvados.safeapi.ThreadSafeApiCache':
+) -> ThreadSafeAPIClient:
"""Dynamically build an Arvados API client
This function provides a high-level "do what I mean" interface to build an
like you would write in user configuration; or pass additional arguments
for lower-level control over the client.
- This function returns a `arvados.safeapi.ThreadSafeApiCache`, an
+ This function returns a `arvados.api.ThreadSafeAPIClient`, an
API-compatible wrapper around `googleapiclient.discovery.Resource`. If
you're handling concurrency yourself and/or your application is very
performance-sensitive, consider calling `api_client` directly.
else:
kwargs.update(api_kwargs_from_config(version))
version = kwargs.pop('version')
- # We do the import here to avoid a circular import at the top level.
- from .safeapi import ThreadSafeApiCache
- return ThreadSafeApiCache({}, {}, kwargs, version)
+ return ThreadSafeAPIClient({}, {}, kwargs, version)
def api_from_config(
version: Optional[str]=None,
apiconfig: Optional[Mapping[str, str]]=None,
**kwargs: Any
-) -> 'arvados.safeapi.ThreadSafeApiCache':
+) -> ThreadSafeAPIClient:
"""Build an Arvados API client from a configuration mapping
This function builds an Arvados API client from a mapping with user
configuration. It accepts that mapping as an argument, so you can use a
configuration that's different from what the user has set up.
- This function returns a `arvados.safeapi.ThreadSafeApiCache`, an
+ This function returns a `arvados.api.ThreadSafeAPIClient`, an
API-compatible wrapper around `googleapiclient.discovery.Resource`. If
you're handling concurrency yourself and/or your application is very
performance-sensitive, consider calling `api_client` directly.
docstring for more information about their meaning.
"""
return api(**api_kwargs_from_config(version, apiconfig, **kwargs))
-
-class OrderedJsonModel(apiclient.model.JsonModel):
- """Model class for JSON that preserves the contents' order
-
- .. WARNING:: Deprecated
- This model is redundant now that Python dictionaries preserve insertion
- ordering. Code that passes this model to API constructors can remove it.
-
- In Python versions before 3.6, API clients that cared about preserving the
- order of fields in API server responses could use this model to do so.
- Typical usage looked like:
-
- from arvados.api import OrderedJsonModel
- client = arvados.api('v1', ..., model=OrderedJsonModel())
- """
- @util._deprecated(preferred="the default model and rely on Python's built-in dictionary ordering")
- def __init__(self, data_wrapper=False):
- return super().__init__(data_wrapper)
-
-
-RETRY_DELAY_INITIAL = 0
-"""
-.. WARNING:: Deprecated
- This constant was used by retry code in previous versions of the Arvados SDK.
- Changing the value has no effect anymore.
- Prefer passing `num_retries` to an API client constructor instead.
- Refer to the constructor docstrings for details.
-"""
-
-RETRY_DELAY_BACKOFF = 0
-"""
-.. WARNING:: Deprecated
- This constant was used by retry code in previous versions of the Arvados SDK.
- Changing the value has no effect anymore.
- Prefer passing `num_retries` to an API client constructor instead.
- Refer to the constructor docstrings for details.
-"""
-
-RETRY_COUNT = 0
-"""
-.. WARNING:: Deprecated
- This constant was used by retry code in previous versions of the Arvados SDK.
- Changing the value has no effect anymore.
- Prefer passing `num_retries` to an API client constructor instead.
- Refer to the constructor docstrings for details.
-"""
projects / arvados.git / blobdiff