projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge branch '12707-storage-classes-columns'
[arvados.git]
/
sdk
/
python
/
arvados
/
api.py
diff --git
a/sdk/python/arvados/api.py
b/sdk/python/arvados/api.py
index ccf16a5fcef3be02b6450bd4b527e6ff682c88d1..4611a1aadf80043eb9afdeeaff727b27a09eecbc 100644
(file)
--- a/
sdk/python/arvados/api.py
+++ b/
sdk/python/arvados/api.py
@@
-1,5
+1,13
@@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
import collections
import collections
-import http
lib
+import http
.client
import httplib2
import json
import logging
import httplib2
import json
import logging
@@
-12,9
+20,10
@@
import types
import apiclient
from apiclient import discovery as apiclient_discovery
from apiclient import errors as apiclient_errors
import apiclient
from apiclient import discovery as apiclient_discovery
from apiclient import errors as apiclient_errors
-import config
-import errors
-import util
+from . import config
+from . import errors
+from . import util
+from . import cache
_logger = logging.getLogger('arvados.api')
_logger = logging.getLogger('arvados.api')
@@
-43,21
+52,20
@@
class OrderedJsonModel(apiclient.model.JsonModel):
return body
return body
-def _intercept_http_request(self, uri, **kwargs):
+def _intercept_http_request(self, uri,
method="GET", headers={},
**kwargs):
if (self.max_request_size and
kwargs.get('body') and
self.max_request_size < len(kwargs['body'])):
raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
if (self.max_request_size and
kwargs.get('body') and
self.max_request_size < len(kwargs['body'])):
raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
- if 'headers' not in kwargs:
- kwargs['headers'] = {}
-
if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
-
kwargs['headers']
['X-External-Client'] = '1'
+
headers
['X-External-Client'] = '1'
- kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
+ headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
+ if not headers.get('X-Request-Id'):
+ headers['X-Request-Id'] = self._request_id()
- retryable =
kwargs.get('method', 'GET')
in [
+ retryable =
method
in [
'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
retry_count = self._retry_count if retryable else 0
'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
retry_count = self._retry_count if retryable else 0
@@
-66,7
+74,7
@@
def _intercept_http_request(self, uri, **kwargs):
# High probability of failure due to connection atrophy. Make
# sure this request [re]opens a new connection by closing and
# forgetting all cached connections first.
# High probability of failure due to connection atrophy. Make
# sure this request [re]opens a new connection by closing and
# forgetting all cached connections first.
- for conn in self.connections.
iter
values():
+ for conn in self.connections.values():
conn.close()
self.connections.clear()
conn.close()
self.connections.clear()
@@
-74,8
+82,8
@@
def _intercept_http_request(self, uri, **kwargs):
for _ in range(retry_count):
self._last_request_time = time.time()
try:
for _ in range(retry_count):
self._last_request_time = time.time()
try:
- return self.orig_http_request(uri, **kwargs)
- except http
lib
.HTTPException:
+ return self.orig_http_request(uri,
method, headers=headers,
**kwargs)
+ except http
.client
.HTTPException:
_logger.debug("Retrying API request in %d s after HTTP error",
delay, exc_info=True)
except socket.error:
_logger.debug("Retrying API request in %d s after HTTP error",
delay, exc_info=True)
except socket.error:
@@
-86,13
+94,13
@@
def _intercept_http_request(self, uri, **kwargs):
# httplib2 reopens connections when needed.
_logger.debug("Retrying API request in %d s after socket error",
delay, exc_info=True)
# httplib2 reopens connections when needed.
_logger.debug("Retrying API request in %d s after socket error",
delay, exc_info=True)
- for conn in self.connections.
iter
values():
+ for conn in self.connections.values():
conn.close()
time.sleep(delay)
delay = delay * self._retry_delay_backoff
self._last_request_time = time.time()
conn.close()
time.sleep(delay)
delay = delay * self._retry_delay_backoff
self._last_request_time = time.time()
- return self.orig_http_request(uri, **kwargs)
+ return self.orig_http_request(uri,
method, headers=headers,
**kwargs)
def _patch_http_request(http, api_token):
http.arvados_api_token = api_token
def _patch_http_request(http, api_token):
http.arvados_api_token = api_token
@@
-104,6
+112,7
@@
def _patch_http_request(http, api_token):
http._retry_delay_initial = RETRY_DELAY_INITIAL
http._retry_delay_backoff = RETRY_DELAY_BACKOFF
http._retry_count = RETRY_COUNT
http._retry_delay_initial = RETRY_DELAY_INITIAL
http._retry_delay_backoff = RETRY_DELAY_BACKOFF
http._retry_count = RETRY_COUNT
+ http._request_id = util.new_request_id
return http
# Monkey patch discovery._cast() so objects and arrays get serialized
return http
# Monkey patch discovery._cast() so objects and arrays get serialized
@@
-112,6
+121,7
@@
_cast_orig = apiclient_discovery._cast
def _cast_objects_too(value, schema_type):
global _cast_orig
if (type(value) != type('') and
def _cast_objects_too(value, schema_type):
global _cast_orig
if (type(value) != type('') and
+ type(value) != type(b'') and
(schema_type == 'object' or schema_type == 'array')):
return json.dumps(value)
else:
(schema_type == 'object' or schema_type == 'array')):
return json.dumps(value)
else:
@@
-135,10
+145,11
@@
def http_cache(data_type):
try:
util.mkdir_dash_p(path)
except OSError:
try:
util.mkdir_dash_p(path)
except OSError:
-
path =
None
- return
path
+
return
None
+ return
cache.SafeHTTPCache(path, max_age=60*60*24*2)
-def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
+def api(version=None, cache=True, host=None, token=None, insecure=False,
+ request_id=None, **kwargs):
"""Return an apiclient Resources object for an Arvados instance.
:version:
"""Return an apiclient Resources object for an Arvados instance.
:version:
@@
-158,6
+169,12
@@
def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
:insecure:
If True, ignore SSL certificate validation errors.
:insecure:
If True, ignore SSL certificate validation errors.
+ :request_id:
+ Default X-Request-Id header value for outgoing requests that
+ don't already provide one. If None or omitted, generate a random
+ ID. When retrying failed requests, the same ID is used on all
+ attempts.
+
Additional keyword arguments will be passed directly to
`apiclient_discovery.build` if a new Resource object is created.
If the `discoveryServiceUrl` or `http` keyword arguments are
Additional keyword arguments will be passed directly to
`apiclient_discovery.build` if a new Resource object is created.
If the `discoveryServiceUrl` or `http` keyword arguments are
@@
-182,7
+199,8
@@
def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
elif host and token:
pass
elif not host and not token:
elif host and token:
pass
elif not host and not token:
- return api_from_config(version=version, cache=cache, **kwargs)
+ return api_from_config(
+ version=version, cache=cache, request_id=request_id, **kwargs)
else:
# Caller provided one but not the other
if not host:
else:
# Caller provided one but not the other
if not host:
@@
-208,8
+226,10
@@
def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs)
svc.api_token = token
svc.insecure = insecure
svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs)
svc.api_token = token
svc.insecure = insecure
+ svc.request_id = request_id
kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
kwargs['http'].cache = None
kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
kwargs['http'].cache = None
+ kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id()
return svc
def api_from_config(version=None, apiconfig=None, **kwargs):
return svc
def api_from_config(version=None, apiconfig=None, **kwargs):