1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import absolute_import
6 from future import standard_library
7 standard_library.install_aliases()
8 from builtins import range
21 from apiclient import discovery as apiclient_discovery
22 from apiclient import errors as apiclient_errors
28 _logger = logging.getLogger('arvados.api')
30 MAX_IDLE_CONNECTION_DURATION = 30
31 RETRY_DELAY_INITIAL = 2
32 RETRY_DELAY_BACKOFF = 2
35 class OrderedJsonModel(apiclient.model.JsonModel):
36 """Model class for JSON that preserves the contents' order.
38 API clients that care about preserving the order of fields in API
39 server responses can use this model to do so, like this::
41 from arvados.api import OrderedJsonModel
42 client = arvados.api('v1', ..., model=OrderedJsonModel())
45 def deserialize(self, content):
46 # This is a very slightly modified version of the parent class'
47 # implementation. Copyright (c) 2010 Google.
48 content = content.decode('utf-8')
49 body = json.loads(content, object_pairs_hook=collections.OrderedDict)
50 if self._data_wrapper and isinstance(body, dict) and 'data' in body:
55 def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs):
56 if (self.max_request_size and
57 kwargs.get('body') and
58 self.max_request_size < len(kwargs['body'])):
59 raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
61 if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
62 headers['X-External-Client'] = '1'
64 headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
65 if not headers.get('X-Request-Id'):
66 headers['X-Request-Id'] = self._request_id()
68 retryable = method in [
69 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
70 retry_count = self._retry_count if retryable else 0
73 time.time() - self._last_request_time > self._max_keepalive_idle):
74 # High probability of failure due to connection atrophy. Make
75 # sure this request [re]opens a new connection by closing and
76 # forgetting all cached connections first.
77 for conn in self.connections.values():
79 self.connections.clear()
81 delay = self._retry_delay_initial
82 for _ in range(retry_count):
83 self._last_request_time = time.time()
85 return self.orig_http_request(uri, method, headers=headers, **kwargs)
86 except http.client.HTTPException:
87 _logger.debug("Retrying API request in %d s after HTTP error",
90 # This is the one case where httplib2 doesn't close the
91 # underlying connection first. Close all open
92 # connections, expecting this object only has the one
93 # connection to the API server. This is safe because
94 # httplib2 reopens connections when needed.
95 _logger.debug("Retrying API request in %d s after socket error",
97 for conn in self.connections.values():
99 except httplib2.SSLHandshakeError as e:
100 # Intercept and re-raise with a better error message.
101 raise httplib2.SSLHandshakeError("Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e))
104 delay = delay * self._retry_delay_backoff
106 self._last_request_time = time.time()
107 return self.orig_http_request(uri, method, headers=headers, **kwargs)
109 def _patch_http_request(http, api_token):
110 http.arvados_api_token = api_token
111 http.max_request_size = 0
112 http.orig_http_request = http.request
113 http.request = types.MethodType(_intercept_http_request, http)
114 http._last_request_time = 0
115 http._max_keepalive_idle = MAX_IDLE_CONNECTION_DURATION
116 http._retry_delay_initial = RETRY_DELAY_INITIAL
117 http._retry_delay_backoff = RETRY_DELAY_BACKOFF
118 http._retry_count = RETRY_COUNT
119 http._request_id = util.new_request_id
122 # Monkey patch discovery._cast() so objects and arrays get serialized
123 # with json.dumps() instead of str().
124 _cast_orig = apiclient_discovery._cast
125 def _cast_objects_too(value, schema_type):
127 if (type(value) != type('') and
128 type(value) != type(b'') and
129 (schema_type == 'object' or schema_type == 'array')):
130 return json.dumps(value)
132 return _cast_orig(value, schema_type)
133 apiclient_discovery._cast = _cast_objects_too
135 # Convert apiclient's HttpErrors into our own API error subclass for better
137 # Reassigning apiclient_errors.HttpError is not sufficient because most of the
138 # apiclient submodules import the class into their own namespace.
139 def _new_http_error(cls, *args, **kwargs):
140 return super(apiclient_errors.HttpError, cls).__new__(
141 errors.ApiError, *args, **kwargs)
142 apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
144 def http_cache(data_type):
145 homedir = os.environ.get('HOME')
146 if not homedir or len(homedir) == 0:
148 path = homedir + '/.cache/arvados/' + data_type
150 util.mkdir_dash_p(path)
153 return cache.SafeHTTPCache(path, max_age=60*60*24*2)
155 def api(version=None, cache=True, host=None, token=None, insecure=False,
156 request_id=None, **kwargs):
157 """Return an apiclient Resources object for an Arvados instance.
160 A string naming the version of the Arvados API to use (for
164 Use a cache (~/.cache/arvados/discovery) for the discovery
168 The Arvados API server host (and optional :port) to connect to.
171 The authentication token to send with each API call.
174 If True, ignore SSL certificate validation errors.
177 Default X-Request-Id header value for outgoing requests that
178 don't already provide one. If None or omitted, generate a random
179 ID. When retrying failed requests, the same ID is used on all
182 Additional keyword arguments will be passed directly to
183 `apiclient_discovery.build` if a new Resource object is created.
184 If the `discoveryServiceUrl` or `http` keyword arguments are
185 missing, this function will set default values for them, based on
186 the current Arvados configuration settings.
192 _logger.info("Using default API version. " +
193 "Call arvados.api('%s') instead." %
195 if 'discoveryServiceUrl' in kwargs:
197 raise ValueError("both discoveryServiceUrl and host provided")
198 # Here we can't use a token from environment, config file,
199 # etc. Those probably have nothing to do with the host
200 # provided by the caller.
202 raise ValueError("discoveryServiceUrl provided, but token missing")
205 elif not host and not token:
206 return api_from_config(
207 version=version, cache=cache, request_id=request_id, **kwargs)
209 # Caller provided one but not the other
211 raise ValueError("token argument provided, but host missing.")
213 raise ValueError("host argument provided, but token missing.")
216 # Caller wants us to build the discoveryServiceUrl
217 kwargs['discoveryServiceUrl'] = (
218 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,))
220 if 'http' not in kwargs:
221 http_kwargs = {'ca_certs': util.ca_certs_path()}
223 http_kwargs['cache'] = http_cache('discovery')
225 http_kwargs['disable_ssl_certificate_validation'] = True
226 kwargs['http'] = httplib2.Http(**http_kwargs)
228 kwargs['http'] = _patch_http_request(kwargs['http'], token)
230 svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs)
231 svc.api_token = token
232 svc.insecure = insecure
233 svc.request_id = request_id
234 kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
235 kwargs['http'].cache = None
236 kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id()
239 def api_from_config(version=None, apiconfig=None, **kwargs):
240 """Return an apiclient Resources object enabling access to an Arvados server
244 A string naming the version of the Arvados REST API to use (for
248 If provided, this should be a dict-like object (must support the get()
249 method) with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and
250 optionally ARVADOS_API_HOST_INSECURE. If not provided, use
251 arvados.config (which gets these parameters from the environment by
254 Other keyword arguments such as `cache` will be passed along `api()`
257 # Load from user configuration or environment
258 if apiconfig is None:
259 apiconfig = config.settings()
262 for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
263 if x not in apiconfig:
266 raise ValueError(" and ".join(errors)+" not set.\nPlease set in %s or export environment variable." % config.default_config_file)
267 host = apiconfig.get('ARVADOS_API_HOST')
268 token = apiconfig.get('ARVADOS_API_TOKEN')
269 insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig)
271 return api(version=version, host=host, token=token, insecure=insecure, **kwargs)