1 from __future__ import absolute_import
2 from future import standard_library
3 standard_library.install_aliases()
4 from builtins import range
17 from apiclient import discovery as apiclient_discovery
18 from apiclient import errors as apiclient_errors
24 _logger = logging.getLogger('arvados.api')
26 MAX_IDLE_CONNECTION_DURATION = 30
27 RETRY_DELAY_INITIAL = 2
28 RETRY_DELAY_BACKOFF = 2
31 class OrderedJsonModel(apiclient.model.JsonModel):
32 """Model class for JSON that preserves the contents' order.
34 API clients that care about preserving the order of fields in API
35 server responses can use this model to do so, like this::
37 from arvados.api import OrderedJsonModel
38 client = arvados.api('v1', ..., model=OrderedJsonModel())
41 def deserialize(self, content):
42 # This is a very slightly modified version of the parent class'
43 # implementation. Copyright (c) 2010 Google.
44 content = content.decode('utf-8')
45 body = json.loads(content, object_pairs_hook=collections.OrderedDict)
46 if self._data_wrapper and isinstance(body, dict) and 'data' in body:
51 def _intercept_http_request(self, uri, method="GET", **kwargs):
52 if (self.max_request_size and
53 kwargs.get('body') and
54 self.max_request_size < len(kwargs['body'])):
55 raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
57 if 'headers' not in kwargs:
58 kwargs['headers'] = {}
60 if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
61 kwargs['headers']['X-External-Client'] = '1'
63 kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
65 retryable = method in [
66 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
67 retry_count = self._retry_count if retryable else 0
70 time.time() - self._last_request_time > self._max_keepalive_idle):
71 # High probability of failure due to connection atrophy. Make
72 # sure this request [re]opens a new connection by closing and
73 # forgetting all cached connections first.
74 for conn in self.connections.values():
76 self.connections.clear()
78 delay = self._retry_delay_initial
79 for _ in range(retry_count):
80 self._last_request_time = time.time()
82 return self.orig_http_request(uri, method, **kwargs)
83 except http.client.HTTPException:
84 _logger.debug("Retrying API request in %d s after HTTP error",
87 # This is the one case where httplib2 doesn't close the
88 # underlying connection first. Close all open
89 # connections, expecting this object only has the one
90 # connection to the API server. This is safe because
91 # httplib2 reopens connections when needed.
92 _logger.debug("Retrying API request in %d s after socket error",
94 for conn in self.connections.values():
97 delay = delay * self._retry_delay_backoff
99 self._last_request_time = time.time()
100 return self.orig_http_request(uri, method, **kwargs)
102 def _patch_http_request(http, api_token):
103 http.arvados_api_token = api_token
104 http.max_request_size = 0
105 http.orig_http_request = http.request
106 http.request = types.MethodType(_intercept_http_request, http)
107 http._last_request_time = 0
108 http._max_keepalive_idle = MAX_IDLE_CONNECTION_DURATION
109 http._retry_delay_initial = RETRY_DELAY_INITIAL
110 http._retry_delay_backoff = RETRY_DELAY_BACKOFF
111 http._retry_count = RETRY_COUNT
114 # Monkey patch discovery._cast() so objects and arrays get serialized
115 # with json.dumps() instead of str().
116 _cast_orig = apiclient_discovery._cast
117 def _cast_objects_too(value, schema_type):
119 if (type(value) != type('') and
120 type(value) != type(b'') and
121 (schema_type == 'object' or schema_type == 'array')):
122 return json.dumps(value)
124 return _cast_orig(value, schema_type)
125 apiclient_discovery._cast = _cast_objects_too
127 # Convert apiclient's HttpErrors into our own API error subclass for better
129 # Reassigning apiclient_errors.HttpError is not sufficient because most of the
130 # apiclient submodules import the class into their own namespace.
131 def _new_http_error(cls, *args, **kwargs):
132 return super(apiclient_errors.HttpError, cls).__new__(
133 errors.ApiError, *args, **kwargs)
134 apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
136 def http_cache(data_type):
137 homedir = os.environ.get('HOME')
138 if not homedir or len(homedir) == 0:
140 path = homedir + '/.cache/arvados/' + data_type
142 util.mkdir_dash_p(path)
145 return cache.SafeHTTPCache(path, max_age=60*60*24*2)
147 def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
148 """Return an apiclient Resources object for an Arvados instance.
151 A string naming the version of the Arvados API to use (for
155 Use a cache (~/.cache/arvados/discovery) for the discovery
159 The Arvados API server host (and optional :port) to connect to.
162 The authentication token to send with each API call.
165 If True, ignore SSL certificate validation errors.
167 Additional keyword arguments will be passed directly to
168 `apiclient_discovery.build` if a new Resource object is created.
169 If the `discoveryServiceUrl` or `http` keyword arguments are
170 missing, this function will set default values for them, based on
171 the current Arvados configuration settings.
177 _logger.info("Using default API version. " +
178 "Call arvados.api('%s') instead." %
180 if 'discoveryServiceUrl' in kwargs:
182 raise ValueError("both discoveryServiceUrl and host provided")
183 # Here we can't use a token from environment, config file,
184 # etc. Those probably have nothing to do with the host
185 # provided by the caller.
187 raise ValueError("discoveryServiceUrl provided, but token missing")
190 elif not host and not token:
191 return api_from_config(version=version, cache=cache, **kwargs)
193 # Caller provided one but not the other
195 raise ValueError("token argument provided, but host missing.")
197 raise ValueError("host argument provided, but token missing.")
200 # Caller wants us to build the discoveryServiceUrl
201 kwargs['discoveryServiceUrl'] = (
202 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,))
204 if 'http' not in kwargs:
205 http_kwargs = {'ca_certs': util.ca_certs_path()}
207 http_kwargs['cache'] = http_cache('discovery')
209 http_kwargs['disable_ssl_certificate_validation'] = True
210 kwargs['http'] = httplib2.Http(**http_kwargs)
212 kwargs['http'] = _patch_http_request(kwargs['http'], token)
214 svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs)
215 svc.api_token = token
216 svc.insecure = insecure
217 kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
218 kwargs['http'].cache = None
221 def api_from_config(version=None, apiconfig=None, **kwargs):
222 """Return an apiclient Resources object enabling access to an Arvados server
226 A string naming the version of the Arvados REST API to use (for
230 If provided, this should be a dict-like object (must support the get()
231 method) with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and
232 optionally ARVADOS_API_HOST_INSECURE. If not provided, use
233 arvados.config (which gets these parameters from the environment by
236 Other keyword arguments such as `cache` will be passed along `api()`
239 # Load from user configuration or environment
240 if apiconfig is None:
241 apiconfig = config.settings()
243 for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
244 if x not in apiconfig:
245 raise ValueError("%s is not set. Aborting." % x)
246 host = apiconfig.get('ARVADOS_API_HOST')
247 token = apiconfig.get('ARVADOS_API_TOKEN')
248 insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig)
250 return api(version=version, host=host, token=token, insecure=insecure, **kwargs)