11 from apiclient import discovery as apiclient_discovery
12 from apiclient import errors as apiclient_errors
17 _logger = logging.getLogger('arvados.api')
19 class OrderedJsonModel(apiclient.model.JsonModel):
20 """Model class for JSON that preserves the contents' order.
22 API clients that care about preserving the order of fields in API
23 server responses can use this model to do so, like this::
25 from arvados.api import OrderedJsonModel
26 client = arvados.api('v1', ..., model=OrderedJsonModel())
29 def deserialize(self, content):
30 # This is a very slightly modified version of the parent class'
31 # implementation. Copyright (c) 2010 Google.
32 content = content.decode('utf-8')
33 body = json.loads(content, object_pairs_hook=collections.OrderedDict)
34 if self._data_wrapper and isinstance(body, dict) and 'data' in body:
39 def _intercept_http_request(self, uri, **kwargs):
40 from httplib import BadStatusLine
42 if (self.max_request_size and
43 kwargs.get('body') and
44 self.max_request_size < len(kwargs['body'])):
45 raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
47 if 'headers' not in kwargs:
48 kwargs['headers'] = {}
50 if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
51 kwargs['headers']['X-External-Client'] = '1'
53 kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
55 return self.orig_http_request(uri, **kwargs)
57 # This is how httplib tells us that it tried to reuse an
58 # existing connection but it was already closed by the
59 # server. In that case, yes, we would like to retry.
60 # Unfortunately, we are not absolutely certain that the
61 # previous call did not succeed, so this is slightly
63 return self.orig_http_request(uri, **kwargs)
65 # This is the one case where httplib2 doesn't close the
66 # underlying connection first. Close all open connections,
67 # expecting this object only has the one connection to the API
68 # server. This is safe because httplib2 reopens connections when
70 _logger.debug("Retrying API request after socket error", exc_info=True)
71 for conn in self.connections.itervalues():
73 return self.orig_http_request(uri, **kwargs)
75 def _patch_http_request(http, api_token):
76 http.arvados_api_token = api_token
77 http.max_request_size = 0
78 http.orig_http_request = http.request
79 http.request = types.MethodType(_intercept_http_request, http)
82 # Monkey patch discovery._cast() so objects and arrays get serialized
83 # with json.dumps() instead of str().
84 _cast_orig = apiclient_discovery._cast
85 def _cast_objects_too(value, schema_type):
87 if (type(value) != type('') and
88 (schema_type == 'object' or schema_type == 'array')):
89 return json.dumps(value)
91 return _cast_orig(value, schema_type)
92 apiclient_discovery._cast = _cast_objects_too
94 # Convert apiclient's HttpErrors into our own API error subclass for better
96 # Reassigning apiclient_errors.HttpError is not sufficient because most of the
97 # apiclient submodules import the class into their own namespace.
98 def _new_http_error(cls, *args, **kwargs):
99 return super(apiclient_errors.HttpError, cls).__new__(
100 errors.ApiError, *args, **kwargs)
101 apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
103 def http_cache(data_type):
104 homedir = os.environ.get('HOME')
105 if not homedir or len(homedir) == 0:
107 path = homedir + '/.cache/arvados/' + data_type
109 util.mkdir_dash_p(path)
114 def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
115 """Return an apiclient Resources object for an Arvados instance.
118 A string naming the version of the Arvados API to use (for
122 Use a cache (~/.cache/arvados/discovery) for the discovery
126 The Arvados API server host (and optional :port) to connect to.
129 The authentication token to send with each API call.
132 If True, ignore SSL certificate validation errors.
134 Additional keyword arguments will be passed directly to
135 `apiclient_discovery.build` if a new Resource object is created.
136 If the `discoveryServiceUrl` or `http` keyword arguments are
137 missing, this function will set default values for them, based on
138 the current Arvados configuration settings.
144 _logger.info("Using default API version. " +
145 "Call arvados.api('%s') instead." %
147 if 'discoveryServiceUrl' in kwargs:
149 raise ValueError("both discoveryServiceUrl and host provided")
150 # Here we can't use a token from environment, config file,
151 # etc. Those probably have nothing to do with the host
152 # provided by the caller.
154 raise ValueError("discoveryServiceUrl provided, but token missing")
157 elif not host and not token:
158 return api_from_config(version=version, cache=cache, **kwargs)
160 # Caller provided one but not the other
162 raise ValueError("token argument provided, but host missing.")
164 raise ValueError("host argument provided, but token missing.")
167 # Caller wants us to build the discoveryServiceUrl
168 kwargs['discoveryServiceUrl'] = (
169 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,))
171 if 'http' not in kwargs:
172 http_kwargs = {'ca_certs': util.ca_certs_path()}
174 http_kwargs['cache'] = http_cache('discovery')
176 http_kwargs['disable_ssl_certificate_validation'] = True
177 kwargs['http'] = httplib2.Http(**http_kwargs)
179 kwargs['http'] = _patch_http_request(kwargs['http'], token)
181 svc = apiclient_discovery.build('arvados', version, **kwargs)
182 svc.api_token = token
183 kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
184 kwargs['http'].cache = None
187 def api_from_config(version=None, apiconfig=None, **kwargs):
188 """Return an apiclient Resources object enabling access to an Arvados server
192 A string naming the version of the Arvados REST API to use (for
196 If provided, this should be a dict-like object (must support the get()
197 method) with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and
198 optionally ARVADOS_API_HOST_INSECURE. If not provided, use
199 arvados.config (which gets these parameters from the environment by
202 Other keyword arguments such as `cache` will be passed along `api()`
205 # Load from user configuration or environment
206 if apiconfig is None:
207 apiconfig = config.settings()
209 for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
210 if x not in apiconfig:
211 raise ValueError("%s is not set. Aborting." % x)
212 host = apiconfig.get('ARVADOS_API_HOST')
213 token = apiconfig.get('ARVADOS_API_TOKEN')
214 insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig)
216 return api(version=version, host=host, token=token, insecure=insecure, **kwargs)