From: Tom Clegg Date: Sat, 16 Aug 2014 06:06:20 +0000 (-0400) Subject: Merge branch '2800-python-global-state' into 2800-pgs X-Git-Tag: 1.1.0~2316^2~5 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/0bd1c28bed9a0756c61037947d5a9dccd5066f00 Merge branch '2800-python-global-state' into 2800-pgs Conflicts: sdk/python/arvados/api.py --- 0bd1c28bed9a0756c61037947d5a9dccd5066f00 diff --cc sdk/python/arvados/api.py index 7c60f51bd9,64c2125ed8..1db694f762 --- a/sdk/python/arvados/api.py +++ b/sdk/python/arvados/api.py @@@ -12,10 -12,12 +13,13 @@@ import confi import errors import util -services = {} +_logger = logging.getLogger('arvados.api') - services = {} ++conncache = {} + + class CredentialsFromToken(object): + def __init__(self, api_token): + self.api_token = api_token - class CredentialsFromEnv(object): @staticmethod def http_request(self, uri, **kwargs): from httplib import BadStatusLine @@@ -70,54 -64,56 +75,85 @@@ def http_cache(data_type) path = None return path - def api(version=None, cache=True, **kwargs): -def api(version=None, cache=True, host=None, token=None, insecure=False): - global services - - if 'ARVADOS_DEBUG' in config.settings(): - logging.basicConfig(level=logging.DEBUG) ++def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs): + """Return an apiclient Resources object for an Arvados instance. + + Arguments: + * version: A string naming the version of the Arvados API to use (for + example, 'v1'). - * cache: If True (default), return an existing resources object, or use - a cached discovery document to build one. ++ * cache: If True (default), return an existing Resources object if ++ one already exists with the same endpoint and credentials. If ++ False, create a new one, and do not keep it in the cache (i.e., ++ do not return it from subsequent api(cache=True) calls with ++ matching endpoint and credentials). ++ * host: The Arvados API server host (and optional :port) to connect to. ++ * token: The authentication token to send with each API call. ++ * insecure: If True, ignore SSL certificate validation errors. + + Additional keyword arguments will be passed directly to - `apiclient.discovery.build`. If the `discoveryServiceUrl` or `http` - keyword arguments are missing, this function will set default values for - them, based on the current Arvados configuration settings.""" - if not cache or not services.get(version): - if not version: - version = 'v1' - _logger.info("Using default API version. " + - "Call arvados.api('%s') instead.", - version) - - if 'discoveryServiceUrl' not in kwargs: - api_host = config.get('ARVADOS_API_HOST') - if not api_host: - raise ValueError( - "No discoveryServiceUrl or ARVADOS_API_HOST set.") - kwargs['discoveryServiceUrl'] = ( - 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % - (api_host,)) - - if 'http' not in kwargs: - http_kwargs = {} - # Prefer system's CA certificates (if available) over httplib2's. - certs_path = '/etc/ssl/certs/ca-certificates.crt' - if os.path.exists(certs_path): - http_kwargs['ca_certs'] = certs_path - if cache: - http_kwargs['cache'] = http_cache('discovery') - if (config.get('ARVADOS_API_HOST_INSECURE', '').lower() in - ('yes', 'true', '1')): - http_kwargs['disable_ssl_certificate_validation'] = True - kwargs['http'] = httplib2.Http(**http_kwargs) - - kwargs['http'] = CredentialsFromEnv().authorize(kwargs['http']) - services[version] = apiclient.discovery.build('arvados', version, - **kwargs) - kwargs['http'].cache = None - return services[version] - - def uncache_api(version): - if version in services: - del services[version] ++ `apiclient.discovery.build` if a new Resource object is created. ++ If the `discoveryServiceUrl` or `http` keyword arguments are ++ missing, this function will set default values for them, based on ++ the current Arvados configuration settings. ++ ++ """ + + if not version: + version = 'v1' + logging.info("Using default API version. " + + "Call arvados.api('%s') instead." % + version) + if host and token: - # Provided by caller - pass ++ apiinsecure = insecure + elif not host and not token: + # Load from user configuration or environment + for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']: + if x not in config.settings(): + raise Exception("%s is not set. Aborting." % x) + host = config.get('ARVADOS_API_HOST') + token = config.get('ARVADOS_API_TOKEN') - apiinsecure = re.match(r'(?i)^(true|1|yes)$', - config.get('ARVADOS_API_HOST_INSECURE', 'no')) ++ apiinsecure = (config.get('ARVADOS_API_HOST_INSECURE', '').lower() in ++ ('yes', 'true', '1')) + else: + # Caller provided one but not the other + if not host: + raise Exception("token argument provided, but host missing.") + else: + raise Exception("host argument provided, but token missing.") + - connprofile = hashlib.sha1(' '.join([ - version, host, token, ('y' if apiinsecure else 'n') - ])).hexdigest() ++ if cache: ++ connprofile = hashlib.sha1(' '.join([ ++ version, host, token, ('y' if apiinsecure else 'n') ++ ])).hexdigest() ++ svc = conncache.get(connprofile) ++ if svc: ++ return svc ++ ++ if 'http' not in kwargs: ++ http_kwargs = {} ++ # Prefer system's CA certificates (if available) over httplib2's. ++ certs_path = '/etc/ssl/certs/ca-certificates.crt' ++ if os.path.exists(certs_path): ++ http_kwargs['ca_certs'] = certs_path ++ if cache: ++ http_kwargs['cache'] = http_cache('discovery') ++ if apiinsecure: ++ http_kwargs['disable_ssl_certificate_validation'] = True ++ kwargs['http'] = httplib2.Http(**http_kwargs) + - if not cache or not services.get(connprofile): - url = 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % host - credentials = CredentialsFromToken(api_token=token) ++ credentials = CredentialsFromToken(api_token=token) ++ kwargs['http'] = credentials.authorize(kwargs['http']) + - # Use system's CA certificates (if we find them) instead of httplib2's - ca_certs = '/etc/ssl/certs/ca-certificates.crt' - if not os.path.exists(ca_certs): - ca_certs = None # use httplib2 default ++ if 'discoveryServiceUrl' not in kwargs: ++ kwargs['discoveryServiceUrl'] = ( ++ 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,)) + - http = httplib2.Http(ca_certs=ca_certs, - cache=(http_cache('discovery') if cache else None)) - http = credentials.authorize(http) - if apiinsecure: - http.disable_ssl_certificate_validation = True - services[connprofile] = apiclient.discovery.build( - 'arvados', version, http=http, discoveryServiceUrl=url) - http.cache = None ++ svc = apiclient.discovery.build('arvados', version, **kwargs) ++ kwargs['http'].cache = None ++ if cache: ++ conncache[connprofile] = svc ++ return svc + - return services[connprofile] ++def unload_connection_cache(): ++ for connprofile in conncache: ++ del conncache[connprofile] diff --cc sdk/python/tests/test_api.py index 4c485712ec,0000000000..4917d004ea mode 100644,000000..100644 --- a/sdk/python/tests/test_api.py +++ b/sdk/python/tests/test_api.py @@@ -1,76 -1,0 +1,72 @@@ +#!/usr/bin/env python + +import apiclient.errors +import arvados +import httplib2 +import json +import mimetypes +import unittest ++import os ++import run_test_server + +from apiclient.http import RequestMockBuilder +from httplib import responses as HTTP_RESPONSES + +if not mimetypes.inited: + mimetypes.init() + +class ArvadosApiClientTest(unittest.TestCase): + @classmethod + def response_from_code(cls, code): + return httplib2.Response( + {'status': code, + 'reason': HTTP_RESPONSES.get(code, "Unknown Response"), + 'Content-Type': mimetypes.types_map['.json']}) + + @classmethod + def api_error_response(cls, code, *errors): + return (cls.response_from_code(code), + json.dumps({'errors': errors, + 'error_token': '1234567890+12345678'})) + + @classmethod + def setUpClass(cls): - # The apiclient library has support for mocking requests for - # testing, but it doesn't extend to the discovery document - # itself. Point it at a known stable discovery document for now. - # FIXME: Figure out a better way to stub this out. - cls.orig_api_host = arvados.config.get('ARVADOS_API_HOST') - arvados.config.settings()['ARVADOS_API_HOST'] = 'qr1hi.arvadosapi.com' ++ run_test_server.run() + mock_responses = { + 'arvados.humans.delete': (cls.response_from_code(500), ""), + 'arvados.humans.get': cls.api_error_response( + 422, "Bad UUID format", "Bad output format"), + 'arvados.humans.list': (None, json.dumps( + {'items_available': 0, 'items': []})), + } + req_builder = RequestMockBuilder(mock_responses) - cls.api = arvados.api('v1', False, requestBuilder=req_builder) ++ cls.api = arvados.api('v1', cache=False, ++ host=os.environ['ARVADOS_API_HOST'], ++ token='discovery-doc-only-no-token-needed', ++ insecure=True, ++ requestBuilder=req_builder) + + @classmethod + def tearDownClass(cls): - if cls.orig_api_host is None: - del arvados.config.settings()['ARVADOS_API_HOST'] - else: - arvados.config.settings()['ARVADOS_API_HOST'] = cls.orig_api_host - # Prevent other tests from using our mocked API client. - arvados.uncache_api('v1') ++ run_test_server.stop() + + def test_basic_list(self): + answer = self.api.humans().list( + filters=[['uuid', 'is', None]]).execute() + self.assertEqual(answer['items_available'], len(answer['items'])) + + def test_exceptions_include_errors(self): + with self.assertRaises(apiclient.errors.HttpError) as err_ctx: + self.api.humans().get(uuid='xyz-xyz-abcdef').execute() + err_s = str(err_ctx.exception) + for msg in ["Bad UUID format", "Bad output format"]: + self.assertIn(msg, err_s) + + def test_exceptions_without_errors_have_basic_info(self): + with self.assertRaises(apiclient.errors.HttpError) as err_ctx: + self.api.humans().delete(uuid='xyz-xyz-abcdef').execute() + self.assertIn("500", str(err_ctx.exception)) + + +if __name__ == '__main__': + unittest.main()