X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/7b4ec6d3c3ed209a42f542e1b646b8e672847fea..de3276d1b2067be4aa40b51605c2ce509ea80915:/sdk/python/tests/run_test_server.py?ds=sidebyside diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index 987c67303f..34342059f3 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -26,6 +26,11 @@ import time import unittest import yaml +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + MY_DIRNAME = os.path.dirname(os.path.realpath(__file__)) if __name__ == '__main__' and os.path.exists( os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')): @@ -395,10 +400,12 @@ def get_config(): return yaml.safe_load(f) def internal_port_from_config(service): - return int(list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys())[0].split(":")[2]) + return int(urlparse( + list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys())[0]). + netloc.split(":")[1]) def external_port_from_config(service): - return int(get_config()["Clusters"]["zzzzz"]["Services"][service]["ExternalURL"].split(":")[2]) + return int(urlparse(get_config()["Clusters"]["zzzzz"]["Services"][service]["ExternalURL"]).netloc.split(":")[1]) def run_controller(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: @@ -537,10 +544,11 @@ def run_keep_proxy(): env['ARVADOS_API_TOKEN'] = auth_token('anonymous') logf = open(_logfilename('keepproxy'), 'a') kp = subprocess.Popen( - ['keepproxy', - '-pid='+_pidfile('keepproxy'), - '-listen=:{}'.format(port)], - env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) + ['keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) + + with open(_pidfile('keepproxy'), 'w') as f: + f.write(str(kp.pid)) + _wait_until_port_listens(port) print("Using API %s token %s" % (os.environ['ARVADOS_API_HOST'], auth_token('admin')), file=sys.stdout) api = arvados.api( @@ -570,16 +578,11 @@ def run_arv_git_httpd(): return stop_arv_git_httpd() - gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git') gitport = internal_port_from_config("GitHTTP") env = os.environ.copy() env.pop('ARVADOS_API_TOKEN', None) logf = open(_logfilename('arv-git-httpd'), 'a') - agh = subprocess.Popen( - ['arv-git-httpd', - '-repo-root='+gitdir+'/test', - '-management-token=e687950a23c3a9bceec28c6223a06c79', - '-address=:'+str(gitport)], + agh = subprocess.Popen(['arv-git-httpd'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) with open(_pidfile('arv-git-httpd'), 'w') as f: f.write(str(agh.pid)) @@ -597,14 +600,9 @@ def run_keep_web(): keepwebport = internal_port_from_config("WebDAV") env = os.environ.copy() - env['ARVADOS_API_TOKEN'] = auth_token('anonymous') logf = open(_logfilename('keep-web'), 'a') keepweb = subprocess.Popen( - ['keep-web', - '-allow-anonymous', - '-attachment-only-host=download', - '-management-token=e687950a23c3a9bceec28c6223a06c79', - '-listen=:'+str(keepwebport)], + ['keep-web'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) with open(_pidfile('keep-web'), 'w') as f: f.write(str(keepweb.pid)) @@ -670,54 +668,56 @@ def setup_config(): keep_web_dl_port = find_available_port() keep_web_dl_external_port = find_available_port() - if "CONFIGSRC" in os.environ: - dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml") - else: - dbconf = "/etc/arvados/config.yml" + dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml") print("Getting config from %s" % dbconf, file=sys.stderr) - pgconnection = list(yaml.safe_load(open(dbconf))["Clusters"].values())[0]["PostgreSQL"]["Connection"] - - if "test" not in pgconnection["dbname"]: - pgconnection["dbname"] = "arvados_test" + pgconnection = yaml.safe_load(open(dbconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"] + localhost = "127.0.0.1" services = { "RailsAPI": { - "InternalURLs": { } + "InternalURLs": { + "https://%s:%s"%(localhost, rails_api_port): {} + } }, "Controller": { - "ExternalURL": "https://localhost:%s" % controller_external_port, - "InternalURLs": { } + "ExternalURL": "https://%s:%s" % (localhost, controller_external_port), + "InternalURLs": { + "http://%s:%s"%(localhost, controller_port): {} + } }, "Websocket": { - "ExternalURL": "https://localhost:%s" % websocket_external_port, - "InternalURLs": { } + "ExternalURL": "wss://%s:%s/websocket" % (localhost, websocket_external_port), + "InternalURLs": { + "http://%s:%s"%(localhost, websocket_port): {} + } }, "GitHTTP": { - "ExternalURL": "https://localhost:%s" % git_httpd_external_port, - "InternalURLs": { } + "ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port), + "InternalURLs": { + "http://%s:%s"%(localhost, git_httpd_port): {} + } }, "Keepproxy": { - "ExternalURL": "https://localhost:%s" % keepproxy_external_port, - "InternalURLs": { } + "ExternalURL": "https://%s:%s" % (localhost, keepproxy_external_port), + "InternalURLs": { + "http://%s:%s"%(localhost, keepproxy_port): {} + } }, "WebDAV": { - "ExternalURL": "https://localhost:%s" % keep_web_external_port, - "InternalURLs": { } + "ExternalURL": "https://%s:%s" % (localhost, keep_web_external_port), + "InternalURLs": { + "http://%s:%s"%(localhost, keep_web_port): {} + } }, "WebDAVDownload": { - "ExternalURL": "https://localhost:%s" % keep_web_dl_external_port, - "InternalURLs": { } + "ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port), + "InternalURLs": { + "http://%s:%s"%(localhost, keep_web_dl_port): {} + } } } - services["RailsAPI"]["InternalURLs"]["https://localhost:%s"%rails_api_port] = {} - services["Controller"]["InternalURLs"]["http://localhost:%s"%controller_port] = {} - services["Websocket"]["InternalURLs"]["http://localhost:%s"%websocket_port] = {} - services["GitHTTP"]["InternalURLs"]["http://localhost:%s"%git_httpd_port] = {} - services["Keepproxy"]["InternalURLs"]["http://localhost:%s"%keepproxy_port] = {} - services["WebDAV"]["InternalURLs"]["http://localhost:%s"%keep_web_port] = {} - services["WebDAVDownload"]["InternalURLs"]["http://localhost:%s"%keep_web_dl_port] = {} config = { "Clusters": { @@ -736,7 +736,16 @@ def setup_config(): "TLS": { "Insecure": True }, - "Services": services + "Services": services, + "Users": { + "AnonymousUserToken": auth_token('anonymous') + }, + "Collections": { + "TrustAllContent": True + }, + "Git": { + "Repositories": "%s/test" % os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git') + } } } } @@ -746,7 +755,6 @@ def setup_config(): yaml.safe_dump(config, f) ex = "export ARVADOS_CONFIG="+conf - print(ex, file=sys.stderr) print(ex) @@ -758,35 +766,6 @@ def stop_nginx(): def _pidfile(program): return os.path.join(TEST_TMPDIR, program + '.pid') -def _dbconfig(key): - global _cached_db_config - if not _cached_db_config: - if "ARVADOS_CONFIG" in os.environ: - _cached_db_config = list(yaml.safe_load(open(os.environ["ARVADOS_CONFIG"]))["Clusters"].values())[0]["PostgreSQL"]["Connection"] - else: - _cached_db_config = yaml.safe_load(open(os.path.join( - SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))["test"] - _cached_db_config["dbname"] = _cached_db_config["database"] - _cached_db_config["user"] = _cached_db_config["username"] - return _cached_db_config[key] - -def _apiconfig(key): - global _cached_config - if _cached_config: - return _cached_config[key] - def _load(f, required=True): - fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f) - if not required and not os.path.exists(fullpath): - return {} - return yaml.safe_load(fullpath) - cdefault = _load('application.default.yml') - csite = _load('application.yml', required=False) - _cached_config = {} - for section in [cdefault.get('common',{}), cdefault.get('test',{}), - csite.get('common',{}), csite.get('test',{})]: - _cached_config.update(section) - return _cached_config[key] - def fixture(fix): '''load a fixture yaml file''' with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",