X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/70c7c9f19692ea7275541006e720c3ea56ba32e1..a4acb3ae95b2fc7f4b5f1e174c910a54cc6681da:/sdk/python/tests/run_test_server.py diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index a7b60e7d7b..f8f8b18d76 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -3,6 +3,7 @@ from __future__ import print_function import argparse import atexit +import errno import httplib2 import os import pipes @@ -43,8 +44,6 @@ if not os.path.exists(TEST_TMPDIR): my_api_host = None _cached_config = {} -keep_existing = False -enforce_permissions = False def find_server_pid(PID_PATH, wait=10): now = time.time() @@ -56,9 +55,7 @@ def find_server_pid(PID_PATH, wait=10): with open(PID_PATH, 'r') as f: server_pid = int(f.read()) good_pid = (os.kill(server_pid, 0) is None) - except IOError: - good_pid = False - except OSError: + except EnvironmentError: good_pid = False now = time.time() @@ -93,9 +90,7 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False): os.getpgid(server_pid) time.sleep(0.1) now = time.time() - except IOError: - pass - except OSError: + except EnvironmentError: pass def find_available_port(): @@ -325,9 +320,8 @@ def _start_keep(n, keep_args): return port -def run_keep(blob_signing_key=None): - if not keep_existing: - stop_keep() +def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2): + stop_keep(num_servers) keep_args = {} if not blob_signing_key: @@ -348,17 +342,12 @@ def run_keep(blob_signing_key=None): token=os.environ['ARVADOS_API_TOKEN'], insecure=True) - for d in api.keep_services().list().execute()['items']: + for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']: api.keep_services().delete(uuid=d['uuid']).execute() for d in api.keep_disks().list().execute()['items']: api.keep_disks().delete(uuid=d['uuid']).execute() - start_index = 0 - end_index = 2 - if keep_existing: - start_index = 2 - end_index = 3 - for d in range(start_index, end_index): + for d in range(0, num_servers): port = _start_keep(d, keep_args) svc = api.keep_services().create(body={'keep_service': { 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d), @@ -371,6 +360,12 @@ def run_keep(blob_signing_key=None): 'keep_disk': {'keep_service_uuid': svc['uuid'] } }).execute() + # If keepproxy is running, send SIGHUP to make it discover the new + # keepstore services. + proxypidfile = _pidfile('keepproxy') + if os.path.exists(proxypidfile): + os.kill(int(open(proxypidfile).read()), signal.SIGHUP) + def _stop_keep(n): kill_server_pid(_pidfile('keep{}'.format(n)), 0) if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)): @@ -380,11 +375,9 @@ def _stop_keep(n): if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")): os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")) -def stop_keep(): - _stop_keep(0) - _stop_keep(1) - # We may have created an additional keep servers when keep_existing is used - _stop_keep(2) +def stop_keep(num_servers=2): + for n in range(0, num_servers): + _stop_keep(n) def run_keep_proxy(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: @@ -448,16 +441,42 @@ def stop_arv_git_httpd(): return kill_server_pid(_pidfile('arv-git-httpd'), wait=0) +def run_keep_web(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + stop_keep_web() + + keepwebport = find_available_port() + env = os.environ.copy() + env.pop('ARVADOS_API_TOKEN', None) + keepweb = subprocess.Popen( + ['keep-web', + '-attachment-only-host=localhost:'+str(keepwebport), + '-address=:'+str(keepwebport)], + env=env, stdin=open('/dev/null'), stdout=sys.stderr) + with open(_pidfile('keep-web'), 'w') as f: + f.write(str(keepweb.pid)) + _setport('keep-web', keepwebport) + _wait_until_port_listens(keepwebport) + +def stop_keep_web(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + kill_server_pid(_pidfile('keep-web'), wait=0) + def run_nginx(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: return nginxconf = {} + nginxconf['KEEPWEBPORT'] = _getport('keep-web') + nginxconf['KEEPWEBSSLPORT'] = find_available_port() nginxconf['KEEPPROXYPORT'] = _getport('keepproxy') nginxconf['KEEPPROXYSSLPORT'] = find_available_port() nginxconf['GITPORT'] = _getport('arv-git-httpd') nginxconf['GITSSLPORT'] = find_available_port() nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem') nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key') + nginxconf['ACCESSLOG'] = os.path.join(TEST_TMPDIR, 'nginx_access_log.fifo') conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf') conffile = os.path.join(TEST_TMPDIR, 'nginx.conf') @@ -469,12 +488,24 @@ def run_nginx(): env = os.environ.copy() env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin' + + try: + os.remove(nginxconf['ACCESSLOG']) + except OSError as error: + if error.errno != errno.ENOENT: + raise + + os.mkfifo(nginxconf['ACCESSLOG'], 0700) nginx = subprocess.Popen( ['nginx', '-g', 'error_log stderr info;', '-g', 'pid '+_pidfile('nginx')+';', '-c', conffile], env=env, stdin=open('/dev/null'), stdout=sys.stderr) + cat_access = subprocess.Popen( + ['cat', nginxconf['ACCESSLOG']], + stdout=sys.stderr) + _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT']) _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT']) _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT']) @@ -556,6 +587,7 @@ class TestCaseWithServers(unittest.TestCase): MAIN_SERVER = None KEEP_SERVER = None KEEP_PROXY_SERVER = None + KEEP_WEB_SERVER = None @staticmethod def _restore_dict(src, dest): @@ -574,7 +606,8 @@ class TestCaseWithServers(unittest.TestCase): for server_kwargs, start_func, stop_func in ( (cls.MAIN_SERVER, run, reset), (cls.KEEP_SERVER, run_keep, stop_keep), - (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)): + (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy), + (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)): if server_kwargs is not None: start_func(**server_kwargs) cls._cleanup_funcs.append(stop_func) @@ -600,22 +633,18 @@ if __name__ == "__main__": 'start', 'stop', 'start_keep', 'stop_keep', 'start_keep_proxy', 'stop_keep_proxy', + 'start_keep-web', 'stop_keep-web', 'start_arv-git-httpd', 'stop_arv-git-httpd', 'start_nginx', 'stop_nginx', ] parser = argparse.ArgumentParser() parser.add_argument('action', type=str, help="one of {}".format(actions)) parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture') - parser.add_argument('--keep-existing', type=str, help="Used to add additional keep servers, without terminating existing servers") - parser.add_argument('--keep-enforce-permissions', type=str, help="Enforce keep permissions") + parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired") + parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions") args = parser.parse_args() - if args.keep_existing == 'true': - keep_existing = True - if args.keep_enforce_permissions == 'true': - enforce_permissions = True - if args.action not in actions: print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr) sys.exit(1) @@ -633,7 +662,7 @@ if __name__ == "__main__": elif args.action == 'stop': stop(force=('ARVADOS_TEST_API_HOST' not in os.environ)) elif args.action == 'start_keep': - run_keep() + run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers) elif args.action == 'stop_keep': stop_keep() elif args.action == 'start_keep_proxy': @@ -644,6 +673,10 @@ if __name__ == "__main__": run_arv_git_httpd() elif args.action == 'stop_arv-git-httpd': stop_arv_git_httpd() + elif args.action == 'start_keep-web': + run_keep_web() + elif args.action == 'stop_keep-web': + stop_keep_web() elif args.action == 'start_nginx': run_nginx() elif args.action == 'stop_nginx':