X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ad69b48e324c3ce29a4d2c84732dfd3d0288ebb3..22928daa55bc0de7d539793ff27503affac8a39a:/sdk/python/tests/run_test_server.py diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index b9502f0f8e..d90d2ad1a7 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function import argparse import atexit import httplib2 @@ -41,6 +42,7 @@ if not os.path.exists(TEST_TMPDIR): os.mkdir(TEST_TMPDIR) my_api_host = None +_cached_config = {} def find_server_pid(PID_PATH, wait=10): now = time.time() @@ -113,6 +115,33 @@ def find_available_port(): sock.close() return port +def _wait_until_port_listens(port, timeout=10): + """Wait for a process to start listening on the given port. + + If nothing listens on the port within the specified timeout (given + in seconds), print a warning on stderr before returning. + """ + try: + subprocess.check_output(['which', 'lsof']) + except subprocess.CalledProcessError: + print("WARNING: No `lsof` -- cannot wait for port to listen. "+ + "Sleeping 0.5 and hoping for the best.") + time.sleep(0.5) + return + deadline = time.time() + timeout + while time.time() < deadline: + try: + subprocess.check_output( + ['lsof', '-t', '-i', 'tcp:'+str(port)]) + except subprocess.CalledProcessError: + time.sleep(0.1) + continue + return + print( + "WARNING: Nothing is listening on port {} (waited {} seconds).". + format(port, timeout), + file=sys.stderr) + def run(leave_running_atexit=False): """Ensure an API server is running, and ARVADOS_API_* env vars have admin credentials for it. @@ -163,6 +192,15 @@ def run(leave_running_atexit=False): # died, or we have lost our credentials, or something else is # preventing us from calling reset(). Start a new one. + if not os.path.exists('tmp'): + os.makedirs('tmp') + + if not os.path.exists('tmp/api'): + os.makedirs('tmp/api') + + if not os.path.exists('tmp/logs'): + os.makedirs('tmp/logs') + if not os.path.exists('tmp/self-signed.pem'): # We assume here that either passenger reports its listening # address as https:/0.0.0.0:port/. If it reports "127.0.0.1" @@ -178,6 +216,13 @@ def run(leave_running_atexit=False): '-subj', '/CN=0.0.0.0'], stdout=sys.stderr) + # Install the git repository fixtures. + gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git') + gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar') + if not os.path.isdir(gitdir): + os.makedirs(gitdir) + subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball]) + port = find_available_port() env = os.environ.copy() env['RAILS_ENV'] = 'test' @@ -206,8 +251,10 @@ def run(leave_running_atexit=False): my_api_host = match.group(1) os.environ['ARVADOS_API_HOST'] = my_api_host - # Make sure the server has written its pid file before continuing + # Make sure the server has written its pid file and started + # listening on its TCP port find_server_pid(pid_file) + _wait_until_port_listens(port) reset() os.chdir(restore_cwd) @@ -256,44 +303,54 @@ def _start_keep(n, keep_args): keep0 = tempfile.mkdtemp() port = find_available_port() keep_cmd = ["keepstore", - "-volumes={}".format(keep0), + "-volume={}".format(keep0), "-listen=:{}".format(port), - "-pid={}".format("{}/keep{}.pid".format(TEST_TMPDIR, n))] + "-pid="+_pidfile('keep{}'.format(n))] for arg, val in keep_args.iteritems(): keep_cmd.append("{}={}".format(arg, val)) - kp0 = subprocess.Popen(keep_cmd) - with open("{}/keep{}.pid".format(TEST_TMPDIR, n), 'w') as f: + logf = open(os.path.join(TEST_TMPDIR, 'keep{}.log'.format(n)), 'a+') + kp0 = subprocess.Popen( + keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) + with open(_pidfile('keep{}'.format(n)), 'w') as f: f.write(str(kp0.pid)) with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f: f.write(keep0) + _wait_until_port_listens(port) + return port -def run_keep(blob_signing_key=None, enforce_permissions=False): - stop_keep() +def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2): + stop_keep(num_servers) keep_args = {} - if blob_signing_key: - with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f: - keep_args['--permission-key-file'] = f.name - f.write(blob_signing_key) + if not blob_signing_key: + blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc' + with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f: + keep_args['-blob-signing-key-file'] = f.name + f.write(blob_signing_key) if enforce_permissions: - keep_args['--enforce-permissions'] = 'true' + keep_args['-enforce-permissions'] = 'true' + with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f: + keep_args['-data-manager-token-file'] = f.name + f.write(os.environ['ARVADOS_API_TOKEN']) + keep_args['-never-delete'] = 'false' api = arvados.api( version='v1', host=os.environ['ARVADOS_API_HOST'], token=os.environ['ARVADOS_API_TOKEN'], insecure=True) + for d in api.keep_services().list().execute()['items']: api.keep_services().delete(uuid=d['uuid']).execute() for d in api.keep_disks().list().execute()['items']: api.keep_disks().delete(uuid=d['uuid']).execute() - for d in range(0, 2): + for d in range(0, num_servers): port = _start_keep(d, keep_args) svc = api.keep_services().create(body={'keep_service': { 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d), @@ -307,7 +364,7 @@ def run_keep(blob_signing_key=None, enforce_permissions=False): }).execute() def _stop_keep(n): - kill_server_pid("{}/keep{}.pid".format(TEST_TMPDIR, n), 0) + kill_server_pid(_pidfile('keep{}'.format(n)), 0) if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)): with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r: shutil.rmtree(r.read(), True) @@ -315,11 +372,13 @@ def _stop_keep(n): if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")): os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")) -def stop_keep(): - _stop_keep(0) - _stop_keep(1) +def stop_keep(num_servers=2): + for n in range(0, num_servers): + _stop_keep(n) def run_keep_proxy(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return stop_keep_proxy() admin_token = auth_token('admin') @@ -328,9 +387,9 @@ def run_keep_proxy(): env['ARVADOS_API_TOKEN'] = admin_token kp = subprocess.Popen( ['keepproxy', - '-pid={}/keepproxy.pid'.format(TEST_TMPDIR), + '-pid='+_pidfile('keepproxy'), '-listen=:{}'.format(port)], - env=env) + env=env, stdin=open('/dev/null'), stdout=sys.stderr) api = arvados.api( version='v1', @@ -347,9 +406,105 @@ def run_keep_proxy(): 'service_ssl_flag': False, }}).execute() os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port) + _setport('keepproxy', port) + _wait_until_port_listens(port) def stop_keep_proxy(): - kill_server_pid(os.path.join(TEST_TMPDIR, "keepproxy.pid"), 0) + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + kill_server_pid(_pidfile('keepproxy'), wait=0) + +def run_arv_git_httpd(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + stop_arv_git_httpd() + + gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git') + gitport = find_available_port() + env = os.environ.copy() + env.pop('ARVADOS_API_TOKEN', None) + agh = subprocess.Popen( + ['arv-git-httpd', + '-repo-root='+gitdir+'/test', + '-address=:'+str(gitport)], + env=env, stdin=open('/dev/null'), stdout=sys.stderr) + with open(_pidfile('arv-git-httpd'), 'w') as f: + f.write(str(agh.pid)) + _setport('arv-git-httpd', gitport) + _wait_until_port_listens(gitport) + +def stop_arv_git_httpd(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + kill_server_pid(_pidfile('arv-git-httpd'), wait=0) + +def run_nginx(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + nginxconf = {} + nginxconf['KEEPPROXYPORT'] = _getport('keepproxy') + nginxconf['KEEPPROXYSSLPORT'] = find_available_port() + nginxconf['GITPORT'] = _getport('arv-git-httpd') + nginxconf['GITSSLPORT'] = find_available_port() + nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem') + nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key') + + conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf') + conffile = os.path.join(TEST_TMPDIR, 'nginx.conf') + with open(conffile, 'w') as f: + f.write(re.sub( + r'{{([A-Z]+)}}', + lambda match: str(nginxconf.get(match.group(1))), + open(conftemplatefile).read())) + + env = os.environ.copy() + env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin' + nginx = subprocess.Popen( + ['nginx', + '-g', 'error_log stderr info;', + '-g', 'pid '+_pidfile('nginx')+';', + '-c', conffile], + env=env, stdin=open('/dev/null'), stdout=sys.stderr) + _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT']) + _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT']) + +def stop_nginx(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + kill_server_pid(_pidfile('nginx'), wait=0) + +def _pidfile(program): + return os.path.join(TEST_TMPDIR, program + '.pid') + +def _portfile(program): + return os.path.join(TEST_TMPDIR, program + '.port') + +def _setport(program, port): + with open(_portfile(program), 'w') as f: + f.write(str(port)) + +# Returns 9 if program is not up. +def _getport(program): + try: + return int(open(_portfile(program)).read()) + except IOError: + return 9 + +def _apiconfig(key): + if _cached_config: + return _cached_config[key] + def _load(f, required=True): + fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f) + if not required and not os.path.exists(fullpath): + return {} + return yaml.load(fullpath) + cdefault = _load('application.default.yml') + csite = _load('application.yml', required=False) + _cached_config = {} + for section in [cdefault.get('common',{}), cdefault.get('test',{}), + csite.get('common',{}), csite.get('test',{})]: + _cached_config.update(section) + return _cached_config[key] def fixture(fix): '''load a fixture yaml file''' @@ -431,14 +586,24 @@ class TestCaseWithServers(unittest.TestCase): if __name__ == "__main__": - actions = ['start', 'stop', - 'start_keep', 'stop_keep', - 'start_keep_proxy', 'stop_keep_proxy'] + actions = [ + 'start', 'stop', + 'start_keep', 'stop_keep', + 'start_keep_proxy', 'stop_keep_proxy', + 'start_arv-git-httpd', 'stop_arv-git-httpd', + 'start_nginx', 'stop_nginx', + ] parser = argparse.ArgumentParser() parser.add_argument('action', type=str, help="one of {}".format(actions)) parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture') + parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired") + parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions") + args = parser.parse_args() + if args.action not in actions: + print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr) + sys.exit(1) if args.action == 'start': stop(force=('ARVADOS_TEST_API_HOST' not in os.environ)) run(leave_running_atexit=True) @@ -453,12 +618,20 @@ if __name__ == "__main__": elif args.action == 'stop': stop(force=('ARVADOS_TEST_API_HOST' not in os.environ)) elif args.action == 'start_keep': - run_keep() + run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers) elif args.action == 'stop_keep': stop_keep() elif args.action == 'start_keep_proxy': run_keep_proxy() elif args.action == 'stop_keep_proxy': stop_keep_proxy() + elif args.action == 'start_arv-git-httpd': + run_arv_git_httpd() + elif args.action == 'stop_arv-git-httpd': + stop_arv_git_httpd() + elif args.action == 'start_nginx': + run_nginx() + elif args.action == 'stop_nginx': + stop_nginx() else: - print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions)) + raise Exception("action recognized but not implemented!?")