X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2cf42c27a7e8b37e29462d0b695e24cb6f3ad5ce..8e1e7e6bf355eb0f1defc7278f1434c393a75a75:/sdk/python/tests/run_test_server.py diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index 2998f64408..972b7f9d51 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -1,7 +1,9 @@ #!/usr/bin/env python +from __future__ import print_function import argparse import atexit +import errno import httplib2 import os import pipes @@ -9,7 +11,9 @@ import random import re import shutil import signal +import socket import subprocess +import string import sys import tempfile import time @@ -23,7 +27,7 @@ if __name__ == '__main__' and os.path.exists( # Add the Python SDK source to the library path. sys.path.insert(1, os.path.dirname(MY_DIRNAME)) -import arvados.api +import arvados import arvados.config ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..')) @@ -39,6 +43,7 @@ if not os.path.exists(TEST_TMPDIR): os.mkdir(TEST_TMPDIR) my_api_host = None +_cached_config = {} def find_server_pid(PID_PATH, wait=10): now = time.time() @@ -50,9 +55,7 @@ def find_server_pid(PID_PATH, wait=10): with open(PID_PATH, 'r') as f: server_pid = int(f.read()) good_pid = (os.kill(server_pid, 0) is None) - except IOError: - good_pid = False - except OSError: + except EnvironmentError: good_pid = False now = time.time() @@ -61,17 +64,17 @@ def find_server_pid(PID_PATH, wait=10): return server_pid -def kill_server_pid(pidfile, wait=10, passenger=False): +def kill_server_pid(pidfile, wait=10, passenger_root=False): # Must re-import modules in order to work during atexit import os import signal import subprocess import time try: - if passenger: + if passenger_root: # First try to shut down nicely restore_cwd = os.getcwd() - os.chdir(os.path.join(SERVICES_SRC_DIR, 'api')) + os.chdir(passenger_root) subprocess.call([ 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile]) os.chdir(restore_cwd) @@ -80,18 +83,62 @@ def kill_server_pid(pidfile, wait=10, passenger=False): with open(pidfile, 'r') as f: server_pid = int(f.read()) while now <= timeout: - if not passenger or timeout - now < wait / 2: + if not passenger_root or timeout - now < wait / 2: # Half timeout has elapsed. Start sending SIGTERM os.kill(server_pid, signal.SIGTERM) # Raise OSError if process has disappeared os.getpgid(server_pid) time.sleep(0.1) now = time.time() - except IOError: - pass - except OSError: + except EnvironmentError: pass +def find_available_port(): + """Return an IPv4 port number that is not in use right now. + + We assume whoever needs to use the returned port is able to reuse + a recently used port without waiting for TIME_WAIT (see + SO_REUSEADDR / SO_REUSEPORT). + + Some opportunity for races here, but it's better than choosing + something at random and not checking at all. If all of our servers + (hey Passenger) knew that listening on port 0 was a thing, the OS + would take care of the races, and this wouldn't be needed at all. + """ + + sock = socket.socket() + sock.bind(('0.0.0.0', 0)) + port = sock.getsockname()[1] + sock.close() + return port + +def _wait_until_port_listens(port, timeout=10): + """Wait for a process to start listening on the given port. + + If nothing listens on the port within the specified timeout (given + in seconds), print a warning on stderr before returning. + """ + try: + subprocess.check_output(['which', 'lsof']) + except subprocess.CalledProcessError: + print("WARNING: No `lsof` -- cannot wait for port to listen. "+ + "Sleeping 0.5 and hoping for the best.") + time.sleep(0.5) + return + deadline = time.time() + timeout + while time.time() < deadline: + try: + subprocess.check_output( + ['lsof', '-t', '-i', 'tcp:'+str(port)]) + except subprocess.CalledProcessError: + time.sleep(0.1) + continue + return + print( + "WARNING: Nothing is listening on port {} (waited {} seconds).". + format(port, timeout), + file=sys.stderr) + def run(leave_running_atexit=False): """Ensure an API server is running, and ARVADOS_API_* env vars have admin credentials for it. @@ -112,28 +159,45 @@ def run(leave_running_atexit=False): # Delete cached discovery document. shutil.rmtree(arvados.http_cache('discovery')) - os.environ['ARVADOS_API_TOKEN'] = auth_token('admin') - os.environ['ARVADOS_API_HOST_INSECURE'] = 'true' - pid_file = os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH) pid_file_ok = find_server_pid(pid_file, 0) existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host) if existing_api_host and pid_file_ok: - try: - os.environ['ARVADOS_API_HOST'] = existing_api_host - reset() - return - except: - pass + if existing_api_host == my_api_host: + try: + return reset() + except: + # Fall through to shutdown-and-start case. + pass + else: + # Server was provided by parent. Can't recover if it's + # unresettable. + return reset() + + # Before trying to start up our own server, call stop() to avoid + # "Phusion Passenger Standalone is already running on PID 12345". + # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so + # we know the server is ours to kill.) + stop(force=True) restore_cwd = os.getcwd() - os.chdir(os.path.join(SERVICES_SRC_DIR, 'api')) + api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api') + os.chdir(api_src_dir) # Either we haven't started a server of our own yet, or it has # died, or we have lost our credentials, or something else is # preventing us from calling reset(). Start a new one. + if not os.path.exists('tmp'): + os.makedirs('tmp') + + if not os.path.exists('tmp/api'): + os.makedirs('tmp/api') + + if not os.path.exists('tmp/logs'): + os.makedirs('tmp/logs') + if not os.path.exists('tmp/self-signed.pem'): # We assume here that either passenger reports its listening # address as https:/0.0.0.0:port/. If it reports "127.0.0.1" @@ -146,9 +210,17 @@ def run(leave_running_atexit=False): '-out', 'tmp/self-signed.pem', '-keyout', 'tmp/self-signed.key', '-days', '3650', - '-subj', '/CN=0.0.0.0']) + '-subj', '/CN=0.0.0.0'], + stdout=sys.stderr) - port = random.randint(20000, 40000) + # Install the git repository fixtures. + gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git') + gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar') + if not os.path.isdir(gitdir): + os.makedirs(gitdir) + subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball]) + + port = find_available_port() env = os.environ.copy() env['RAILS_ENV'] = 'test' env['ARVADOS_WEBSOCKETS'] = 'yes' @@ -167,7 +239,7 @@ def run(leave_running_atexit=False): env=env) if not leave_running_atexit: - atexit.register(kill_server_pid, pid_file, passenger=True) + atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir) match = re.search(r'Accessible via: https://(.*?)/', start_msg) if not match: @@ -176,8 +248,10 @@ def run(leave_running_atexit=False): my_api_host = match.group(1) os.environ['ARVADOS_API_HOST'] = my_api_host - # Make sure the server has written its pid file before continuing + # Make sure the server has written its pid file and started + # listening on its TCP port find_server_pid(pid_file) + _wait_until_port_listens(port) reset() os.chdir(restore_cwd) @@ -187,6 +261,9 @@ def reset(): This resets the ARVADOS_TEST_API_HOST provided by a parent process if any, otherwise the server started by run(). + + It also resets ARVADOS_* environment vars to point to the test + server with admin credentials. """ existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host) token = auth_token('admin') @@ -196,6 +273,9 @@ def reset(): 'https://{}/database/reset'.format(existing_api_host), 'POST', headers={'Authorization': 'OAuth2 {}'.format(token)}) + os.environ['ARVADOS_API_HOST_INSECURE'] = 'true' + os.environ['ARVADOS_API_HOST'] = existing_api_host + os.environ['ARVADOS_API_TOKEN'] = token def stop(force=False): """Stop the API server, if one is running. @@ -218,46 +298,56 @@ def stop(force=False): def _start_keep(n, keep_args): keep0 = tempfile.mkdtemp() - port = random.randint(20000, 40000) + port = find_available_port() keep_cmd = ["keepstore", - "-volumes={}".format(keep0), + "-volume={}".format(keep0), "-listen=:{}".format(port), - "-pid={}".format("{}/keep{}.pid".format(TEST_TMPDIR, n))] + "-pid="+_pidfile('keep{}'.format(n))] for arg, val in keep_args.iteritems(): keep_cmd.append("{}={}".format(arg, val)) - kp0 = subprocess.Popen(keep_cmd) - with open("{}/keep{}.pid".format(TEST_TMPDIR, n), 'w') as f: + logf = open(os.path.join(TEST_TMPDIR, 'keep{}.log'.format(n)), 'a+') + kp0 = subprocess.Popen( + keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) + with open(_pidfile('keep{}'.format(n)), 'w') as f: f.write(str(kp0.pid)) with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f: f.write(keep0) + _wait_until_port_listens(port) + return port -def run_keep(blob_signing_key=None, enforce_permissions=False): - stop_keep() +def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2): + stop_keep(num_servers) keep_args = {} - if blob_signing_key: - with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f: - keep_args['--permission-key-file'] = f.name - f.write(blob_signing_key) + if not blob_signing_key: + blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc' + with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f: + keep_args['-blob-signing-key-file'] = f.name + f.write(blob_signing_key) if enforce_permissions: - keep_args['--enforce-permissions'] = 'true' + keep_args['-enforce-permissions'] = 'true' + with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f: + keep_args['-data-manager-token-file'] = f.name + f.write(auth_token('data_manager')) + keep_args['-never-delete'] = 'false' api = arvados.api( - 'v1', cache=False, + version='v1', host=os.environ['ARVADOS_API_HOST'], token=os.environ['ARVADOS_API_TOKEN'], insecure=True) + for d in api.keep_services().list().execute()['items']: api.keep_services().delete(uuid=d['uuid']).execute() for d in api.keep_disks().list().execute()['items']: api.keep_disks().delete(uuid=d['uuid']).execute() - for d in range(0, 2): + for d in range(0, num_servers): port = _start_keep(d, keep_args) svc = api.keep_services().create(body={'keep_service': { 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d), @@ -271,7 +361,7 @@ def run_keep(blob_signing_key=None, enforce_permissions=False): }).execute() def _stop_keep(n): - kill_server_pid("{}/keep{}.pid".format(TEST_TMPDIR, n), 0) + kill_server_pid(_pidfile('keep{}'.format(n)), 0) if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)): with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r: shutil.rmtree(r.read(), True) @@ -279,25 +369,27 @@ def _stop_keep(n): if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")): os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")) -def stop_keep(): - _stop_keep(0) - _stop_keep(1) +def stop_keep(num_servers=2): + for n in range(0, num_servers): + _stop_keep(n) def run_keep_proxy(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return stop_keep_proxy() admin_token = auth_token('admin') - port = random.randint(20000,40000) + port = find_available_port() env = os.environ.copy() env['ARVADOS_API_TOKEN'] = admin_token kp = subprocess.Popen( ['keepproxy', - '-pid={}/keepproxy.pid'.format(TEST_TMPDIR), + '-pid='+_pidfile('keepproxy'), '-listen=:{}'.format(port)], - env=env) + env=env, stdin=open('/dev/null'), stdout=sys.stderr) api = arvados.api( - 'v1', cache=False, + version='v1', host=os.environ['ARVADOS_API_HOST'], token=admin_token, insecure=True) @@ -311,9 +403,117 @@ def run_keep_proxy(): 'service_ssl_flag': False, }}).execute() os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port) + _setport('keepproxy', port) + _wait_until_port_listens(port) def stop_keep_proxy(): - kill_server_pid(os.path.join(TEST_TMPDIR, "keepproxy.pid"), 0) + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + kill_server_pid(_pidfile('keepproxy'), wait=0) + +def run_arv_git_httpd(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + stop_arv_git_httpd() + + gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git') + gitport = find_available_port() + env = os.environ.copy() + env.pop('ARVADOS_API_TOKEN', None) + agh = subprocess.Popen( + ['arv-git-httpd', + '-repo-root='+gitdir+'/test', + '-address=:'+str(gitport)], + env=env, stdin=open('/dev/null'), stdout=sys.stderr) + with open(_pidfile('arv-git-httpd'), 'w') as f: + f.write(str(agh.pid)) + _setport('arv-git-httpd', gitport) + _wait_until_port_listens(gitport) + +def stop_arv_git_httpd(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + kill_server_pid(_pidfile('arv-git-httpd'), wait=0) + +def run_nginx(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + nginxconf = {} + nginxconf['KEEPPROXYPORT'] = _getport('keepproxy') + nginxconf['KEEPPROXYSSLPORT'] = find_available_port() + nginxconf['GITPORT'] = _getport('arv-git-httpd') + nginxconf['GITSSLPORT'] = find_available_port() + nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem') + nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key') + nginxconf['ACCESSLOG'] = os.path.join(TEST_TMPDIR, 'nginx_access_log.fifo') + + conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf') + conffile = os.path.join(TEST_TMPDIR, 'nginx.conf') + with open(conffile, 'w') as f: + f.write(re.sub( + r'{{([A-Z]+)}}', + lambda match: str(nginxconf.get(match.group(1))), + open(conftemplatefile).read())) + + env = os.environ.copy() + env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin' + + try: + os.remove(nginxconf['ACCESSLOG']) + except OSError as error: + if error.errno != errno.ENOENT: + raise + + os.mkfifo(nginxconf['ACCESSLOG'], 0700) + nginx = subprocess.Popen( + ['nginx', + '-g', 'error_log stderr info;', + '-g', 'pid '+_pidfile('nginx')+';', + '-c', conffile], + env=env, stdin=open('/dev/null'), stdout=sys.stderr) + cat_access = subprocess.Popen( + ['cat', nginxconf['ACCESSLOG']], + stdout=sys.stderr) + _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT']) + _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT']) + +def stop_nginx(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + kill_server_pid(_pidfile('nginx'), wait=0) + +def _pidfile(program): + return os.path.join(TEST_TMPDIR, program + '.pid') + +def _portfile(program): + return os.path.join(TEST_TMPDIR, program + '.port') + +def _setport(program, port): + with open(_portfile(program), 'w') as f: + f.write(str(port)) + +# Returns 9 if program is not up. +def _getport(program): + try: + return int(open(_portfile(program)).read()) + except IOError: + return 9 + +def _apiconfig(key): + if _cached_config: + return _cached_config[key] + def _load(f, required=True): + fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f) + if not required and not os.path.exists(fullpath): + return {} + return yaml.load(fullpath) + cdefault = _load('application.default.yml') + csite = _load('application.yml', required=False) + _cached_config = {} + for section in [cdefault.get('common',{}), cdefault.get('test',{}), + csite.get('common',{}), csite.get('test',{})]: + _cached_config.update(section) + return _cached_config[key] def fixture(fix): '''load a fixture yaml file''' @@ -395,14 +595,24 @@ class TestCaseWithServers(unittest.TestCase): if __name__ == "__main__": - actions = ['start', 'stop', - 'start_keep', 'stop_keep', - 'start_keep_proxy', 'stop_keep_proxy'] + actions = [ + 'start', 'stop', + 'start_keep', 'stop_keep', + 'start_keep_proxy', 'stop_keep_proxy', + 'start_arv-git-httpd', 'stop_arv-git-httpd', + 'start_nginx', 'stop_nginx', + ] parser = argparse.ArgumentParser() parser.add_argument('action', type=str, help="one of {}".format(actions)) parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture') + parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired") + parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions") + args = parser.parse_args() + if args.action not in actions: + print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr) + sys.exit(1) if args.action == 'start': stop(force=('ARVADOS_TEST_API_HOST' not in os.environ)) run(leave_running_atexit=True) @@ -417,12 +627,20 @@ if __name__ == "__main__": elif args.action == 'stop': stop(force=('ARVADOS_TEST_API_HOST' not in os.environ)) elif args.action == 'start_keep': - run_keep() + run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers) elif args.action == 'stop_keep': stop_keep() elif args.action == 'start_keep_proxy': run_keep_proxy() elif args.action == 'stop_keep_proxy': stop_keep_proxy() + elif args.action == 'start_arv-git-httpd': + run_arv_git_httpd() + elif args.action == 'stop_arv-git-httpd': + stop_arv_git_httpd() + elif args.action == 'start_nginx': + run_nginx() + elif args.action == 'stop_nginx': + stop_nginx() else: - print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions)) + raise Exception("action recognized but not implemented!?")