X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c59093180fd92f0d7c6607a49458446212ebd058..8e1e7e6bf355eb0f1defc7278f1434c393a75a75:/sdk/python/tests/run_test_server.py diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index 271b525df2..972b7f9d51 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -3,6 +3,7 @@ from __future__ import print_function import argparse import atexit +import errno import httplib2 import os import pipes @@ -54,9 +55,7 @@ def find_server_pid(PID_PATH, wait=10): with open(PID_PATH, 'r') as f: server_pid = int(f.read()) good_pid = (os.kill(server_pid, 0) is None) - except IOError: - good_pid = False - except OSError: + except EnvironmentError: good_pid = False now = time.time() @@ -91,9 +90,7 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False): os.getpgid(server_pid) time.sleep(0.1) now = time.time() - except IOError: - pass - except OSError: + except EnvironmentError: pass def find_available_port(): @@ -115,6 +112,33 @@ def find_available_port(): sock.close() return port +def _wait_until_port_listens(port, timeout=10): + """Wait for a process to start listening on the given port. + + If nothing listens on the port within the specified timeout (given + in seconds), print a warning on stderr before returning. + """ + try: + subprocess.check_output(['which', 'lsof']) + except subprocess.CalledProcessError: + print("WARNING: No `lsof` -- cannot wait for port to listen. "+ + "Sleeping 0.5 and hoping for the best.") + time.sleep(0.5) + return + deadline = time.time() + timeout + while time.time() < deadline: + try: + subprocess.check_output( + ['lsof', '-t', '-i', 'tcp:'+str(port)]) + except subprocess.CalledProcessError: + time.sleep(0.1) + continue + return + print( + "WARNING: Nothing is listening on port {} (waited {} seconds).". + format(port, timeout), + file=sys.stderr) + def run(leave_running_atexit=False): """Ensure an API server is running, and ARVADOS_API_* env vars have admin credentials for it. @@ -165,6 +189,15 @@ def run(leave_running_atexit=False): # died, or we have lost our credentials, or something else is # preventing us from calling reset(). Start a new one. + if not os.path.exists('tmp'): + os.makedirs('tmp') + + if not os.path.exists('tmp/api'): + os.makedirs('tmp/api') + + if not os.path.exists('tmp/logs'): + os.makedirs('tmp/logs') + if not os.path.exists('tmp/self-signed.pem'): # We assume here that either passenger reports its listening # address as https:/0.0.0.0:port/. If it reports "127.0.0.1" @@ -215,8 +248,10 @@ def run(leave_running_atexit=False): my_api_host = match.group(1) os.environ['ARVADOS_API_HOST'] = my_api_host - # Make sure the server has written its pid file before continuing + # Make sure the server has written its pid file and started + # listening on its TCP port find_server_pid(pid_file) + _wait_until_port_listens(port) reset() os.chdir(restore_cwd) @@ -272,38 +307,47 @@ def _start_keep(n, keep_args): for arg, val in keep_args.iteritems(): keep_cmd.append("{}={}".format(arg, val)) + logf = open(os.path.join(TEST_TMPDIR, 'keep{}.log'.format(n)), 'a+') kp0 = subprocess.Popen( - keep_cmd, stdin=open('/dev/null'), stdout=sys.stderr) + keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) with open(_pidfile('keep{}'.format(n)), 'w') as f: f.write(str(kp0.pid)) with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f: f.write(keep0) + _wait_until_port_listens(port) + return port -def run_keep(blob_signing_key=None, enforce_permissions=False): - stop_keep() +def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2): + stop_keep(num_servers) keep_args = {} - if blob_signing_key: - with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f: - keep_args['--permission-key-file'] = f.name - f.write(blob_signing_key) + if not blob_signing_key: + blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc' + with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f: + keep_args['-blob-signing-key-file'] = f.name + f.write(blob_signing_key) if enforce_permissions: - keep_args['--enforce-permissions'] = 'true' + keep_args['-enforce-permissions'] = 'true' + with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f: + keep_args['-data-manager-token-file'] = f.name + f.write(auth_token('data_manager')) + keep_args['-never-delete'] = 'false' api = arvados.api( version='v1', host=os.environ['ARVADOS_API_HOST'], token=os.environ['ARVADOS_API_TOKEN'], insecure=True) + for d in api.keep_services().list().execute()['items']: api.keep_services().delete(uuid=d['uuid']).execute() for d in api.keep_disks().list().execute()['items']: api.keep_disks().delete(uuid=d['uuid']).execute() - for d in range(0, 2): + for d in range(0, num_servers): port = _start_keep(d, keep_args) svc = api.keep_services().create(body={'keep_service': { 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d), @@ -325,9 +369,9 @@ def _stop_keep(n): if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")): os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")) -def stop_keep(): - _stop_keep(0) - _stop_keep(1) +def stop_keep(num_servers=2): + for n in range(0, num_servers): + _stop_keep(n) def run_keep_proxy(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: @@ -360,6 +404,7 @@ def run_keep_proxy(): }}).execute() os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port) _setport('keepproxy', port) + _wait_until_port_listens(port) def stop_keep_proxy(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: @@ -383,6 +428,7 @@ def run_arv_git_httpd(): with open(_pidfile('arv-git-httpd'), 'w') as f: f.write(str(agh.pid)) _setport('arv-git-httpd', gitport) + _wait_until_port_listens(gitport) def stop_arv_git_httpd(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: @@ -399,6 +445,7 @@ def run_nginx(): nginxconf['GITSSLPORT'] = find_available_port() nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem') nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key') + nginxconf['ACCESSLOG'] = os.path.join(TEST_TMPDIR, 'nginx_access_log.fifo') conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf') conffile = os.path.join(TEST_TMPDIR, 'nginx.conf') @@ -410,12 +457,23 @@ def run_nginx(): env = os.environ.copy() env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin' + + try: + os.remove(nginxconf['ACCESSLOG']) + except OSError as error: + if error.errno != errno.ENOENT: + raise + + os.mkfifo(nginxconf['ACCESSLOG'], 0700) nginx = subprocess.Popen( ['nginx', '-g', 'error_log stderr info;', '-g', 'pid '+_pidfile('nginx')+';', '-c', conffile], env=env, stdin=open('/dev/null'), stdout=sys.stderr) + cat_access = subprocess.Popen( + ['cat', nginxconf['ACCESSLOG']], + stdout=sys.stderr) _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT']) _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT']) @@ -444,10 +502,13 @@ def _getport(program): def _apiconfig(key): if _cached_config: return _cached_config[key] - def _load(f): - return yaml.load(os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)) + def _load(f, required=True): + fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f) + if not required and not os.path.exists(fullpath): + return {} + return yaml.load(fullpath) cdefault = _load('application.default.yml') - csite = _load('application.yml') + csite = _load('application.yml', required=False) _cached_config = {} for section in [cdefault.get('common',{}), cdefault.get('test',{}), csite.get('common',{}), csite.get('test',{})]: @@ -544,6 +605,9 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('action', type=str, help="one of {}".format(actions)) parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture') + parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired") + parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions") + args = parser.parse_args() if args.action not in actions: @@ -563,7 +627,7 @@ if __name__ == "__main__": elif args.action == 'stop': stop(force=('ARVADOS_TEST_API_HOST' not in os.environ)) elif args.action == 'start_keep': - run_keep() + run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers) elif args.action == 'stop_keep': stop_keep() elif args.action == 'start_keep_proxy':