X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/644f5de63e2b8b02e054fcbb3e9af39560cffae3..f87210bb3a0a48db83d94bf424c6e5c8ba9060d3:/sdk/python/tests/run_test_server.py?ds=sidebyside diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index 567b3b3bfa..2e4c5be830 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -174,7 +174,7 @@ def find_available_port(): sock.close() return port -def _wait_until_port_listens(port, timeout=10): +def _wait_until_port_listens(port, timeout=10, warn=True): """Wait for a process to start listening on the given port. If nothing listens on the port within the specified timeout (given @@ -196,20 +196,29 @@ def _wait_until_port_listens(port, timeout=10): except subprocess.CalledProcessError: time.sleep(0.1) continue - return - print( - "WARNING: Nothing is listening on port {} (waited {} seconds).". - format(port, timeout), - file=sys.stderr) + return True + if warn: + print( + "WARNING: Nothing is listening on port {} (waited {} seconds).". + format(port, timeout), + file=sys.stderr) + return False + +def _logfilename(label): + """Set up a labelled log file, and return a path to write logs to. -def _fifo2stderr(label): - """Create a fifo, and copy it to stderr, prepending label to each line. + Normally, the returned path is {tmpdir}/{label}.log. - Return value is the path to the new FIFO. + In debug mode, logs are also written to stderr, with [label] + prepended to each line. The returned path is a FIFO. +label+ should contain only alphanumerics: it is also used as part of the FIFO filename. + """ + logfilename = os.path.join(TEST_TMPDIR, label+'.log') + if not os.environ.get('ARVADOS_DEBUG', ''): + return logfilename fifo = os.path.join(TEST_TMPDIR, label+'.fifo') try: os.remove(fifo) @@ -217,8 +226,21 @@ def _fifo2stderr(label): if error.errno != errno.ENOENT: raise os.mkfifo(fifo, 0o700) + stdbuf = ['stdbuf', '-i0', '-oL', '-eL'] + # open(fifo, 'r') would block waiting for someone to open the fifo + # for writing, so we need a separate cat process to open it for + # us. + cat = subprocess.Popen( + stdbuf+['cat', fifo], + stdin=open('/dev/null'), + stdout=subprocess.PIPE) + tee = subprocess.Popen( + stdbuf+['tee', '-a', logfilename], + stdin=cat.stdout, + stdout=subprocess.PIPE) subprocess.Popen( - ['stdbuf', '-i0', '-oL', '-eL', 'sed', '-e', 's/^/['+label+'] /', fifo], + stdbuf+['sed', '-e', 's/^/['+label+'] /'], + stdin=tee.stdout, stdout=sys.stderr) return fifo @@ -355,8 +377,11 @@ def reset(): 'POST', headers={'Authorization': 'OAuth2 {}'.format(token)}) os.environ['ARVADOS_API_HOST_INSECURE'] = 'true' - os.environ['ARVADOS_API_HOST'] = existing_api_host os.environ['ARVADOS_API_TOKEN'] = token + if _wait_until_port_listens(_getport('controller-ssl'), timeout=0.5, warn=False): + os.environ['ARVADOS_API_HOST'] = '0.0.0.0:'+str(_getport('controller-ssl')) + else: + os.environ['ARVADOS_API_HOST'] = existing_api_host def stop(force=False): """Stop the API server, if one is running. @@ -377,6 +402,65 @@ def stop(force=False): kill_server_pid(_pidfile('api')) my_api_host = None +def run_controller(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + stop_controller() + rails_api_port = int(string.split(os.environ.get('ARVADOS_TEST_API_HOST', my_api_host), ':')[-1]) + port = find_available_port() + conf = os.path.join(TEST_TMPDIR, 'arvados.yml') + with open(conf, 'w') as f: + f.write(""" +Clusters: + zzzzz: + EnableBetaController14287: {beta14287} + ManagementToken: e687950a23c3a9bceec28c6223a06c79 + API: + RequestTimeout: 30s + Logging: + Level: "{loglevel}" + HTTPRequestTimeout: 30s + PostgreSQL: + ConnectionPool: 32 + Connection: + host: {dbhost} + dbname: {dbname} + user: {dbuser} + password: {dbpass} + TLS: + Insecure: true + Services: + Controller: + InternalURLs: + "http://localhost:{controllerport}": {{}} + RailsAPI: + InternalURLs: + "https://localhost:{railsport}": {{}} + """.format( + beta14287=('true' if '14287' in os.environ.get('ARVADOS_EXPERIMENTAL', '') else 'false'), + loglevel=('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'), + dbhost=_dbconfig('host'), + dbname=_dbconfig('dbname'), + dbuser=_dbconfig('user'), + dbpass=_dbconfig('password'), + controllerport=port, + railsport=rails_api_port, + )) + logf = open(_logfilename('controller'), 'a') + controller = subprocess.Popen( + ["arvados-server", "controller", "-config", conf], + stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) + with open(_pidfile('controller'), 'w') as f: + f.write(str(controller.pid)) + _wait_until_port_listens(port) + _setport('controller', port) + return port + +def stop_controller(): + if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + return + kill_server_pid(_pidfile('controller')) + def run_ws(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: return @@ -400,10 +484,10 @@ Postgres: port, ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'), _dbconfig('host'), - _dbconfig('database'), - _dbconfig('username'), + _dbconfig('dbname'), + _dbconfig('user'), _dbconfig('password'))) - logf = open(_fifo2stderr('ws'), 'w') + logf = open(_logfilename('ws'), 'a') ws = subprocess.Popen( ["ws", "-config", conf], stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) @@ -429,7 +513,7 @@ def _start_keep(n, keep_args): for arg, val in keep_args.items(): keep_cmd.append("{}={}".format(arg, val)) - logf = open(_fifo2stderr('keep{}'.format(n)), 'w') + logf = open(_logfilename('keep{}'.format(n)), 'a') kp0 = subprocess.Popen( keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) @@ -507,13 +591,14 @@ def stop_keep(num_servers=2): def run_keep_proxy(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: + os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(_getport('keepproxy')) return stop_keep_proxy() port = find_available_port() env = os.environ.copy() env['ARVADOS_API_TOKEN'] = auth_token('anonymous') - logf = open(_fifo2stderr('keepproxy'), 'w') + logf = open(_logfilename('keepproxy'), 'a') kp = subprocess.Popen( ['keepproxy', '-pid='+_pidfile('keepproxy'), @@ -552,10 +637,11 @@ def run_arv_git_httpd(): gitport = find_available_port() env = os.environ.copy() env.pop('ARVADOS_API_TOKEN', None) - logf = open(_fifo2stderr('arv-git-httpd'), 'w') + logf = open(_logfilename('arv-git-httpd'), 'a') agh = subprocess.Popen( ['arv-git-httpd', '-repo-root='+gitdir+'/test', + '-management-token=e687950a23c3a9bceec28c6223a06c79', '-address=:'+str(gitport)], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) with open(_pidfile('arv-git-httpd'), 'w') as f: @@ -576,11 +662,12 @@ def run_keep_web(): keepwebport = find_available_port() env = os.environ.copy() env['ARVADOS_API_TOKEN'] = auth_token('anonymous') - logf = open(_fifo2stderr('keep-web'), 'w') + logf = open(_logfilename('keep-web'), 'a') keepweb = subprocess.Popen( ['keep-web', '-allow-anonymous', - '-attachment-only-host=download:'+str(keepwebport), + '-attachment-only-host=download', + '-management-token=e687950a23c3a9bceec28c6223a06c79', '-listen=:'+str(keepwebport)], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) with open(_pidfile('keep-web'), 'w') as f: @@ -598,6 +685,8 @@ def run_nginx(): return stop_nginx() nginxconf = {} + nginxconf['CONTROLLERPORT'] = _getport('controller') + nginxconf['CONTROLLERSSLPORT'] = find_available_port() nginxconf['KEEPWEBPORT'] = _getport('keep-web') nginxconf['KEEPWEBDLSSLPORT'] = find_available_port() nginxconf['KEEPWEBSSLPORT'] = find_available_port() @@ -609,7 +698,9 @@ def run_nginx(): nginxconf['WSSPORT'] = _getport('wss') nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem') nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key') - nginxconf['ACCESSLOG'] = _fifo2stderr('nginx_access_log') + nginxconf['ACCESSLOG'] = _logfilename('nginx_access') + nginxconf['ERRORLOG'] = _logfilename('nginx_error') + nginxconf['TMPDIR'] = TEST_TMPDIR conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf') conffile = os.path.join(TEST_TMPDIR, 'nginx.conf') @@ -628,6 +719,7 @@ def run_nginx(): '-g', 'pid '+_pidfile('nginx')+';', '-c', conffile], env=env, stdin=open('/dev/null'), stdout=sys.stderr) + _setport('controller-ssl', nginxconf['CONTROLLERSSLPORT']) _setport('keep-web-dl-ssl', nginxconf['KEEPWEBDLSSLPORT']) _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT']) _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT']) @@ -658,9 +750,14 @@ def _getport(program): def _dbconfig(key): global _cached_db_config if not _cached_db_config: - _cached_db_config = yaml.load(open(os.path.join( - SERVICES_SRC_DIR, 'api', 'config', 'database.yml'))) - return _cached_db_config['test'][key] + if "ARVADOS_CONFIG" in os.environ: + _cached_db_config = list(yaml.safe_load(open(os.environ["ARVADOS_CONFIG"]))["Clusters"].values())[0]["PostgreSQL"]["Connection"] + else: + _cached_db_config = yaml.safe_load(open(os.path.join( + SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))["test"] + _cached_db_config["dbname"] = _cached_db_config["database"] + _cached_db_config["user"] = _cached_db_config["username"] + return _cached_db_config[key] def _apiconfig(key): global _cached_config @@ -670,7 +767,7 @@ def _apiconfig(key): fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f) if not required and not os.path.exists(fullpath): return {} - return yaml.load(fullpath) + return yaml.safe_load(fullpath) cdefault = _load('application.default.yml') csite = _load('application.yml', required=False) _cached_config = {} @@ -689,7 +786,7 @@ def fixture(fix): yaml_file = yaml_file[0:trim_index] except ValueError: pass - return yaml.load(yaml_file) + return yaml.safe_load(yaml_file) def auth_token(token_name): return fixture("api_client_authorizations")[token_name]["api_token"] @@ -766,6 +863,7 @@ if __name__ == "__main__": actions = [ 'start', 'stop', 'start_ws', 'stop_ws', + 'start_controller', 'stop_controller', 'start_keep', 'stop_keep', 'start_keep_proxy', 'stop_keep_proxy', 'start_keep-web', 'stop_keep-web', @@ -802,6 +900,10 @@ if __name__ == "__main__": run_ws() elif args.action == 'stop_ws': stop_ws() + elif args.action == 'start_controller': + run_controller() + elif args.action == 'stop_controller': + stop_controller() elif args.action == 'start_keep': run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers) elif args.action == 'stop_keep': @@ -820,6 +922,7 @@ if __name__ == "__main__": stop_keep_web() elif args.action == 'start_nginx': run_nginx() + print("export ARVADOS_API_HOST=0.0.0.0:{}".format(_getport('controller-ssl'))) elif args.action == 'stop_nginx': stop_nginx() else: