X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/7db74f672f64b3e647a98c1d8e5978b50d79538d..404f868129e22cd627d350d61f74806f5b31a8ad:/sdk/python/tests/run_test_server.py diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index eb2784c714..787837b723 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -2,10 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -from __future__ import print_function -from __future__ import division -from builtins import str -from builtins import range import argparse import atexit import errno @@ -18,7 +14,6 @@ import shlex import shutil import signal import socket -import string import subprocess import sys import tempfile @@ -26,10 +21,7 @@ import time import unittest import yaml -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse +from urllib.parse import urlparse MY_DIRNAME = os.path.dirname(os.path.realpath(__file__)) if __name__ == '__main__' and os.path.exists( @@ -41,6 +33,15 @@ if __name__ == '__main__' and os.path.exists( import arvados import arvados.config +# This module starts subprocesses and records them in pidfiles so they +# can be managed by other processes (incl. after this process +# exits). But if we don't keep a reference to each subprocess object +# somewhere, the subprocess destructor runs, and we get a lot of +# ResourceWarning noise in test logs. This is our bucket of subprocess +# objects whose destructors we don't want to run but are otherwise +# unneeded. +_detachedSubprocesses = [] + ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..')) SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services') @@ -248,14 +249,17 @@ def _logfilename(label): stdbuf+['cat', fifo], stdin=open('/dev/null'), stdout=subprocess.PIPE) + _detachedSubprocesses.append(cat) tee = subprocess.Popen( stdbuf+['tee', '-a', logfilename], stdin=cat.stdout, stdout=subprocess.PIPE) - subprocess.Popen( + _detachedSubprocesses.append(tee) + sed = subprocess.Popen( stdbuf+['sed', '-e', 's/^/['+label+'] /'], stdin=tee.stdout, stdout=sys.stderr) + _detachedSubprocesses.append(sed) return fifo def run(leave_running_atexit=False): @@ -367,6 +371,7 @@ def run(leave_running_atexit=False): '--ssl-certificate', 'tmp/self-signed.pem', '--ssl-certificate-key', 'tmp/self-signed.key'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) + _detachedSubprocesses.append(railsapi) if not leave_running_atexit: atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir) @@ -444,6 +449,7 @@ def run_controller(): controller = subprocess.Popen( ["arvados-server", "controller"], stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) + _detachedSubprocesses.append(controller) with open(_pidfile('controller'), 'w') as f: f.write(str(controller.pid)) _wait_until_port_listens(port) @@ -463,6 +469,7 @@ def run_ws(): ws = subprocess.Popen( ["arvados-server", "ws"], stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) + _detachedSubprocesses.append(ws) with open(_pidfile('ws'), 'w') as f: f.write(str(ws.pid)) _wait_until_port_listens(port) @@ -496,6 +503,7 @@ def _start_keep(n, blob_signing=False): with open('/dev/null') as _stdin: child = subprocess.Popen( keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True) + _detachedSubprocesses.append(child) print('child.pid is %d'%child.pid, file=sys.stderr) with open(_pidfile('keep{}'.format(n)), 'w') as f: @@ -562,6 +570,7 @@ def run_keep_proxy(): logf = open(_logfilename('keepproxy'), WRITE_MODE) kp = subprocess.Popen( ['arvados-server', 'keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) + _detachedSubprocesses.append(kp) with open(_pidfile('keepproxy'), 'w') as f: f.write(str(kp.pid)) @@ -601,6 +610,7 @@ def run_arv_git_httpd(): logf = open(_logfilename('githttpd'), WRITE_MODE) agh = subprocess.Popen(['arvados-server', 'git-httpd'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) + _detachedSubprocesses.append(agh) with open(_pidfile('githttpd'), 'w') as f: f.write(str(agh.pid)) _wait_until_port_listens(gitport) @@ -621,6 +631,7 @@ def run_keep_web(): keepweb = subprocess.Popen( ['arvados-server', 'keep-web'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) + _detachedSubprocesses.append(keepweb) with open(_pidfile('keep-web'), 'w') as f: f.write(str(keepweb.pid)) _wait_until_port_listens(keepwebport) @@ -651,7 +662,6 @@ def run_nginx(): nginxconf['HEALTHSSLPORT'] = external_port_from_config("Health") nginxconf['WSPORT'] = internal_port_from_config("Websocket") nginxconf['WSSSLPORT'] = external_port_from_config("Websocket") - nginxconf['WORKBENCH1PORT'] = internal_port_from_config("Workbench1") nginxconf['WORKBENCH1SSLPORT'] = external_port_from_config("Workbench1") nginxconf['WORKBENCH2PORT'] = internal_port_from_config("Workbench2") nginxconf['WORKBENCH2SSLPORT'] = external_port_from_config("Workbench2") @@ -678,6 +688,7 @@ def run_nginx(): '-g', 'error_log stderr info; pid '+_pidfile('nginx')+';', '-c', conffile], env=env, stdin=open('/dev/null'), stdout=sys.stderr) + _detachedSubprocesses.append(nginx) _wait_until_port_listens(nginxconf['CONTROLLERSSLPORT']) def setup_config(): @@ -686,7 +697,6 @@ def setup_config(): controller_external_port = find_available_port() websocket_port = find_available_port() websocket_external_port = find_available_port() - workbench1_port = find_available_port() workbench1_external_port = find_available_port() workbench2_port = find_available_port() workbench2_external_port = find_available_port() @@ -738,9 +748,6 @@ def setup_config(): }, "Workbench1": { "ExternalURL": "https://%s:%s/" % (localhost, workbench1_external_port), - "InternalURLs": { - "http://%s:%s"%(localhost, workbench1_port): {}, - }, }, "Workbench2": { "ExternalURL": "https://%s:%s/" % (localhost, workbench2_external_port),