19830: Clean run_test_server imports

[arvados.git] / sdk / python / tests / run_test_server.py

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index b14ad141e07e776acc428450e95e622bd84be2c4..a2cc6030a6184e53fde9e3ebb9179a874f70833f 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -2,10 +2,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from __future__ import print_function
-from __future__ import division
-from builtins import str
-from builtins import range
 import argparse
 import atexit
 import errno
@@ -18,7 +14,6 @@ import shlex
 import shutil
 import signal
 import socket
-import string
 import subprocess
 import sys
 import tempfile
@@ -26,10 +21,7 @@ import time
 import unittest
 import yaml
 
-try:
-    from urllib.parse import urlparse
-except ImportError:
-    from urlparse import urlparse
+from urllib.parse import urlparse
 
 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
 if __name__ == '__main__' and os.path.exists(
@@ -41,6 +33,15 @@ if __name__ == '__main__' and os.path.exists(
 import arvados
 import arvados.config
 
+# This module starts subprocesses and records them in pidfiles so they
+# can be managed by other processes (incl. after this process
+# exits). But if we don't keep a reference to each subprocess object
+# somewhere, the subprocess destructor runs, and we get a lot of
+# ResourceWarning noise in test logs. This is our bucket of subprocess
+# objects whose destructors we don't want to run but are otherwise
+# unneeded.
+_detachedSubprocesses = []
+
 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
 
@@ -248,14 +249,17 @@ def _logfilename(label):
         stdbuf+['cat', fifo],
         stdin=open('/dev/null'),
         stdout=subprocess.PIPE)
+    _detachedSubprocesses.append(cat)
     tee = subprocess.Popen(
         stdbuf+['tee', '-a', logfilename],
         stdin=cat.stdout,
         stdout=subprocess.PIPE)
-    subprocess.Popen(
+    _detachedSubprocesses.append(tee)
+    sed = subprocess.Popen(
         stdbuf+['sed', '-e', 's/^/['+label+'] /'],
         stdin=tee.stdout,
         stdout=sys.stderr)
+    _detachedSubprocesses.append(sed)
     return fifo
 
 def run(leave_running_atexit=False):
@@ -338,7 +342,7 @@ def run(leave_running_atexit=False):
     resdir = subprocess.check_output(['bundle', 'exec', 'passenger-config', 'about', 'resourcesdir']).decode().rstrip()
     with open(resdir + '/templates/standalone/config.erb') as f:
         template = f.read()
-    newtemplate = re.sub('http {', 'http {\n        passenger_stat_throttle_rate 0;', template)
+    newtemplate = re.sub(r'http \{', 'http {\n        passenger_stat_throttle_rate 0;', template)
     if newtemplate == template:
         raise "template edit failed"
     with open('tmp/passenger-nginx.conf.erb', 'w') as f:
@@ -367,6 +371,7 @@ def run(leave_running_atexit=False):
          '--ssl-certificate', 'tmp/self-signed.pem',
          '--ssl-certificate-key', 'tmp/self-signed.key'],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+    _detachedSubprocesses.append(railsapi)
 
     if not leave_running_atexit:
         atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
@@ -444,6 +449,7 @@ def run_controller():
     controller = subprocess.Popen(
         ["arvados-server", "controller"],
         stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+    _detachedSubprocesses.append(controller)
     with open(_pidfile('controller'), 'w') as f:
         f.write(str(controller.pid))
     _wait_until_port_listens(port)
@@ -463,6 +469,7 @@ def run_ws():
     ws = subprocess.Popen(
         ["arvados-server", "ws"],
         stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+    _detachedSubprocesses.append(ws)
     with open(_pidfile('ws'), 'w') as f:
         f.write(str(ws.pid))
     _wait_until_port_listens(port)
@@ -496,6 +503,7 @@ def _start_keep(n, blob_signing=False):
         with open('/dev/null') as _stdin:
             child = subprocess.Popen(
                 keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
+            _detachedSubprocesses.append(child)
 
     print('child.pid is %d'%child.pid, file=sys.stderr)
     with open(_pidfile('keep{}'.format(n)), 'w') as f:
@@ -562,6 +570,7 @@ def run_keep_proxy():
     logf = open(_logfilename('keepproxy'), WRITE_MODE)
     kp = subprocess.Popen(
         ['arvados-server', 'keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+    _detachedSubprocesses.append(kp)
 
     with open(_pidfile('keepproxy'), 'w') as f:
         f.write(str(kp.pid))
@@ -601,6 +610,7 @@ def run_arv_git_httpd():
     logf = open(_logfilename('githttpd'), WRITE_MODE)
     agh = subprocess.Popen(['arvados-server', 'git-httpd'],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+    _detachedSubprocesses.append(agh)
     with open(_pidfile('githttpd'), 'w') as f:
         f.write(str(agh.pid))
     _wait_until_port_listens(gitport)
@@ -621,6 +631,7 @@ def run_keep_web():
     keepweb = subprocess.Popen(
         ['arvados-server', 'keep-web'],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+    _detachedSubprocesses.append(keepweb)
     with open(_pidfile('keep-web'), 'w') as f:
         f.write(str(keepweb.pid))
     _wait_until_port_listens(keepwebport)
@@ -678,6 +689,7 @@ def run_nginx():
          '-g', 'error_log stderr info; pid '+_pidfile('nginx')+';',
          '-c', conffile],
         env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+    _detachedSubprocesses.append(nginx)
     _wait_until_port_listens(nginxconf['CONTROLLERSSLPORT'])
 
 def setup_config():