13497: Rename SystemNodes to NodeProfiles in config.

[arvados.git] / sdk / python / tests / run_test_server.py

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index f0fbfe74217946267e346f2c68eb71fe6f268973..f7ca6daf6f65c190ccd22a1a04ad3cb1996f3e75 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -202,14 +202,21 @@ def _wait_until_port_listens(port, timeout=10):
         format(port, timeout),
         file=sys.stderr)
 
-def _fifo2stderr(label):
-    """Create a fifo, and copy it to stderr, prepending label to each line.
+def _logfilename(label):
+    """Set up a labelled log file, and return a path to write logs to.
 
-    Return value is the path to the new FIFO.
+    Normally, the returned path is {tmpdir}/{label}.log.
+
+    In debug mode, logs are also written to stderr, with [label]
+    prepended to each line. The returned path is a FIFO.
 
     +label+ should contain only alphanumerics: it is also used as part
     of the FIFO filename.
+
     """
+    logfilename = os.path.join(TEST_TMPDIR, label+'.log')
+    if not os.environ.get('ARVADOS_DEBUG', ''):
+        return logfilename
     fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
     try:
         os.remove(fifo)
@@ -217,8 +224,21 @@ def _fifo2stderr(label):
         if error.errno != errno.ENOENT:
             raise
     os.mkfifo(fifo, 0o700)
+    stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
+    # open(fifo, 'r') would block waiting for someone to open the fifo
+    # for writing, so we need a separate cat process to open it for
+    # us.
+    cat = subprocess.Popen(
+        stdbuf+['cat', fifo],
+        stdin=open('/dev/null'),
+        stdout=subprocess.PIPE)
+    tee = subprocess.Popen(
+        stdbuf+['tee', '-a', logfilename],
+        stdin=cat.stdout,
+        stdout=subprocess.PIPE)
     subprocess.Popen(
-        ['stdbuf', '-i0', '-oL', '-eL', 'sed', '-e', 's/^/['+label+'] /', fifo],
+        stdbuf+['sed', '-e', 's/^/['+label+'] /'],
+        stdin=tee.stdout,
         stdout=sys.stderr)
     return fifo
 
@@ -388,7 +408,7 @@ def run_controller():
         f.write("""
 Clusters:
   zzzzz:
-    SystemNodes:
+    NodeProfiles:
       "*":
         "arvados-controller":
           Listen: ":{}"
@@ -396,7 +416,7 @@ Clusters:
           Listen: ":{}"
           TLS: true
         """.format(port, rails_api_port))
-    logf = open(_fifo2stderr('controller'), 'w')
+    logf = open(_logfilename('controller'), 'a')
     controller = subprocess.Popen(
         ["arvados-server", "controller", "-config", conf],
         stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
@@ -437,7 +457,7 @@ Postgres:
                    _dbconfig('database'),
                    _dbconfig('username'),
                    _dbconfig('password')))
-    logf = open(_fifo2stderr('ws'), 'w')
+    logf = open(_logfilename('ws'), 'a')
     ws = subprocess.Popen(
         ["ws", "-config", conf],
         stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
@@ -463,7 +483,7 @@ def _start_keep(n, keep_args):
     for arg, val in keep_args.items():
         keep_cmd.append("{}={}".format(arg, val))
 
-    logf = open(_fifo2stderr('keep{}'.format(n)), 'w')
+    logf = open(_logfilename('keep{}'.format(n)), 'a')
     kp0 = subprocess.Popen(
         keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
 
@@ -547,7 +567,7 @@ def run_keep_proxy():
     port = find_available_port()
     env = os.environ.copy()
     env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
-    logf = open(_fifo2stderr('keepproxy'), 'w')
+    logf = open(_logfilename('keepproxy'), 'a')
     kp = subprocess.Popen(
         ['keepproxy',
          '-pid='+_pidfile('keepproxy'),
@@ -586,7 +606,7 @@ def run_arv_git_httpd():
     gitport = find_available_port()
     env = os.environ.copy()
     env.pop('ARVADOS_API_TOKEN', None)
-    logf = open(_fifo2stderr('arv-git-httpd'), 'w')
+    logf = open(_logfilename('arv-git-httpd'), 'a')
     agh = subprocess.Popen(
         ['arv-git-httpd',
          '-repo-root='+gitdir+'/test',
@@ -610,7 +630,7 @@ def run_keep_web():
     keepwebport = find_available_port()
     env = os.environ.copy()
     env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
-    logf = open(_fifo2stderr('keep-web'), 'w')
+    logf = open(_logfilename('keep-web'), 'a')
     keepweb = subprocess.Popen(
         ['keep-web',
          '-allow-anonymous',
@@ -645,7 +665,8 @@ def run_nginx():
     nginxconf['WSSPORT'] = _getport('wss')
     nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
     nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
-    nginxconf['ACCESSLOG'] = _fifo2stderr('nginx_access_log')
+    nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
+    nginxconf['ERRORLOG'] = _logfilename('nginx_error')
     nginxconf['TMPDIR'] = TEST_TMPDIR
 
     conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')