15531: Refactor federation-migrate
[arvados.git] / sdk / python / tests / run_test_server.py
index 642b7ccbad51846a9f1c25acc86f6b0505897c62..34342059f3062820313127514c47e0061f080a77 100644 (file)
@@ -1,6 +1,11 @@
-#!/usr/bin/env python
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
 
 from __future__ import print_function
+from __future__ import division
+from builtins import str
+from builtins import range
 import argparse
 import atexit
 import errno
@@ -21,6 +26,11 @@ import time
 import unittest
 import yaml
 
+try:
+    from urllib.parse import urlparse
+except ImportError:
+    from urlparse import urlparse
+
 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
 if __name__ == '__main__' and os.path.exists(
       os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
@@ -34,9 +44,19 @@ import arvados.config
 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
 if 'GOPATH' in os.environ:
+    # Add all GOPATH bin dirs to PATH -- but insert them after the
+    # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
+    # command, not the golang.org/x/tools/cmd/bundle command.
     gopaths = os.environ['GOPATH'].split(':')
-    gobins = [os.path.join(path, 'bin') for path in gopaths]
-    os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
+    addbins = [os.path.join(path, 'bin') for path in gopaths]
+    newbins = []
+    for path in os.environ['PATH'].split(':'):
+        newbins.append(path)
+        if os.path.exists(os.path.join(path, 'bundle')):
+            newbins += addbins
+            addbins = []
+    newbins += addbins
+    os.environ['PATH'] = ':'.join(newbins)
 
 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
 if not os.path.exists(TEST_TMPDIR):
@@ -44,6 +64,7 @@ if not os.path.exists(TEST_TMPDIR):
 
 my_api_host = None
 _cached_config = {}
+_cached_db_config = {}
 
 def find_server_pid(PID_PATH, wait=10):
     now = time.time()
@@ -85,7 +106,7 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
         # Use up to half of the +wait+ period waiting for "passenger
         # stop" to work. If the process hasn't exited by then, start
         # sending TERM signals.
-        startTERM += wait/2
+        startTERM += wait//2
 
     server_pid = None
     while now <= deadline and server_pid is None:
@@ -108,6 +129,7 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
         try:
             exited, _ = os.waitpid(server_pid, os.WNOHANG)
             if exited > 0:
+                _remove_pidfile(pidfile)
                 return
         except OSError:
             # already exited, or isn't our child process
@@ -121,6 +143,7 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
             if error.errno == errno.ESRCH:
                 # Thrown by os.getpgid() or os.kill() if the process
                 # does not exist, i.e., our work here is done.
+                _remove_pidfile(pidfile)
                 return
             raise
         time.sleep(0.1)
@@ -130,6 +153,13 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
           format(server_pid, pidfile, wait),
           file=sys.stderr)
 
+def _remove_pidfile(pidfile):
+    try:
+        os.unlink(pidfile)
+    except:
+        if os.path.lexists(pidfile):
+            raise
+
 def find_available_port():
     """Return an IPv4 port number that is not in use right now.
 
@@ -149,51 +179,69 @@ def find_available_port():
     sock.close()
     return port
 
-def _wait_until_port_listens(port, timeout=10):
+def _wait_until_port_listens(port, timeout=10, warn=True):
     """Wait for a process to start listening on the given port.
 
     If nothing listens on the port within the specified timeout (given
     in seconds), print a warning on stderr before returning.
     """
     try:
-        subprocess.check_output(['which', 'lsof'])
+        subprocess.check_output(['which', 'netstat'])
     except subprocess.CalledProcessError:
-        print("WARNING: No `lsof` -- cannot wait for port to listen. "+
+        print("WARNING: No `netstat` -- cannot wait for port to listen. "+
               "Sleeping 0.5 and hoping for the best.",
               file=sys.stderr)
         time.sleep(0.5)
         return
     deadline = time.time() + timeout
     while time.time() < deadline:
-        try:
-            subprocess.check_output(
-                ['lsof', '-t', '-i', 'tcp:'+str(port)])
-        except subprocess.CalledProcessError:
-            time.sleep(0.1)
-            continue
-        return
-    print(
-        "WARNING: Nothing is listening on port {} (waited {} seconds).".
-        format(port, timeout),
-        file=sys.stderr)
+        if re.search(r'\ntcp.*:'+str(port)+' .* LISTEN *\n', subprocess.check_output(['netstat', '-Wln']).decode()):
+            return True
+        time.sleep(0.1)
+    if warn:
+        print(
+            "WARNING: Nothing is listening on port {} (waited {} seconds).".
+            format(port, timeout),
+            file=sys.stderr)
+    return False
+
+def _logfilename(label):
+    """Set up a labelled log file, and return a path to write logs to.
 
-def _fifo2stderr(label):
-    """Create a fifo, and copy it to stderr, prepending label to each line.
+    Normally, the returned path is {tmpdir}/{label}.log.
 
-    Return value is the path to the new FIFO.
+    In debug mode, logs are also written to stderr, with [label]
+    prepended to each line. The returned path is a FIFO.
 
     +label+ should contain only alphanumerics: it is also used as part
     of the FIFO filename.
+
     """
+    logfilename = os.path.join(TEST_TMPDIR, label+'.log')
+    if not os.environ.get('ARVADOS_DEBUG', ''):
+        return logfilename
     fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
     try:
         os.remove(fifo)
     except OSError as error:
         if error.errno != errno.ENOENT:
             raise
-    os.mkfifo(fifo, 0700)
+    os.mkfifo(fifo, 0o700)
+    stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
+    # open(fifo, 'r') would block waiting for someone to open the fifo
+    # for writing, so we need a separate cat process to open it for
+    # us.
+    cat = subprocess.Popen(
+        stdbuf+['cat', fifo],
+        stdin=open('/dev/null'),
+        stdout=subprocess.PIPE)
+    tee = subprocess.Popen(
+        stdbuf+['tee', '-a', logfilename],
+        stdin=cat.stdout,
+        stdout=subprocess.PIPE)
     subprocess.Popen(
-        ['stdbuf', '-i0', '-oL', '-eL', 'sed', '-e', 's/^/['+label+'] /', fifo],
+        stdbuf+['sed', '-e', 's/^/['+label+'] /'],
+        stdin=tee.stdout,
         stdout=sys.stderr)
     return fifo
 
@@ -219,8 +267,9 @@ def run(leave_running_atexit=False):
     # This will clear cached docs that belong to other processes (like
     # concurrent test suites) even if they're still running. They should
     # be able to tolerate that.
-    for fn in glob.glob(os.path.join(arvados.http_cache('discovery'),
-                                     '*,arvados,v1,rest,*')):
+    for fn in glob.glob(os.path.join(
+            str(arvados.http_cache('discovery')),
+            '*,arvados,v1,rest,*')):
         os.unlink(fn)
 
     pid_file = _pidfile('api')
@@ -262,21 +311,6 @@ def run(leave_running_atexit=False):
     if not os.path.exists('tmp/logs'):
         os.makedirs('tmp/logs')
 
-    if not os.path.exists('tmp/self-signed.pem'):
-        # We assume here that either passenger reports its listening
-        # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
-        # then the certificate won't match the host and reset() will
-        # fail certificate verification. If it reports "localhost",
-        # clients (notably Python SDK's websocket client) might
-        # resolve localhost as ::1 and then fail to connect.
-        subprocess.check_call([
-            'openssl', 'req', '-new', '-x509', '-nodes',
-            '-out', 'tmp/self-signed.pem',
-            '-keyout', 'tmp/self-signed.key',
-            '-days', '3650',
-            '-subj', '/CN=0.0.0.0'],
-        stdout=sys.stderr)
-
     # Install the git repository fixtures.
     gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
     gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
@@ -284,10 +318,10 @@ def run(leave_running_atexit=False):
         os.makedirs(gitdir)
     subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
 
-    port = find_available_port()
+    port = internal_port_from_config("RailsAPI")
     env = os.environ.copy()
     env['RAILS_ENV'] = 'test'
-    env['ARVADOS_WEBSOCKETS'] = 'yes'
+    env.pop('ARVADOS_WEBSOCKETS', None)
     env.pop('ARVADOS_TEST_API_HOST', None)
     env.pop('ARVADOS_API_HOST', None)
     env.pop('ARVADOS_API_HOST_INSECURE', None)
@@ -336,10 +370,11 @@ def reset():
     httpclient.request(
         'https://{}/database/reset'.format(existing_api_host),
         'POST',
-        headers={'Authorization': 'OAuth2 {}'.format(token)})
+        headers={'Authorization': 'OAuth2 {}'.format(token), 'Connection':'close'})
+
     os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
-    os.environ['ARVADOS_API_HOST'] = existing_api_host
     os.environ['ARVADOS_API_TOKEN'] = token
+    os.environ['ARVADOS_API_HOST'] = existing_api_host
 
 def stop(force=False):
     """Stop the API server, if one is running.
@@ -360,6 +395,55 @@ def stop(force=False):
         kill_server_pid(_pidfile('api'))
         my_api_host = None
 
+def get_config():
+    with open(os.environ["ARVADOS_CONFIG"]) as f:
+        return yaml.safe_load(f)
+
+def internal_port_from_config(service):
+    return int(urlparse(
+        list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys())[0]).
+               netloc.split(":")[1])
+
+def external_port_from_config(service):
+    return int(urlparse(get_config()["Clusters"]["zzzzz"]["Services"][service]["ExternalURL"]).netloc.split(":")[1])
+
+def run_controller():
+    if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        return
+    stop_controller()
+    logf = open(_logfilename('controller'), 'a')
+    port = internal_port_from_config("Controller")
+    controller = subprocess.Popen(
+        ["arvados-server", "controller"],
+        stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+    with open(_pidfile('controller'), 'w') as f:
+        f.write(str(controller.pid))
+    _wait_until_port_listens(port)
+    return port
+
+def stop_controller():
+    if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        return
+    kill_server_pid(_pidfile('controller'))
+
+def run_ws():
+    if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        return
+    stop_ws()
+    port = internal_port_from_config("Websocket")
+    logf = open(_logfilename('ws'), 'a')
+    ws = subprocess.Popen(["ws"],
+        stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+    with open(_pidfile('ws'), 'w') as f:
+        f.write(str(ws.pid))
+    _wait_until_port_listens(port)
+    return port
+
+def stop_ws():
+    if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        return
+    kill_server_pid(_pidfile('ws'))
+
 def _start_keep(n, keep_args):
     keep0 = tempfile.mkdtemp()
     port = find_available_port()
@@ -368,12 +452,13 @@ def _start_keep(n, keep_args):
                 "-listen=:{}".format(port),
                 "-pid="+_pidfile('keep{}'.format(n))]
 
-    for arg, val in keep_args.iteritems():
+    for arg, val in keep_args.items():
         keep_cmd.append("{}={}".format(arg, val))
 
-    logf = open(_fifo2stderr('keep{}'.format(n)), 'w')
-    kp0 = subprocess.Popen(
-        keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+    with open(_logfilename('keep{}'.format(n)), 'a') as logf:
+        with open('/dev/null') as _stdin:
+            kp0 = subprocess.Popen(
+                keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
 
     with open(_pidfile('keep{}'.format(n)), 'w') as f:
         f.write(str(kp0.pid))
@@ -424,14 +509,16 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
             'keep_disk': {'keep_service_uuid': svc['uuid'] }
         }).execute()
 
-    # If keepproxy is running, send SIGHUP to make it discover the new
-    # keepstore services.
-    proxypidfile = _pidfile('keepproxy')
-    if os.path.exists(proxypidfile):
-        try:
-            os.kill(int(open(proxypidfile).read()), signal.SIGHUP)
-        except OSError:
-            os.remove(proxypidfile)
+    # If keepproxy and/or keep-web is running, send SIGHUP to make
+    # them discover the new keepstore services.
+    for svc in ('keepproxy', 'keep-web'):
+        pidfile = _pidfile('keepproxy')
+        if os.path.exists(pidfile):
+            try:
+                with open(pidfile) as pid:
+                    os.kill(int(pid.read()), signal.SIGHUP)
+            except OSError:
+                os.remove(pidfile)
 
 def _stop_keep(n):
     kill_server_pid(_pidfile('keep{}'.format(n)))
@@ -448,19 +535,22 @@ def stop_keep(num_servers=2):
 
 def run_keep_proxy():
     if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(internal_port_from_config('Keepproxy'))
         return
     stop_keep_proxy()
 
-    port = find_available_port()
+    port = internal_port_from_config("Keepproxy")
     env = os.environ.copy()
     env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
-    logf = open(_fifo2stderr('keepproxy'), 'w')
+    logf = open(_logfilename('keepproxy'), 'a')
     kp = subprocess.Popen(
-        ['keepproxy',
-         '-pid='+_pidfile('keepproxy'),
-         '-listen=:{}'.format(port)],
-        env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+        ['keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+
+    with open(_pidfile('keepproxy'), 'w') as f:
+        f.write(str(kp.pid))
+    _wait_until_port_listens(port)
 
+    print("Using API %s token %s" % (os.environ['ARVADOS_API_HOST'], auth_token('admin')), file=sys.stdout)
     api = arvados.api(
         version='v1',
         host=os.environ['ARVADOS_API_HOST'],
@@ -476,7 +566,6 @@ def run_keep_proxy():
         'service_ssl_flag': False,
     }}).execute()
     os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
-    _setport('keepproxy', port)
     _wait_until_port_listens(port)
 
 def stop_keep_proxy():
@@ -489,19 +578,14 @@ def run_arv_git_httpd():
         return
     stop_arv_git_httpd()
 
-    gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
-    gitport = find_available_port()
+    gitport = internal_port_from_config("GitHTTP")
     env = os.environ.copy()
     env.pop('ARVADOS_API_TOKEN', None)
-    logf = open(_fifo2stderr('arv-git-httpd'), 'w')
-    agh = subprocess.Popen(
-        ['arv-git-httpd',
-         '-repo-root='+gitdir+'/test',
-         '-address=:'+str(gitport)],
+    logf = open(_logfilename('arv-git-httpd'), 'a')
+    agh = subprocess.Popen(['arv-git-httpd'],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('arv-git-httpd'), 'w') as f:
         f.write(str(agh.pid))
-    _setport('arv-git-httpd', gitport)
     _wait_until_port_listens(gitport)
 
 def stop_arv_git_httpd():
@@ -514,19 +598,14 @@ def run_keep_web():
         return
     stop_keep_web()
 
-    keepwebport = find_available_port()
+    keepwebport = internal_port_from_config("WebDAV")
     env = os.environ.copy()
-    env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
-    logf = open(_fifo2stderr('keep-web'), 'w')
+    logf = open(_logfilename('keep-web'), 'a')
     keepweb = subprocess.Popen(
-        ['keep-web',
-         '-allow-anonymous',
-         '-attachment-only-host=download:'+str(keepwebport),
-         '-listen=:'+str(keepwebport)],
+        ['keep-web'],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('keep-web'), 'w') as f:
         f.write(str(keepweb.pid))
-    _setport('keep-web', keepwebport)
     _wait_until_port_listens(keepwebport)
 
 def stop_keep_web():
@@ -537,17 +616,24 @@ def stop_keep_web():
 def run_nginx():
     if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
         return
+    stop_nginx()
     nginxconf = {}
-    nginxconf['KEEPWEBPORT'] = _getport('keep-web')
-    nginxconf['KEEPWEBDLSSLPORT'] = find_available_port()
-    nginxconf['KEEPWEBSSLPORT'] = find_available_port()
-    nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
-    nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
-    nginxconf['GITPORT'] = _getport('arv-git-httpd')
-    nginxconf['GITSSLPORT'] = find_available_port()
+    nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
+    nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")
+    nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV")
+    nginxconf['KEEPWEBDLSSLPORT'] = external_port_from_config("WebDAVDownload")
+    nginxconf['KEEPWEBSSLPORT'] = external_port_from_config("WebDAV")
+    nginxconf['KEEPPROXYPORT'] = internal_port_from_config("Keepproxy")
+    nginxconf['KEEPPROXYSSLPORT'] = external_port_from_config("Keepproxy")
+    nginxconf['GITPORT'] = internal_port_from_config("GitHTTP")
+    nginxconf['GITSSLPORT'] = external_port_from_config("GitHTTP")
+    nginxconf['WSPORT'] = internal_port_from_config("Websocket")
+    nginxconf['WSSPORT'] = external_port_from_config("Websocket")
     nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
     nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
-    nginxconf['ACCESSLOG'] = _fifo2stderr('nginx_access_log')
+    nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
+    nginxconf['ERRORLOG'] = _logfilename('nginx_error')
+    nginxconf['TMPDIR'] = TEST_TMPDIR
 
     conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
     conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
@@ -566,10 +652,111 @@ def run_nginx():
          '-g', 'pid '+_pidfile('nginx')+';',
          '-c', conffile],
         env=env, stdin=open('/dev/null'), stdout=sys.stderr)
-    _setport('keep-web-dl-ssl', nginxconf['KEEPWEBDLSSLPORT'])
-    _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
-    _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
-    _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
+
+def setup_config():
+    rails_api_port = find_available_port()
+    controller_port = find_available_port()
+    controller_external_port = find_available_port()
+    websocket_port = find_available_port()
+    websocket_external_port = find_available_port()
+    git_httpd_port = find_available_port()
+    git_httpd_external_port = find_available_port()
+    keepproxy_port = find_available_port()
+    keepproxy_external_port = find_available_port()
+    keep_web_port = find_available_port()
+    keep_web_external_port = find_available_port()
+    keep_web_dl_port = find_available_port()
+    keep_web_dl_external_port = find_available_port()
+
+    dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
+
+    print("Getting config from %s" % dbconf, file=sys.stderr)
+
+    pgconnection = yaml.safe_load(open(dbconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
+
+    localhost = "127.0.0.1"
+    services = {
+        "RailsAPI": {
+            "InternalURLs": {
+                "https://%s:%s"%(localhost, rails_api_port): {}
+            }
+        },
+        "Controller": {
+            "ExternalURL": "https://%s:%s" % (localhost, controller_external_port),
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, controller_port): {}
+            }
+        },
+        "Websocket": {
+            "ExternalURL": "wss://%s:%s/websocket" % (localhost, websocket_external_port),
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, websocket_port): {}
+            }
+        },
+        "GitHTTP": {
+            "ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port),
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, git_httpd_port): {}
+            }
+        },
+        "Keepproxy": {
+            "ExternalURL": "https://%s:%s" % (localhost, keepproxy_external_port),
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, keepproxy_port): {}
+            }
+        },
+        "WebDAV": {
+            "ExternalURL": "https://%s:%s" % (localhost, keep_web_external_port),
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, keep_web_port): {}
+            }
+        },
+        "WebDAVDownload": {
+            "ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port),
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, keep_web_dl_port): {}
+            }
+        }
+    }
+
+    config = {
+        "Clusters": {
+            "zzzzz": {
+                "EnableBetaController14287": ('14287' in os.environ.get('ARVADOS_EXPERIMENTAL', '')),
+                "ManagementToken": "e687950a23c3a9bceec28c6223a06c79",
+                "API": {
+                    "RequestTimeout": "30s"
+                },
+                "SystemLogs": {
+                    "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug')
+                },
+                "PostgreSQL": {
+                    "Connection": pgconnection,
+                },
+                "TLS": {
+                    "Insecure": True
+                },
+                "Services": services,
+                "Users": {
+                    "AnonymousUserToken": auth_token('anonymous')
+                },
+                "Collections": {
+                    "TrustAllContent": True
+                },
+                "Git": {
+                    "Repositories": "%s/test" % os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
+                }
+            }
+        }
+    }
+
+    conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
+    with open(conf, 'w') as f:
+        yaml.safe_dump(config, f)
+
+    ex = "export ARVADOS_CONFIG="+conf
+    print(ex)
+
 
 def stop_nginx():
     if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
@@ -579,36 +766,6 @@ def stop_nginx():
 def _pidfile(program):
     return os.path.join(TEST_TMPDIR, program + '.pid')
 
-def _portfile(program):
-    return os.path.join(TEST_TMPDIR, program + '.port')
-
-def _setport(program, port):
-    with open(_portfile(program), 'w') as f:
-        f.write(str(port))
-
-# Returns 9 if program is not up.
-def _getport(program):
-    try:
-        return int(open(_portfile(program)).read())
-    except IOError:
-        return 9
-
-def _apiconfig(key):
-    if _cached_config:
-        return _cached_config[key]
-    def _load(f, required=True):
-        fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
-        if not required and not os.path.exists(fullpath):
-            return {}
-        return yaml.load(fullpath)
-    cdefault = _load('application.default.yml')
-    csite = _load('application.yml', required=False)
-    _cached_config = {}
-    for section in [cdefault.get('common',{}), cdefault.get('test',{}),
-                    csite.get('common',{}), csite.get('test',{})]:
-        _cached_config.update(section)
-    return _cached_config[key]
-
 def fixture(fix):
     '''load a fixture yaml file'''
     with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
@@ -619,7 +776,7 @@ def fixture(fix):
           yaml_file = yaml_file[0:trim_index]
         except ValueError:
           pass
-        return yaml.load(yaml_file)
+        return yaml.safe_load(yaml_file)
 
 def auth_token(token_name):
     return fixture("api_client_authorizations")[token_name]["api_token"]
@@ -647,13 +804,14 @@ class TestCaseWithServers(unittest.TestCase):
     original environment.
     """
     MAIN_SERVER = None
+    WS_SERVER = None
     KEEP_SERVER = None
     KEEP_PROXY_SERVER = None
     KEEP_WEB_SERVER = None
 
     @staticmethod
     def _restore_dict(src, dest):
-        for key in dest.keys():
+        for key in list(dest.keys()):
             if key not in src:
                 del dest[key]
         dest.update(src)
@@ -667,6 +825,7 @@ class TestCaseWithServers(unittest.TestCase):
         os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
         for server_kwargs, start_func, stop_func in (
                 (cls.MAIN_SERVER, run, reset),
+                (cls.WS_SERVER, run_ws, stop_ws),
                 (cls.KEEP_SERVER, run_keep, stop_keep),
                 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
                 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
@@ -693,11 +852,13 @@ class TestCaseWithServers(unittest.TestCase):
 if __name__ == "__main__":
     actions = [
         'start', 'stop',
+        'start_ws', 'stop_ws',
+        'start_controller', 'stop_controller',
         'start_keep', 'stop_keep',
         'start_keep_proxy', 'stop_keep_proxy',
         'start_keep-web', 'stop_keep-web',
         'start_arv-git-httpd', 'stop_arv-git-httpd',
-        'start_nginx', 'stop_nginx',
+        'start_nginx', 'stop_nginx', 'setup_config',
     ]
     parser = argparse.ArgumentParser()
     parser.add_argument('action', type=str, help="one of {}".format(actions))
@@ -725,6 +886,14 @@ if __name__ == "__main__":
             print(host)
     elif args.action == 'stop':
         stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
+    elif args.action == 'start_ws':
+        run_ws()
+    elif args.action == 'stop_ws':
+        stop_ws()
+    elif args.action == 'start_controller':
+        run_controller()
+    elif args.action == 'stop_controller':
+        stop_controller()
     elif args.action == 'start_keep':
         run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
     elif args.action == 'stop_keep':
@@ -743,7 +912,10 @@ if __name__ == "__main__":
         stop_keep_web()
     elif args.action == 'start_nginx':
         run_nginx()
+        print("export ARVADOS_API_HOST=0.0.0.0:{}".format(external_port_from_config('Controller')))
     elif args.action == 'stop_nginx':
         stop_nginx()
+    elif args.action == 'setup_config':
+        setup_config()
     else:
         raise Exception("action recognized but not implemented!?")