Merge branch '9551-python-keep-services-env'

[arvados.git] / sdk / python / tests / run_test_server.py

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index adb8652edc952ab80eb67c0b8a8708da3fb954bd..e72f67dce49049f37c9b2e68794eb62cc780297c 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -4,6 +4,7 @@ from __future__ import print_function
 import argparse
 import atexit
 import errno
+import glob
 import httplib2
 import os
 import pipes
@@ -12,8 +13,8 @@ import re
 import shutil
 import signal
 import socket
-import subprocess
 import string
+import subprocess
 import sys
 import tempfile
 import time
@@ -98,7 +99,8 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
             # Pidfile exists, but we can't parse it. Perhaps the
             # server has created the file but hasn't written its PID
             # yet?
-            print("Parse error reading pidfile {}: {}".format(pidfile, error))
+            print("Parse error reading pidfile {}: {}".format(pidfile, error),
+                  file=sys.stderr)
             time.sleep(0.1)
             now = time.time()
 
@@ -113,7 +115,8 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
         try:
             if now >= startTERM:
                 os.kill(server_pid, signal.SIGTERM)
-                print("Sent SIGTERM to {} ({})".format(server_pid, pidfile))
+                print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
+                      file=sys.stderr)
         except OSError as error:
             if error.errno == errno.ESRCH:
                 # Thrown by os.getpgid() or os.kill() if the process
@@ -124,7 +127,8 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
         now = time.time()
 
     print("Server PID {} ({}) did not exit, giving up after {}s".
-          format(server_pid, pidfile, wait))
+          format(server_pid, pidfile, wait),
+          file=sys.stderr)
 
 def find_available_port():
     """Return an IPv4 port number that is not in use right now.
@@ -155,7 +159,8 @@ def _wait_until_port_listens(port, timeout=10):
         subprocess.check_output(['which', 'lsof'])
     except subprocess.CalledProcessError:
         print("WARNING: No `lsof` -- cannot wait for port to listen. "+
-              "Sleeping 0.5 and hoping for the best.")
+              "Sleeping 0.5 and hoping for the best.",
+              file=sys.stderr)
         time.sleep(0.5)
         return
     deadline = time.time() + timeout
@@ -188,7 +193,7 @@ def _fifo2stderr(label):
             raise
     os.mkfifo(fifo, 0700)
     subprocess.Popen(
-        ['sed', '-e', 's/^/['+label+'] /', fifo],
+        ['stdbuf', '-i0', '-oL', '-eL', 'sed', '-e', 's/^/['+label+'] /', fifo],
         stdout=sys.stderr)
     return fifo
 
@@ -209,8 +214,14 @@ def run(leave_running_atexit=False):
     """
     global my_api_host
 
-    # Delete cached discovery document.
-    shutil.rmtree(arvados.http_cache('discovery'))
+    # Delete cached discovery documents.
+    #
+    # This will clear cached docs that belong to other processes (like
+    # concurrent test suites) even if they're still running. They should
+    # be able to tolerate that.
+    for fn in glob.glob(os.path.join(arvados.http_cache('discovery'),
+                                     '*,arvados,v1,rest,*')):
+        os.unlink(fn)
 
     pid_file = _pidfile('api')
     pid_file_ok = find_server_pid(pid_file, 0)
@@ -418,7 +429,10 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
     # keepstore services.
     proxypidfile = _pidfile('keepproxy')
     if os.path.exists(proxypidfile):
-        os.kill(int(open(proxypidfile).read()), signal.SIGHUP)
+        try:
+            os.kill(int(open(proxypidfile).read()), signal.SIGHUP)
+        except OSError:
+            os.remove(proxypidfile)
 
 def _stop_keep(n):
     kill_server_pid(_pidfile('keep{}'.format(n)))
@@ -462,7 +476,7 @@ def run_keep_proxy():
         'service_type': 'proxy',
         'service_ssl_flag': False,
     }}).execute()
-    os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
+    os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
     _setport('keepproxy', port)
     _wait_until_port_listens(port)
 
@@ -650,7 +664,7 @@ class TestCaseWithServers(unittest.TestCase):
         cls._orig_environ = os.environ.copy()
         cls._orig_config = arvados.config.settings().copy()
         cls._cleanup_funcs = []
-        os.environ.pop('ARVADOS_KEEP_PROXY', None)
+        os.environ.pop('ARVADOS_KEEP_SERVICES', None)
         os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
         for server_kwargs, start_func, stop_func in (
                 (cls.MAIN_SERVER, run, reset),
@@ -695,7 +709,9 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     if args.action not in actions:
-        print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr)
+        print("Unrecognized action '{}'. Actions are: {}.".
+              format(args.action, actions),
+              file=sys.stderr)
         sys.exit(1)
     if args.action == 'start':
         stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))