Merge branch '8784-dir-listings'

[arvados.git] / sdk / python / tests / run_test_server.py

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index b969b12a7abc6374ae0993d0d95c2f5cf85f784b..57efb97c4851ef3b5b678b906bddefb2b05abc83 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -1,6 +1,11 @@
-#!/usr/bin/env python
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
 
 from __future__ import print_function
+from __future__ import division
+from builtins import str
+from builtins import range
 import argparse
 import atexit
 import errno
@@ -34,9 +39,19 @@ import arvados.config
 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
 if 'GOPATH' in os.environ:
+    # Add all GOPATH bin dirs to PATH -- but insert them after the
+    # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
+    # command, not the golang.org/x/tools/cmd/bundle command.
     gopaths = os.environ['GOPATH'].split(':')
-    gobins = [os.path.join(path, 'bin') for path in gopaths]
-    os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
+    addbins = [os.path.join(path, 'bin') for path in gopaths]
+    newbins = []
+    for path in os.environ['PATH'].split(':'):
+        newbins.append(path)
+        if os.path.exists(os.path.join(path, 'bundle')):
+            newbins += addbins
+            addbins = []
+    newbins += addbins
+    os.environ['PATH'] = ':'.join(newbins)
 
 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
 if not os.path.exists(TEST_TMPDIR):
@@ -86,7 +101,7 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
         # Use up to half of the +wait+ period waiting for "passenger
         # stop" to work. If the process hasn't exited by then, start
         # sending TERM signals.
-        startTERM += wait/2
+        startTERM += wait//2
 
     server_pid = None
     while now <= deadline and server_pid is None:
@@ -109,6 +124,7 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
         try:
             exited, _ = os.waitpid(server_pid, os.WNOHANG)
             if exited > 0:
+                _remove_pidfile(pidfile)
                 return
         except OSError:
             # already exited, or isn't our child process
@@ -122,6 +138,7 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
             if error.errno == errno.ESRCH:
                 # Thrown by os.getpgid() or os.kill() if the process
                 # does not exist, i.e., our work here is done.
+                _remove_pidfile(pidfile)
                 return
             raise
         time.sleep(0.1)
@@ -131,6 +148,13 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
           format(server_pid, pidfile, wait),
           file=sys.stderr)
 
+def _remove_pidfile(pidfile):
+    try:
+        os.unlink(pidfile)
+    except:
+        if os.path.lexists(pidfile):
+            raise
+
 def find_available_port():
     """Return an IPv4 port number that is not in use right now.
 
@@ -192,7 +216,7 @@ def _fifo2stderr(label):
     except OSError as error:
         if error.errno != errno.ENOENT:
             raise
-    os.mkfifo(fifo, 0700)
+    os.mkfifo(fifo, 0o700)
     subprocess.Popen(
         ['stdbuf', '-i0', '-oL', '-eL', 'sed', '-e', 's/^/['+label+'] /', fifo],
         stdout=sys.stderr)
@@ -220,8 +244,9 @@ def run(leave_running_atexit=False):
     # This will clear cached docs that belong to other processes (like
     # concurrent test suites) even if they're still running. They should
     # be able to tolerate that.
-    for fn in glob.glob(os.path.join(arvados.http_cache('discovery'),
-                                     '*,arvados,v1,rest,*')):
+    for fn in glob.glob(os.path.join(
+            str(arvados.http_cache('discovery')),
+            '*,arvados,v1,rest,*')):
         os.unlink(fn)
 
     pid_file = _pidfile('api')
@@ -294,10 +319,7 @@ def run(leave_running_atexit=False):
     env = os.environ.copy()
     env['RAILS_ENV'] = 'test'
     env['ARVADOS_TEST_WSS_PORT'] = str(wss_port)
-    if env.get('ARVADOS_TEST_EXPERIMENTAL_WS'):
-        env.pop('ARVADOS_WEBSOCKETS', None)
-    else:
-        env['ARVADOS_WEBSOCKETS'] = 'yes'
+    env.pop('ARVADOS_WEBSOCKETS', None)
     env.pop('ARVADOS_TEST_API_HOST', None)
     env.pop('ARVADOS_API_HOST', None)
     env.pop('ARVADOS_API_HOST_INSECURE', None)
@@ -419,7 +441,7 @@ def _start_keep(n, keep_args):
                 "-listen=:{}".format(port),
                 "-pid="+_pidfile('keep{}'.format(n))]
 
-    for arg, val in keep_args.iteritems():
+    for arg, val in keep_args.items():
         keep_cmd.append("{}={}".format(arg, val))
 
     logf = open(_fifo2stderr('keep{}'.format(n)), 'w')
@@ -475,14 +497,15 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
             'keep_disk': {'keep_service_uuid': svc['uuid'] }
         }).execute()
 
-    # If keepproxy is running, send SIGHUP to make it discover the new
-    # keepstore services.
-    proxypidfile = _pidfile('keepproxy')
-    if os.path.exists(proxypidfile):
-        try:
-            os.kill(int(open(proxypidfile).read()), signal.SIGHUP)
-        except OSError:
-            os.remove(proxypidfile)
+    # If keepproxy and/or keep-web is running, send SIGHUP to make
+    # them discover the new keepstore services.
+    for svc in ('keepproxy', 'keep-web'):
+        pidfile = _pidfile('keepproxy')
+        if os.path.exists(pidfile):
+            try:
+                os.kill(int(open(pidfile).read()), signal.SIGHUP)
+            except OSError:
+                os.remove(pidfile)
 
 def _stop_keep(n):
     kill_server_pid(_pidfile('keep{}'.format(n)))
@@ -716,7 +739,7 @@ class TestCaseWithServers(unittest.TestCase):
 
     @staticmethod
     def _restore_dict(src, dest):
-        for key in dest.keys():
+        for key in list(dest.keys()):
             if key not in src:
                 del dest[key]
         dest.update(src)