3198: Support for mkdir, rmdir, unlink, rename. Improve exception catching.

[arvados.git] / sdk / python / tests / run_test_server.py

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 4f7cbb1a6c4b84bae0b63310b2fcb1567766249d..b9502f0f8e5bb5a6292da85bb69ef1f783163cab 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -9,7 +9,9 @@ import random
 import re
 import shutil
 import signal
+import socket
 import subprocess
+import string
 import sys
 import tempfile
 import time
@@ -23,7 +25,7 @@ if __name__ == '__main__' and os.path.exists(
     # Add the Python SDK source to the library path.
     sys.path.insert(1, os.path.dirname(MY_DIRNAME))
 
-import arvados.api
+import arvados
 import arvados.config
 
 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
@@ -92,6 +94,25 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
     except OSError:
         pass
 
+def find_available_port():
+    """Return an IPv4 port number that is not in use right now.
+
+    We assume whoever needs to use the returned port is able to reuse
+    a recently used port without waiting for TIME_WAIT (see
+    SO_REUSEADDR / SO_REUSEPORT).
+
+    Some opportunity for races here, but it's better than choosing
+    something at random and not checking at all. If all of our servers
+    (hey Passenger) knew that listening on port 0 was a thing, the OS
+    would take care of the races, and this wouldn't be needed at all.
+    """
+
+    sock = socket.socket()
+    sock.bind(('0.0.0.0', 0))
+    port = sock.getsockname()[1]
+    sock.close()
+    return port
+
 def run(leave_running_atexit=False):
     """Ensure an API server is running, and ARVADOS_API_* env vars have
     admin credentials for it.
@@ -112,20 +133,27 @@ def run(leave_running_atexit=False):
     # Delete cached discovery document.
     shutil.rmtree(arvados.http_cache('discovery'))
 
-    os.environ['ARVADOS_API_TOKEN'] = auth_token('admin')
-    os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
-
     pid_file = os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH)
     pid_file_ok = find_server_pid(pid_file, 0)
 
     existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
     if existing_api_host and pid_file_ok:
-        try:
-            os.environ['ARVADOS_API_HOST'] = existing_api_host
-            reset()
-            return
-        except:
-            pass
+        if existing_api_host == my_api_host:
+            try:
+                return reset()
+            except:
+                # Fall through to shutdown-and-start case.
+                pass
+        else:
+            # Server was provided by parent. Can't recover if it's
+            # unresettable.
+            return reset()
+
+    # Before trying to start up our own server, call stop() to avoid
+    # "Phusion Passenger Standalone is already running on PID 12345".
+    # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
+    # we know the server is ours to kill.)
+    stop(force=True)
 
     restore_cwd = os.getcwd()
     api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
@@ -147,9 +175,10 @@ def run(leave_running_atexit=False):
             '-out', 'tmp/self-signed.pem',
             '-keyout', 'tmp/self-signed.key',
             '-days', '3650',
-            '-subj', '/CN=0.0.0.0'])
+            '-subj', '/CN=0.0.0.0'],
+        stdout=sys.stderr)
 
-    port = random.randint(20000, 40000)
+    port = find_available_port()
     env = os.environ.copy()
     env['RAILS_ENV'] = 'test'
     env['ARVADOS_WEBSOCKETS'] = 'yes'
@@ -188,6 +217,9 @@ def reset():
 
     This resets the ARVADOS_TEST_API_HOST provided by a parent process
     if any, otherwise the server started by run().
+
+    It also resets ARVADOS_* environment vars to point to the test
+    server with admin credentials.
     """
     existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
     token = auth_token('admin')
@@ -197,6 +229,9 @@ def reset():
         'https://{}/database/reset'.format(existing_api_host),
         'POST',
         headers={'Authorization': 'OAuth2 {}'.format(token)})
+    os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
+    os.environ['ARVADOS_API_HOST'] = existing_api_host
+    os.environ['ARVADOS_API_TOKEN'] = token
 
 def stop(force=False):
     """Stop the API server, if one is running.
@@ -219,7 +254,7 @@ def stop(force=False):
 
 def _start_keep(n, keep_args):
     keep0 = tempfile.mkdtemp()
-    port = random.randint(20000, 40000)
+    port = find_available_port()
     keep_cmd = ["keepstore",
                 "-volumes={}".format(keep0),
                 "-listen=:{}".format(port),
@@ -249,7 +284,7 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
         keep_args['--enforce-permissions'] = 'true'
 
     api = arvados.api(
-        'v1', cache=False,
+        version='v1',
         host=os.environ['ARVADOS_API_HOST'],
         token=os.environ['ARVADOS_API_TOKEN'],
         insecure=True)
@@ -288,7 +323,7 @@ def run_keep_proxy():
     stop_keep_proxy()
 
     admin_token = auth_token('admin')
-    port = random.randint(20000,40000)
+    port = find_available_port()
     env = os.environ.copy()
     env['ARVADOS_API_TOKEN'] = admin_token
     kp = subprocess.Popen(
@@ -298,7 +333,7 @@ def run_keep_proxy():
         env=env)
 
     api = arvados.api(
-        'v1', cache=False,
+        version='v1',
         host=os.environ['ARVADOS_API_HOST'],
         token=admin_token,
         insecure=True)