X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/86860e7d65589bc9d93df5b514baee3fc5a5103a..970907f28866a09a9fe95da48dffa6cd34ab4dca:/sdk/python/tests/run_test_server.py

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index c54dad0c5f..1c5162b97d 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -115,6 +115,33 @@ def find_available_port():
     sock.close()
     return port
 
+def _wait_until_port_listens(port, timeout=10):
+    """Wait for a process to start listening on the given port.
+
+    If nothing listens on the port within the specified timeout (given
+    in seconds), print a warning on stderr before returning.
+    """
+    try:
+        subprocess.check_output(['which', 'lsof'])
+    except subprocess.CalledProcessError:
+        print("WARNING: No `lsof` -- cannot wait for port to listen. "+
+              "Sleeping 0.5 and hoping for the best.")
+        time.sleep(0.5)
+        return
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        try:
+            subprocess.check_output(
+                ['lsof', '-t', '-i', 'tcp:'+str(port)])
+        except subprocess.CalledProcessError:
+            time.sleep(0.1)
+            continue
+        return
+    print(
+        "WARNING: Nothing is listening on port {} (waited {} seconds).".
+        format(port, timeout),
+        file=sys.stderr)
+
 def run(leave_running_atexit=False):
     """Ensure an API server is running, and ARVADOS_API_* env vars have
     admin credentials for it.
@@ -165,6 +192,15 @@ def run(leave_running_atexit=False):
     # died, or we have lost our credentials, or something else is
     # preventing us from calling reset(). Start a new one.
 
+    if not os.path.exists('tmp'):
+        os.makedirs('tmp')
+
+    if not os.path.exists('tmp/api'):
+        os.makedirs('tmp/api')
+
+    if not os.path.exists('tmp/logs'):
+        os.makedirs('tmp/logs')
+
     if not os.path.exists('tmp/self-signed.pem'):
         # We assume here that either passenger reports its listening
         # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
@@ -215,8 +251,10 @@ def run(leave_running_atexit=False):
     my_api_host = match.group(1)
     os.environ['ARVADOS_API_HOST'] = my_api_host
 
-    # Make sure the server has written its pid file before continuing
+    # Make sure the server has written its pid file and started
+    # listening on its TCP port
     find_server_pid(pid_file)
+    _wait_until_port_listens(port)
 
     reset()
     os.chdir(restore_cwd)
@@ -280,6 +318,8 @@ def _start_keep(n, keep_args):
     with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
         f.write(keep0)
 
+    _wait_until_port_listens(port)
+
     return port
 
 def run_keep(blob_signing_key=None, enforce_permissions=False):
@@ -360,6 +400,7 @@ def run_keep_proxy():
     }}).execute()
     os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
     _setport('keepproxy', port)
+    _wait_until_port_listens(port)
 
 def stop_keep_proxy():
     if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
@@ -374,7 +415,7 @@ def run_arv_git_httpd():
     gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
     gitport = find_available_port()
     env = os.environ.copy()
-    del env['ARVADOS_API_TOKEN']
+    env.pop('ARVADOS_API_TOKEN', None)
     agh = subprocess.Popen(
         ['arv-git-httpd',
          '-repo-root='+gitdir+'/test',
@@ -383,6 +424,7 @@ def run_arv_git_httpd():
     with open(_pidfile('arv-git-httpd'), 'w') as f:
         f.write(str(agh.pid))
     _setport('arv-git-httpd', gitport)
+    _wait_until_port_listens(gitport)
 
 def stop_arv_git_httpd():
     if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
@@ -444,10 +486,13 @@ def _getport(program):
 def _apiconfig(key):
     if _cached_config:
         return _cached_config[key]
-    def _load(f):
-        return yaml.load(os.path.join(SERVICES_SRC_DIR, 'api', 'config', f))
+    def _load(f, required=True):
+        fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
+        if not required and not os.path.exists(fullpath):
+            return {}
+        return yaml.load(fullpath)
     cdefault = _load('application.default.yml')
-    csite = _load('application.yml')
+    csite = _load('application.yml', required=False)
     _cached_config = {}
     for section in [cdefault.get('common',{}), cdefault.get('test',{}),
                     csite.get('common',{}), csite.get('test',{})]: