Merge branch 'master' into 14965-arv-mount-py-three

[arvados.git] / sdk / python / tests / run_test_server.py

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 6687ca491a769140aa8c803a5fd2b1a6ce3b1850..3063ea6473c8f1cd5d496d8a7c88ba3cf1b00a03 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -181,22 +181,18 @@ def _wait_until_port_listens(port, timeout=10, warn=True):
     in seconds), print a warning on stderr before returning.
     """
     try:
-        subprocess.check_output(['which', 'lsof'])
+        subprocess.check_output(['which', 'netstat'])
     except subprocess.CalledProcessError:
-        print("WARNING: No `lsof` -- cannot wait for port to listen. "+
+        print("WARNING: No `netstat` -- cannot wait for port to listen. "+
               "Sleeping 0.5 and hoping for the best.",
               file=sys.stderr)
         time.sleep(0.5)
         return
     deadline = time.time() + timeout
     while time.time() < deadline:
-        try:
-            subprocess.check_output(
-                ['lsof', '-t', '-i', 'tcp:'+str(port)])
-        except subprocess.CalledProcessError:
-            time.sleep(0.1)
-            continue
-        return True
+        if re.search(r'\ntcp.*:'+str(port)+' .* LISTEN *\n', subprocess.check_output(['netstat', '-Wln']).decode()):
+            return True
+        time.sleep(0.1)
     if warn:
         print(
             "WARNING: Nothing is listening on port {} (waited {} seconds).".
@@ -413,29 +409,38 @@ def run_controller():
         f.write("""
 Clusters:
   zzzzz:
+    EnableBetaController14287: {beta14287}
+    ManagementToken: e687950a23c3a9bceec28c6223a06c79
+    API:
+      RequestTimeout: 30s
+    Logging:
+        Level: "{loglevel}"
     HTTPRequestTimeout: 30s
     PostgreSQL:
       ConnectionPool: 32
       Connection:
-        host: {}
-        dbname: {}
-        user: {}
-        password: {}
-    NodeProfiles:
-      "*":
-        "arvados-controller":
-          Listen: ":{}"
-        "arvados-api-server":
-          Listen: ":{}"
-          TLS: true
-          Insecure: true
+        host: {dbhost}
+        dbname: {dbname}
+        user: {dbuser}
+        password: {dbpass}
+    TLS:
+      Insecure: true
+    Services:
+      Controller:
+        InternalURLs:
+          "http://localhost:{controllerport}": {{}}
+      RailsAPI:
+        InternalURLs:
+          "https://localhost:{railsport}": {{}}
         """.format(
-            _dbconfig('host'),
-            _dbconfig('database'),
-            _dbconfig('username'),
-            _dbconfig('password'),
-            port,
-            rails_api_port,
+            beta14287=('true' if '14287' in os.environ.get('ARVADOS_EXPERIMENTAL', '') else 'false'),
+            loglevel=('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
+            dbhost=_dbconfig('host'),
+            dbname=_dbconfig('dbname'),
+            dbuser=_dbconfig('user'),
+            dbpass=_dbconfig('password'),
+            controllerport=port,
+            railsport=rails_api_port,
         ))
     logf = open(_logfilename('controller'), 'a')
     controller = subprocess.Popen(
@@ -475,8 +480,8 @@ Postgres:
                    port,
                    ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
                    _dbconfig('host'),
-                   _dbconfig('database'),
-                   _dbconfig('username'),
+                   _dbconfig('dbname'),
+                   _dbconfig('user'),
                    _dbconfig('password')))
     logf = open(_logfilename('ws'), 'a')
     ws = subprocess.Popen(
@@ -504,9 +509,10 @@ def _start_keep(n, keep_args):
     for arg, val in keep_args.items():
         keep_cmd.append("{}={}".format(arg, val))
 
-    logf = open(_logfilename('keep{}'.format(n)), 'a')
-    kp0 = subprocess.Popen(
-        keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+    with open(_logfilename('keep{}'.format(n)), 'a') as logf:
+        with open('/dev/null') as _stdin:
+            kp0 = subprocess.Popen(
+                keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
 
     with open(_pidfile('keep{}'.format(n)), 'w') as f:
         f.write(str(kp0.pid))
@@ -563,7 +569,8 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
         pidfile = _pidfile('keepproxy')
         if os.path.exists(pidfile):
             try:
-                os.kill(int(open(pidfile).read()), signal.SIGHUP)
+                with open(pidfile) as pid:
+                    os.kill(int(pid.read()), signal.SIGHUP)
             except OSError:
                 os.remove(pidfile)
 
@@ -632,6 +639,7 @@ def run_arv_git_httpd():
     agh = subprocess.Popen(
         ['arv-git-httpd',
          '-repo-root='+gitdir+'/test',
+         '-management-token=e687950a23c3a9bceec28c6223a06c79',
          '-address=:'+str(gitport)],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('arv-git-httpd'), 'w') as f:
@@ -657,6 +665,7 @@ def run_keep_web():
         ['keep-web',
          '-allow-anonymous',
          '-attachment-only-host=download',
+         '-management-token=e687950a23c3a9bceec28c6223a06c79',
          '-listen=:'+str(keepwebport)],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('keep-web'), 'w') as f:
@@ -732,16 +741,22 @@ def _setport(program, port):
 # Returns 9 if program is not up.
 def _getport(program):
     try:
-        return int(open(_portfile(program)).read())
+        with open(_portfile(program)) as prog:
+            return int(prog.read())
     except IOError:
         return 9
 
 def _dbconfig(key):
     global _cached_db_config
     if not _cached_db_config:
-        _cached_db_config = yaml.safe_load(open(os.path.join(
-            SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))
-    return _cached_db_config['test'][key]
+        if "ARVADOS_CONFIG" in os.environ:
+            _cached_db_config = list(yaml.safe_load(open(os.environ["ARVADOS_CONFIG"]))["Clusters"].values())[0]["PostgreSQL"]["Connection"]
+        else:
+            _cached_db_config = yaml.safe_load(open(os.path.join(
+                SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))["test"]
+            _cached_db_config["dbname"] = _cached_db_config["database"]
+            _cached_db_config["user"] = _cached_db_config["username"]
+    return _cached_db_config[key]
 
 def _apiconfig(key):
     global _cached_config