14965: Cleans up test output by reducing resource warnings

[arvados.git] / sdk / python / tests / run_test_server.py

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 7b1f6059aeef07b8ff2a2d03a6d4980f9d5a835f..510ffbeca98d1ef72e875fdc9103b20c20dd1e5f 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -181,22 +181,18 @@ def _wait_until_port_listens(port, timeout=10, warn=True):
     in seconds), print a warning on stderr before returning.
     """
     try:
-        subprocess.check_output(['which', 'lsof'])
+        subprocess.check_output(['which', 'netstat'])
     except subprocess.CalledProcessError:
-        print("WARNING: No `lsof` -- cannot wait for port to listen. "+
+        print("WARNING: No `netstat` -- cannot wait for port to listen. "+
               "Sleeping 0.5 and hoping for the best.",
               file=sys.stderr)
         time.sleep(0.5)
         return
     deadline = time.time() + timeout
     while time.time() < deadline:
-        try:
-            subprocess.check_output(
-                ['lsof', '-t', '-i', 'tcp:'+str(port)])
-        except subprocess.CalledProcessError:
-            time.sleep(0.1)
-            continue
-        return True
+        if re.search(r'\ntcp.*:'+str(port)+' .* LISTEN *\n', str(subprocess.check_output(['netstat', '-an']))):
+            return True
+        time.sleep(0.1)
     if warn:
         print(
             "WARNING: Nothing is listening on port {} (waited {} seconds).".
@@ -413,29 +409,32 @@ def run_controller():
         f.write("""
 Clusters:
   zzzzz:
-    HTTPRequestTimeout: 30s
+    ManagementToken: e687950a23c3a9bceec28c6223a06c79
+    API:
+      RequestTimeout: 30s
     PostgreSQL:
       ConnectionPool: 32
       Connection:
-        host: {}
-        dbname: {}
-        user: {}
-        password: {}
-    NodeProfiles:
-      "*":
-        "arvados-controller":
-          Listen: ":{}"
-        "arvados-api-server":
-          Listen: ":{}"
-          TLS: true
-          Insecure: true
+        host: {dbhost}
+        dbname: {dbname}
+        user: {dbuser}
+        password: {dbpass}
+    TLS:
+      Insecure: true
+    Services:
+      Controller:
+        InternalURLs:
+          "http://localhost:{controllerport}": {{}}
+      RailsAPI:
+        InternalURLs:
+          "https://localhost:{railsport}": {{}}
         """.format(
-            _dbconfig('host'),
-            _dbconfig('database'),
-            _dbconfig('username'),
-            _dbconfig('password'),
-            port,
-            rails_api_port,
+            dbhost=_dbconfig('host'),
+            dbname=_dbconfig('database'),
+            dbuser=_dbconfig('username'),
+            dbpass=_dbconfig('password'),
+            controllerport=port,
+            railsport=rails_api_port,
         ))
     logf = open(_logfilename('controller'), 'a')
     controller = subprocess.Popen(
@@ -504,9 +503,10 @@ def _start_keep(n, keep_args):
     for arg, val in keep_args.items():
         keep_cmd.append("{}={}".format(arg, val))
 
-    logf = open(_logfilename('keep{}'.format(n)), 'a')
-    kp0 = subprocess.Popen(
-        keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+    with open(_logfilename('keep{}'.format(n)), 'a') as logf:
+        with open('/dev/null') as _stdin:
+            kp0 = subprocess.Popen(
+                keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
 
     with open(_pidfile('keep{}'.format(n)), 'w') as f:
         f.write(str(kp0.pid))
@@ -563,7 +563,8 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
         pidfile = _pidfile('keepproxy')
         if os.path.exists(pidfile):
             try:
-                os.kill(int(open(pidfile).read()), signal.SIGHUP)
+                with open(pidfile) as pid:
+                    os.kill(int(pid.read()), signal.SIGHUP)
             except OSError:
                 os.remove(pidfile)
 
@@ -582,6 +583,7 @@ def stop_keep(num_servers=2):
 
 def run_keep_proxy():
     if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(_getport('keepproxy'))
         return
     stop_keep_proxy()
 
@@ -631,6 +633,7 @@ def run_arv_git_httpd():
     agh = subprocess.Popen(
         ['arv-git-httpd',
          '-repo-root='+gitdir+'/test',
+         '-management-token=e687950a23c3a9bceec28c6223a06c79',
          '-address=:'+str(gitport)],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('arv-git-httpd'), 'w') as f:
@@ -656,6 +659,7 @@ def run_keep_web():
         ['keep-web',
          '-allow-anonymous',
          '-attachment-only-host=download',
+         '-management-token=e687950a23c3a9bceec28c6223a06c79',
          '-listen=:'+str(keepwebport)],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('keep-web'), 'w') as f:
@@ -731,14 +735,15 @@ def _setport(program, port):
 # Returns 9 if program is not up.
 def _getport(program):
     try:
-        return int(open(_portfile(program)).read())
+        with open(_portfile(program)) as prog:
+            return int(prog.read())
     except IOError:
         return 9
 
 def _dbconfig(key):
     global _cached_db_config
     if not _cached_db_config:
-        _cached_db_config = yaml.load(open(os.path.join(
+        _cached_db_config = yaml.safe_load(open(os.path.join(
             SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))
     return _cached_db_config['test'][key]
 
@@ -750,7 +755,7 @@ def _apiconfig(key):
         fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
         if not required and not os.path.exists(fullpath):
             return {}
-        return yaml.load(fullpath)
+        return yaml.safe_load(fullpath)
     cdefault = _load('application.default.yml')
     csite = _load('application.yml', required=False)
     _cached_config = {}
@@ -769,7 +774,7 @@ def fixture(fix):
           yaml_file = yaml_file[0:trim_index]
         except ValueError:
           pass
-        return yaml.load(yaml_file)
+        return yaml.safe_load(yaml_file)
 
 def auth_token(token_name):
     return fixture("api_client_authorizations")[token_name]["api_token"]