4717: use keep_services -> read_only flag in python sdk.

[arvados.git] / sdk / python / tests / run_test_server.py

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 19520c86bf78bb79cb49457449bf24642d5d1632..271b525df2bc8cd404feb357c1ea3d080fd037e6 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
 #!/usr/bin/env python
 

+from __future__ import print_function

 import argparse
 import atexit
 import httplib2
 import argparse
 import atexit
 import httplib2


@@ -9,6 +10,7 @@ import random
 import re
 import shutil
 import signal
 import re
 import shutil
 import signal

+import socket

 import subprocess
 import string
 import sys
 import subprocess
 import string
 import sys


@@ -24,7 +26,7 @@ if __name__ == '__main__' and os.path.exists(
     # Add the Python SDK source to the library path.
     sys.path.insert(1, os.path.dirname(MY_DIRNAME))
 
     # Add the Python SDK source to the library path.
     sys.path.insert(1, os.path.dirname(MY_DIRNAME))
 

-import arvados.api
+import arvados

 import arvados.config
 
 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
 import arvados.config
 
 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))


@@ -40,6 +42,7 @@ if not os.path.exists(TEST_TMPDIR):
     os.mkdir(TEST_TMPDIR)
 
 my_api_host = None
     os.mkdir(TEST_TMPDIR)
 
 my_api_host = None

+_cached_config = {}

 
 def find_server_pid(PID_PATH, wait=10):
     now = time.time()
 
 def find_server_pid(PID_PATH, wait=10):
     now = time.time()


@@ -94,28 +97,22 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
         pass
 
 def find_available_port():
         pass
 
 def find_available_port():

-    """Return a port number that is not in use right now.
+    """Return an IPv4 port number that is not in use right now.
+
+    We assume whoever needs to use the returned port is able to reuse
+    a recently used port without waiting for TIME_WAIT (see
+    SO_REUSEADDR / SO_REUSEPORT).

 
     Some opportunity for races here, but it's better than choosing
     something at random and not checking at all. If all of our servers
     (hey Passenger) knew that listening on port 0 was a thing, the OS
     would take care of the races, and this wouldn't be needed at all.
     """
 
     Some opportunity for races here, but it's better than choosing
     something at random and not checking at all. If all of our servers
     (hey Passenger) knew that listening on port 0 was a thing, the OS
     would take care of the races, and this wouldn't be needed at all.
     """

-    port = None
-    while port is None:
-        port = random.randint(20000, 40000)
-        port_hex = ':%04x ' % port
-        try:
-            with open('/proc/net/tcp', 'r') as f:
-                for line in f:
-                    if 0 <= string.find(line, port_hex):
-                        port = None
-                        break
-        except OSError:
-            # This isn't going so well. Just use the random port.
-            pass
-        except IOError:
-            pass
+
+    sock = socket.socket()
+    sock.bind(('0.0.0.0', 0))
+    port = sock.getsockname()[1]
+    sock.close()

     return port
 
 def run(leave_running_atexit=False):
     return port
 
 def run(leave_running_atexit=False):


@@ -180,7 +177,15 @@ def run(leave_running_atexit=False):
             '-out', 'tmp/self-signed.pem',
             '-keyout', 'tmp/self-signed.key',
             '-days', '3650',
             '-out', 'tmp/self-signed.pem',
             '-keyout', 'tmp/self-signed.key',
             '-days', '3650',

-            '-subj', '/CN=0.0.0.0'])
+            '-subj', '/CN=0.0.0.0'],
+        stdout=sys.stderr)
+
+    # Install the git repository fixtures.
+    gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
+    gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
+    if not os.path.isdir(gitdir):
+        os.makedirs(gitdir)
+    subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])

 
     port = find_available_port()
     env = os.environ.copy()
 
     port = find_available_port()
     env = os.environ.copy()


@@ -260,15 +265,16 @@ def _start_keep(n, keep_args):
     keep0 = tempfile.mkdtemp()
     port = find_available_port()
     keep_cmd = ["keepstore",
     keep0 = tempfile.mkdtemp()
     port = find_available_port()
     keep_cmd = ["keepstore",

-                "-volumes={}".format(keep0),
+                "-volume={}".format(keep0),

                 "-listen=:{}".format(port),
                 "-listen=:{}".format(port),

-                "-pid={}".format("{}/keep{}.pid".format(TEST_TMPDIR, n))]
+                "-pid="+_pidfile('keep{}'.format(n))]

 
     for arg, val in keep_args.iteritems():
         keep_cmd.append("{}={}".format(arg, val))
 
 
     for arg, val in keep_args.iteritems():
         keep_cmd.append("{}={}".format(arg, val))
 

-    kp0 = subprocess.Popen(keep_cmd)
-    with open("{}/keep{}.pid".format(TEST_TMPDIR, n), 'w') as f:
+    kp0 = subprocess.Popen(
+        keep_cmd, stdin=open('/dev/null'), stdout=sys.stderr)
+    with open(_pidfile('keep{}'.format(n)), 'w') as f:

         f.write(str(kp0.pid))
 
     with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
         f.write(str(kp0.pid))
 
     with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:


@@ -288,7 +294,7 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
         keep_args['--enforce-permissions'] = 'true'
 
     api = arvados.api(
         keep_args['--enforce-permissions'] = 'true'
 
     api = arvados.api(

-        'v1', cache=False,
+        version='v1',

         host=os.environ['ARVADOS_API_HOST'],
         token=os.environ['ARVADOS_API_TOKEN'],
         insecure=True)
         host=os.environ['ARVADOS_API_HOST'],
         token=os.environ['ARVADOS_API_TOKEN'],
         insecure=True)


@@ -311,7 +317,7 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
         }).execute()
 
 def _stop_keep(n):
         }).execute()
 
 def _stop_keep(n):

-    kill_server_pid("{}/keep{}.pid".format(TEST_TMPDIR, n), 0)
+    kill_server_pid(_pidfile('keep{}'.format(n)), 0)

     if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
         with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
             shutil.rmtree(r.read(), True)
     if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
         with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
             shutil.rmtree(r.read(), True)


@@ -324,6 +330,8 @@ def stop_keep():
     _stop_keep(1)
 
 def run_keep_proxy():
     _stop_keep(1)
 
 def run_keep_proxy():

+    if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        return

     stop_keep_proxy()
 
     admin_token = auth_token('admin')
     stop_keep_proxy()
 
     admin_token = auth_token('admin')


@@ -332,12 +340,12 @@ def run_keep_proxy():
     env['ARVADOS_API_TOKEN'] = admin_token
     kp = subprocess.Popen(
         ['keepproxy',
     env['ARVADOS_API_TOKEN'] = admin_token
     kp = subprocess.Popen(
         ['keepproxy',

-         '-pid={}/keepproxy.pid'.format(TEST_TMPDIR),
+         '-pid='+_pidfile('keepproxy'),

          '-listen=:{}'.format(port)],
          '-listen=:{}'.format(port)],

-        env=env)
+        env=env, stdin=open('/dev/null'), stdout=sys.stderr)

 
     api = arvados.api(
 
     api = arvados.api(

-        'v1', cache=False,
+        version='v1',

         host=os.environ['ARVADOS_API_HOST'],
         token=admin_token,
         insecure=True)
         host=os.environ['ARVADOS_API_HOST'],
         token=admin_token,
         insecure=True)


@@ -351,9 +359,100 @@ def run_keep_proxy():
         'service_ssl_flag': False,
     }}).execute()
     os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
         'service_ssl_flag': False,
     }}).execute()
     os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)

+    _setport('keepproxy', port)

 
 def stop_keep_proxy():
 
 def stop_keep_proxy():

-    kill_server_pid(os.path.join(TEST_TMPDIR, "keepproxy.pid"), 0)
+    if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        return
+    kill_server_pid(_pidfile('keepproxy'), wait=0)
+
+def run_arv_git_httpd():
+    if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        return
+    stop_arv_git_httpd()
+
+    gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
+    gitport = find_available_port()
+    env = os.environ.copy()
+    env.pop('ARVADOS_API_TOKEN', None)
+    agh = subprocess.Popen(
+        ['arv-git-httpd',
+         '-repo-root='+gitdir+'/test',
+         '-address=:'+str(gitport)],
+        env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+    with open(_pidfile('arv-git-httpd'), 'w') as f:
+        f.write(str(agh.pid))
+    _setport('arv-git-httpd', gitport)
+
+def stop_arv_git_httpd():
+    if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        return
+    kill_server_pid(_pidfile('arv-git-httpd'), wait=0)
+
+def run_nginx():
+    if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        return
+    nginxconf = {}
+    nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
+    nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
+    nginxconf['GITPORT'] = _getport('arv-git-httpd')
+    nginxconf['GITSSLPORT'] = find_available_port()
+    nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
+    nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
+
+    conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
+    conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
+    with open(conffile, 'w') as f:
+        f.write(re.sub(
+            r'{{([A-Z]+)}}',
+            lambda match: str(nginxconf.get(match.group(1))),
+            open(conftemplatefile).read()))
+
+    env = os.environ.copy()
+    env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
+    nginx = subprocess.Popen(
+        ['nginx',
+         '-g', 'error_log stderr info;',
+         '-g', 'pid '+_pidfile('nginx')+';',
+         '-c', conffile],
+        env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+    _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
+    _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
+
+def stop_nginx():
+    if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+        return
+    kill_server_pid(_pidfile('nginx'), wait=0)
+
+def _pidfile(program):
+    return os.path.join(TEST_TMPDIR, program + '.pid')
+
+def _portfile(program):
+    return os.path.join(TEST_TMPDIR, program + '.port')
+
+def _setport(program, port):
+    with open(_portfile(program), 'w') as f:
+        f.write(str(port))
+
+# Returns 9 if program is not up.
+def _getport(program):
+    try:
+        return int(open(_portfile(program)).read())
+    except IOError:
+        return 9
+
+def _apiconfig(key):
+    if _cached_config:
+        return _cached_config[key]
+    def _load(f):
+        return yaml.load(os.path.join(SERVICES_SRC_DIR, 'api', 'config', f))
+    cdefault = _load('application.default.yml')
+    csite = _load('application.yml')
+    _cached_config = {}
+    for section in [cdefault.get('common',{}), cdefault.get('test',{}),
+                    csite.get('common',{}), csite.get('test',{})]:
+        _cached_config.update(section)
+    return _cached_config[key]

 
 def fixture(fix):
     '''load a fixture yaml file'''
 
 def fixture(fix):
     '''load a fixture yaml file'''


@@ -435,14 +534,21 @@ class TestCaseWithServers(unittest.TestCase):
 
 
 if __name__ == "__main__":
 
 
 if __name__ == "__main__":

-    actions = ['start', 'stop',
-               'start_keep', 'stop_keep',
-               'start_keep_proxy', 'stop_keep_proxy']
+    actions = [
+        'start', 'stop',
+        'start_keep', 'stop_keep',
+        'start_keep_proxy', 'stop_keep_proxy',
+        'start_arv-git-httpd', 'stop_arv-git-httpd',
+        'start_nginx', 'stop_nginx',
+    ]

     parser = argparse.ArgumentParser()
     parser.add_argument('action', type=str, help="one of {}".format(actions))
     parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
     args = parser.parse_args()
 
     parser = argparse.ArgumentParser()
     parser.add_argument('action', type=str, help="one of {}".format(actions))
     parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
     args = parser.parse_args()
 

+    if args.action not in actions:
+        print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr)
+        sys.exit(1)

     if args.action == 'start':
         stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
         run(leave_running_atexit=True)
     if args.action == 'start':
         stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
         run(leave_running_atexit=True)


@@ -464,5 +570,13 @@ if __name__ == "__main__":
         run_keep_proxy()
     elif args.action == 'stop_keep_proxy':
         stop_keep_proxy()
         run_keep_proxy()
     elif args.action == 'stop_keep_proxy':
         stop_keep_proxy()

+    elif args.action == 'start_arv-git-httpd':
+        run_arv_git_httpd()
+    elif args.action == 'stop_arv-git-httpd':
+        stop_arv_git_httpd()
+    elif args.action == 'start_nginx':
+        run_nginx()
+    elif args.action == 'stop_nginx':
+        stop_nginx()

     else:
     else:

-        print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions))
+        raise Exception("action recognized but not implemented!?")