#!/usr/bin/env python
+from __future__ import print_function
import argparse
import atexit
import httplib2
import re
import shutil
import signal
+import socket
import subprocess
+import string
import sys
import tempfile
import time
# Add the Python SDK source to the library path.
sys.path.insert(1, os.path.dirname(MY_DIRNAME))
-import arvados.api
+import arvados
import arvados.config
ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
os.mkdir(TEST_TMPDIR)
my_api_host = None
+_cached_config = {}
def find_server_pid(PID_PATH, wait=10):
now = time.time()
return server_pid
-def kill_server_pid(pidfile, wait=10, passenger=False):
+def kill_server_pid(pidfile, wait=10, passenger_root=False):
# Must re-import modules in order to work during atexit
import os
import signal
import subprocess
import time
try:
- if passenger:
+ if passenger_root:
# First try to shut down nicely
restore_cwd = os.getcwd()
- os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
+ os.chdir(passenger_root)
subprocess.call([
'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
os.chdir(restore_cwd)
with open(pidfile, 'r') as f:
server_pid = int(f.read())
while now <= timeout:
- if not passenger or timeout - now < wait / 2:
+ if not passenger_root or timeout - now < wait / 2:
# Half timeout has elapsed. Start sending SIGTERM
os.kill(server_pid, signal.SIGTERM)
# Raise OSError if process has disappeared
except OSError:
pass
+def find_available_port():
+ """Return an IPv4 port number that is not in use right now.
+
+ We assume whoever needs to use the returned port is able to reuse
+ a recently used port without waiting for TIME_WAIT (see
+ SO_REUSEADDR / SO_REUSEPORT).
+
+ Some opportunity for races here, but it's better than choosing
+ something at random and not checking at all. If all of our servers
+ (hey Passenger) knew that listening on port 0 was a thing, the OS
+ would take care of the races, and this wouldn't be needed at all.
+ """
+
+ sock = socket.socket()
+ sock.bind(('0.0.0.0', 0))
+ port = sock.getsockname()[1]
+ sock.close()
+ return port
+
def run(leave_running_atexit=False):
"""Ensure an API server is running, and ARVADOS_API_* env vars have
admin credentials for it.
+
+ If ARVADOS_TEST_API_HOST is set, a parent process has started a
+ test server for us to use: we just need to reset() it using the
+ admin token fixture.
+
+ If a previous call to run() started a new server process, and it
+ is still running, we just need to reset() it to fixture state and
+ return.
+
+ If neither of those options work out, we'll really start a new
+ server.
"""
global my_api_host
# Delete cached discovery document.
shutil.rmtree(arvados.http_cache('discovery'))
- os.environ['ARVADOS_API_TOKEN'] = auth_token('admin')
- os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
-
pid_file = os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH)
pid_file_ok = find_server_pid(pid_file, 0)
- if pid_file_ok:
- try:
- reset()
- return
- except:
- pass
+ existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
+ if existing_api_host and pid_file_ok:
+ if existing_api_host == my_api_host:
+ try:
+ return reset()
+ except:
+ # Fall through to shutdown-and-start case.
+ pass
+ else:
+ # Server was provided by parent. Can't recover if it's
+ # unresettable.
+ return reset()
+
+ # Before trying to start up our own server, call stop() to avoid
+ # "Phusion Passenger Standalone is already running on PID 12345".
+ # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
+ # we know the server is ours to kill.)
+ stop(force=True)
restore_cwd = os.getcwd()
- os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
+ api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
+ os.chdir(api_src_dir)
# Either we haven't started a server of our own yet, or it has
# died, or we have lost our credentials, or something else is
# preventing us from calling reset(). Start a new one.
+ if not os.path.exists('tmp'):
+ os.makedirs('tmp')
+
+ if not os.path.exists('tmp/api'):
+ os.makedirs('tmp/api')
+
+ if not os.path.exists('tmp/logs'):
+ os.makedirs('tmp/logs')
+
if not os.path.exists('tmp/self-signed.pem'):
# We assume here that either passenger reports its listening
# address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
'-out', 'tmp/self-signed.pem',
'-keyout', 'tmp/self-signed.key',
'-days', '3650',
- '-subj', '/CN=0.0.0.0'])
+ '-subj', '/CN=0.0.0.0'],
+ stdout=sys.stderr)
+
+ # Install the git repository fixtures.
+ gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
+ gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
+ if not os.path.isdir(gitdir):
+ os.makedirs(gitdir)
+ subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
- port = random.randint(20000, 40000)
+ port = find_available_port()
env = os.environ.copy()
env['RAILS_ENV'] = 'test'
env['ARVADOS_WEBSOCKETS'] = 'yes'
env=env)
if not leave_running_atexit:
- atexit.register(kill_server_pid, pidfile, passenger=True)
+ atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
match = re.search(r'Accessible via: https://(.*?)/', start_msg)
if not match:
os.chdir(restore_cwd)
def reset():
+ """Reset the test server to fixture state.
+
+ This resets the ARVADOS_TEST_API_HOST provided by a parent process
+ if any, otherwise the server started by run().
+
+ It also resets ARVADOS_* environment vars to point to the test
+ server with admin credentials.
+ """
+ existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
token = auth_token('admin')
httpclient = httplib2.Http(ca_certs=os.path.join(
SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
httpclient.request(
- 'https://{}/database/reset'.format(os.environ['ARVADOS_API_HOST']),
+ 'https://{}/database/reset'.format(existing_api_host),
'POST',
headers={'Authorization': 'OAuth2 {}'.format(token)})
+ os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
+ os.environ['ARVADOS_API_HOST'] = existing_api_host
+ os.environ['ARVADOS_API_TOKEN'] = token
def stop(force=False):
- """Stop the API server, if one is running. If force==True, kill it
- even if we didn't start it ourselves.
+ """Stop the API server, if one is running.
+
+ If force==False, kill it only if we started it ourselves. (This
+ supports the use case where a Python test suite calls run(), but
+ run() just uses the ARVADOS_TEST_API_HOST provided by the parent
+ process, and the test suite cleans up after itself by calling
+ stop(). In this case the test server provided by the parent
+ process should be left alone.)
+
+ If force==True, kill it even if we didn't start it
+ ourselves. (This supports the use case in __main__, where "run"
+ and "stop" happen in different processes.)
"""
global my_api_host
if force or my_api_host is not None:
def _start_keep(n, keep_args):
keep0 = tempfile.mkdtemp()
- port = random.randint(20000, 40000)
+ port = find_available_port()
keep_cmd = ["keepstore",
- "-volumes={}".format(keep0),
+ "-volume={}".format(keep0),
"-listen=:{}".format(port),
- "-pid={}".format("{}/keep{}.pid".format(TEST_TMPDIR, n))]
+ "-pid="+_pidfile('keep{}'.format(n))]
for arg, val in keep_args.iteritems():
keep_cmd.append("{}={}".format(arg, val))
- kp0 = subprocess.Popen(keep_cmd)
- with open("{}/keep{}.pid".format(TEST_TMPDIR, n), 'w') as f:
+ kp0 = subprocess.Popen(
+ keep_cmd, stdin=open('/dev/null'), stdout=sys.stderr)
+ with open(_pidfile('keep{}'.format(n)), 'w') as f:
f.write(str(kp0.pid))
with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
keep_args['--enforce-permissions'] = 'true'
api = arvados.api(
- 'v1', cache=False,
+ version='v1',
host=os.environ['ARVADOS_API_HOST'],
token=os.environ['ARVADOS_API_TOKEN'],
insecure=True)
}).execute()
def _stop_keep(n):
- kill_server_pid("{}/keep{}.pid".format(TEST_TMPDIR, n), 0)
+ kill_server_pid(_pidfile('keep{}'.format(n)), 0)
if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
shutil.rmtree(r.read(), True)
_stop_keep(1)
def run_keep_proxy():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
stop_keep_proxy()
admin_token = auth_token('admin')
- port = random.randint(20000,40000)
+ port = find_available_port()
env = os.environ.copy()
env['ARVADOS_API_TOKEN'] = admin_token
kp = subprocess.Popen(
['keepproxy',
- '-pid={}/keepproxy.pid'.format(TEST_TMPDIR),
+ '-pid='+_pidfile('keepproxy'),
'-listen=:{}'.format(port)],
- env=env)
+ env=env, stdin=open('/dev/null'), stdout=sys.stderr)
api = arvados.api(
- 'v1', cache=False,
+ version='v1',
host=os.environ['ARVADOS_API_HOST'],
token=admin_token,
insecure=True)
'service_ssl_flag': False,
}}).execute()
os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
+ _setport('keepproxy', port)
def stop_keep_proxy():
- kill_server_pid(os.path.join(TEST_TMPDIR, "keepproxy.pid"), 0)
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ kill_server_pid(_pidfile('keepproxy'), wait=0)
+
+def run_arv_git_httpd():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ stop_arv_git_httpd()
+
+ gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
+ gitport = find_available_port()
+ env = os.environ.copy()
+ env.pop('ARVADOS_API_TOKEN', None)
+ agh = subprocess.Popen(
+ ['arv-git-httpd',
+ '-repo-root='+gitdir+'/test',
+ '-address=:'+str(gitport)],
+ env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ with open(_pidfile('arv-git-httpd'), 'w') as f:
+ f.write(str(agh.pid))
+ _setport('arv-git-httpd', gitport)
+
+def stop_arv_git_httpd():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ kill_server_pid(_pidfile('arv-git-httpd'), wait=0)
+
+def run_nginx():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ nginxconf = {}
+ nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
+ nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
+ nginxconf['GITPORT'] = _getport('arv-git-httpd')
+ nginxconf['GITSSLPORT'] = find_available_port()
+ nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
+ nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
+
+ conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
+ conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
+ with open(conffile, 'w') as f:
+ f.write(re.sub(
+ r'{{([A-Z]+)}}',
+ lambda match: str(nginxconf.get(match.group(1))),
+ open(conftemplatefile).read()))
+
+ env = os.environ.copy()
+ env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
+ nginx = subprocess.Popen(
+ ['nginx',
+ '-g', 'error_log stderr info;',
+ '-g', 'pid '+_pidfile('nginx')+';',
+ '-c', conffile],
+ env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
+ _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
+
+def stop_nginx():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ kill_server_pid(_pidfile('nginx'), wait=0)
+
+def _pidfile(program):
+ return os.path.join(TEST_TMPDIR, program + '.pid')
+
+def _portfile(program):
+ return os.path.join(TEST_TMPDIR, program + '.port')
+
+def _setport(program, port):
+ with open(_portfile(program), 'w') as f:
+ f.write(str(port))
+
+# Returns 9 if program is not up.
+def _getport(program):
+ try:
+ return int(open(_portfile(program)).read())
+ except IOError:
+ return 9
+
+def _apiconfig(key):
+ if _cached_config:
+ return _cached_config[key]
+ def _load(f, required=True):
+ fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
+ if not required and not os.path.exists(fullpath):
+ return {}
+ return yaml.load(fullpath)
+ cdefault = _load('application.default.yml')
+ csite = _load('application.yml', required=False)
+ _cached_config = {}
+ for section in [cdefault.get('common',{}), cdefault.get('test',{}),
+ csite.get('common',{}), csite.get('test',{})]:
+ _cached_config.update(section)
+ return _cached_config[key]
def fixture(fix):
'''load a fixture yaml file'''
if __name__ == "__main__":
- actions = ['start', 'stop',
- 'start_keep', 'stop_keep',
- 'start_keep_proxy', 'stop_keep_proxy']
+ actions = [
+ 'start', 'stop',
+ 'start_keep', 'stop_keep',
+ 'start_keep_proxy', 'stop_keep_proxy',
+ 'start_arv-git-httpd', 'stop_arv-git-httpd',
+ 'start_nginx', 'stop_nginx',
+ ]
parser = argparse.ArgumentParser()
parser.add_argument('action', type=str, help="one of {}".format(actions))
parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
args = parser.parse_args()
+ if args.action not in actions:
+ print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr)
+ sys.exit(1)
if args.action == 'start':
stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
run(leave_running_atexit=True)
run_keep_proxy()
elif args.action == 'stop_keep_proxy':
stop_keep_proxy()
+ elif args.action == 'start_arv-git-httpd':
+ run_arv_git_httpd()
+ elif args.action == 'stop_arv-git-httpd':
+ stop_arv_git_httpd()
+ elif args.action == 'start_nginx':
+ run_nginx()
+ elif args.action == 'stop_nginx':
+ stop_nginx()
else:
- print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions))
+ raise Exception("action recognized but not implemented!?")
projects / arvados.git / blobdiff