X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a6c0e01403377311a12f6d9ae587d92dbc00e1fd..9c569a32645164e99fa44867626bb5a11887c338:/sdk/python/tests/run_test_server.py diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index dc95d8a9a5..739c754995 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -8,6 +8,7 @@ import subprocess import sys import tempfile import time +import unittest import yaml MY_DIRNAME = os.path.dirname(os.path.realpath(__file__)) @@ -20,10 +21,18 @@ if __name__ == '__main__' and os.path.exists( import arvados.api import arvados.config -ARV_API_SERVER_DIR = '../../../services/api' -KEEP_SERVER_DIR = '../../../services/keep' +SERVICES_SRC_DIR = os.path.join(MY_DIRNAME, '../../../services') SERVER_PID_PATH = 'tmp/pids/webrick-test.pid' WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid' +if 'GOPATH' in os.environ: + gopaths = os.environ['GOPATH'].split(':') + gobins = [os.path.join(path, 'bin') for path in gopaths] + os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH'] + +if os.path.isdir('tests'): + TEST_TMPDIR = 'tests/tmp' +else: + TEST_TMPDIR = 'tmp' def find_server_pid(PID_PATH, wait=10): now = time.time() @@ -34,7 +43,7 @@ def find_server_pid(PID_PATH, wait=10): try: with open(PID_PATH, 'r') as f: server_pid = int(f.read()) - good_pid = (os.kill(server_pid, 0) == None) + good_pid = (os.kill(server_pid, 0) is None) except IOError: good_pid = False except OSError: @@ -53,7 +62,7 @@ def kill_server_pid(PID_PATH, wait=10): with open(PID_PATH, 'r') as f: server_pid = int(f.read()) while now <= timeout: - os.kill(server_pid, signal.SIGTERM) == None + os.kill(server_pid, signal.SIGTERM) os.getpgid(server_pid) # throw OSError if no such pid now = time.time() time.sleep(0.1) @@ -64,7 +73,7 @@ def kill_server_pid(PID_PATH, wait=10): def run(websockets=False, reuse_server=False): cwd = os.getcwd() - os.chdir(os.path.join(MY_DIRNAME, ARV_API_SERVER_DIR)) + os.chdir(os.path.join(SERVICES_SRC_DIR, 'api')) if websockets: pid_file = WEBSOCKETS_SERVER_PID_PATH @@ -73,7 +82,7 @@ def run(websockets=False, reuse_server=False): test_pid = find_server_pid(pid_file, 0) - if test_pid == None or not reuse_server: + if test_pid is None or not reuse_server: # do not try to run both server variants at once stop() @@ -86,27 +95,19 @@ def run(websockets=False, reuse_server=False): subprocess.call(['bundle', 'exec', 'rake', 'db:test:load']) subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load']) + subprocess.call(['bundle', 'exec', 'rails', 'server', '-d', + '--pid', + os.path.join(os.getcwd(), SERVER_PID_PATH), + '-p3000']) + os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000" + if websockets: - os.environ["ARVADOS_WEBSOCKETS"] = "true" - subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes', - '-out', './self-signed.pem', - '-keyout', './self-signed.key', - '-days', '3650', - '-subj', '/CN=localhost']) + os.environ["ARVADOS_WEBSOCKETS"] = "ws-only" subprocess.call(['bundle', 'exec', 'passenger', 'start', '-d', '-p3333', '--pid-file', - os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH), - '--ssl', - '--ssl-certificate', 'self-signed.pem', - '--ssl-certificate-key', 'self-signed.key']) - os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333" - else: - subprocess.call(['bundle', 'exec', 'rails', 'server', '-d', - '--pid', - os.path.join(os.getcwd(), SERVER_PID_PATH), - '-p3001']) - os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001" + os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH) + ]) pid = find_server_pid(SERVER_PID_PATH) @@ -116,7 +117,7 @@ def run(websockets=False, reuse_server=False): def stop(): cwd = os.getcwd() - os.chdir(os.path.join(MY_DIRNAME, ARV_API_SERVER_DIR)) + os.chdir(os.path.join(SERVICES_SRC_DIR, 'api')) kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0) kill_server_pid(SERVER_PID_PATH, 0) @@ -135,48 +136,39 @@ def stop(): def _start_keep(n, keep_args): keep0 = tempfile.mkdtemp() - keep_cmd = ["bin/keep", + keep_cmd = ["keepstore", "-volumes={}".format(keep0), "-listen=:{}".format(25107+n), - "-pid={}".format("tmp/keep{}.pid".format(n))] + "-pid={}".format("{}/keep{}.pid".format(TEST_TMPDIR, n))] for arg, val in keep_args.iteritems(): keep_cmd.append("{}={}".format(arg, val)) kp0 = subprocess.Popen(keep_cmd) - with open("tmp/keep{}.pid".format(n), 'w') as f: + with open("{}/keep{}.pid".format(TEST_TMPDIR, n), 'w') as f: f.write(str(kp0.pid)) - with open("tmp/keep{}.volume".format(n), 'w') as f: + with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f: f.write(keep0) def run_keep(blob_signing_key=None, enforce_permissions=False): stop_keep() - cwd = os.getcwd() - os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR)) - if os.environ.get('GOPATH') == None: - os.environ["GOPATH"] = os.getcwd() - else: - os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"] - - subprocess.call(["./go.sh", "install", "keep"]) - - if not os.path.exists("tmp"): - os.mkdir("tmp") + if not os.path.exists(TEST_TMPDIR): + os.mkdir(TEST_TMPDIR) keep_args = {} if blob_signing_key: - with open("tmp/keep.blob_signing_key", "w") as f: + with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f: + keep_args['--permission-key-file'] = f.name f.write(blob_signing_key) - keep_args['--permission-key-file'] = 'tmp/keep.blob_signing_key' if enforce_permissions: keep_args['--enforce-permissions'] = 'true' _start_keep(0, keep_args) _start_keep(1, keep_args) - os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001" + os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000" os.environ["ARVADOS_API_HOST_INSECURE"] = "true" authorize_with("admin") @@ -186,71 +178,68 @@ def run_keep(blob_signing_key=None, enforce_permissions=False): for d in api.keep_disks().list().execute()['items']: api.keep_disks().delete(uuid=d['uuid']).execute() - s1 = api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25107, "service_type": "disk"} }).execute() - s2 = api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25108, "service_type": "disk"} }).execute() + s1 = api.keep_services().create(body={"keep_service": { + "uuid": "zzzzz-bi6l4-5bo5n1iekkjyz6b", + "service_host": "localhost", + "service_port": 25107, + "service_type": "disk" + }}).execute() + s2 = api.keep_services().create(body={"keep_service": { + "uuid": "zzzzz-bi6l4-2nz60e0ksj7vr3s", + "service_host": "localhost", + "service_port": 25108, + "service_type": "disk" + }}).execute() api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute() api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute() - os.chdir(cwd) - def _stop_keep(n): - kill_server_pid("tmp/keep{}.pid".format(n), 0) - if os.path.exists("tmp/keep{}.volume".format(n)): - with open("tmp/keep{}.volume".format(n), 'r') as r: + kill_server_pid("{}/keep{}.pid".format(TEST_TMPDIR, n), 0) + if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)): + with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r: shutil.rmtree(r.read(), True) - os.unlink("tmp/keep{}.volume".format(n)) - if os.path.exists("tmp/keep.blob_signing_key"): - os.remove("tmp/keep.blob_signing_key") + os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n)) + if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")): + os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")) def stop_keep(): - cwd = os.getcwd() - os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR)) - _stop_keep(0) _stop_keep(1) - os.chdir(cwd) - def run_keep_proxy(auth): stop_keep_proxy() - cwd = os.getcwd() - os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR)) - if os.environ.get('GOPATH') == None: - os.environ["GOPATH"] = os.getcwd() - else: - os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"] - - subprocess.call(["./go.sh", "install", "arvados.org/keepproxy"]) - - if not os.path.exists("tmp"): - os.mkdir("tmp") + if not os.path.exists(TEST_TMPDIR): + os.mkdir(TEST_TMPDIR) - os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001" + os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000" os.environ["ARVADOS_API_HOST_INSECURE"] = "true" os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"] - kp0 = subprocess.Popen(["bin/keepproxy", "-pid=tmp/keepproxy.pid", "-listen=:{}".format(25101)]) + kp0 = subprocess.Popen(["keepproxy", + "-pid={}/keepproxy.pid".format(TEST_TMPDIR), + "-listen=:{}".format(25101)]) authorize_with("admin") api = arvados.api('v1', cache=False) api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25101, "service_type": "proxy"} }).execute() - arvados.config.settings()["ARVADOS_KEEP_PROXY"] = "http://localhost:25101" - - os.chdir(cwd) + os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:25101" def stop_keep_proxy(): - cwd = os.getcwd() - os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR)) - kill_server_pid("tmp/keepproxy.pid", 0) - os.chdir(cwd) + kill_server_pid(os.path.join(TEST_TMPDIR, "keepproxy.pid"), 0) def fixture(fix): '''load a fixture yaml file''' - with open(os.path.join(MY_DIRNAME, ARV_API_SERVER_DIR, "test", "fixtures", + with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures", fix + ".yml")) as f: - return yaml.load(f.read()) + yaml_file = f.read() + try: + trim_index = yaml_file.index("# Test Helper trims the rest of the file") + yaml_file = yaml_file[0:trim_index] + except ValueError: + pass + return yaml.load(yaml_file) def authorize_with(token): '''token is the symbolic name of the token from the api_client_authorizations fixture''' @@ -258,6 +247,65 @@ def authorize_with(token): arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST") arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true" +class TestCaseWithServers(unittest.TestCase): + """TestCase to start and stop supporting Arvados servers. + + Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER + class variables as a dictionary of keyword arguments. If you do, + setUpClass will start the corresponding servers by passing these + keyword arguments to the run, run_keep, and/or run_keep_server + functions, respectively. It will also set Arvados environment + variables to point to these servers appropriately. If you don't + run a Keep or Keep proxy server, setUpClass will set up a + temporary directory for Keep local storage, and set it as + KEEP_LOCAL_STORE. + + tearDownClass will stop any servers started, and restore the + original environment. + """ + MAIN_SERVER = None + KEEP_SERVER = None + KEEP_PROXY_SERVER = None + + @staticmethod + def _restore_dict(src, dest): + for key in dest.keys(): + if key not in src: + del dest[key] + dest.update(src) + + @classmethod + def setUpClass(cls): + cls._orig_environ = os.environ.copy() + cls._orig_config = arvados.config.settings().copy() + cls._cleanup_funcs = [] + for server_kwargs, start_func, stop_func in ( + (cls.MAIN_SERVER, run, stop), + (cls.KEEP_SERVER, run_keep, stop_keep), + (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)): + if server_kwargs is not None: + start_func(**server_kwargs) + cls._cleanup_funcs.append(stop_func) + os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None) + if cls.KEEP_PROXY_SERVER is None: + os.environ.pop('ARVADOS_KEEP_PROXY', None) + if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None): + cls.local_store = tempfile.mkdtemp() + os.environ['KEEP_LOCAL_STORE'] = cls.local_store + cls._cleanup_funcs.append( + lambda: shutil.rmtree(cls.local_store, ignore_errors=True)) + else: + os.environ.pop('KEEP_LOCAL_STORE', None) + arvados.config.initialize() + + @classmethod + def tearDownClass(cls): + for clean_func in cls._cleanup_funcs: + clean_func() + cls._restore_dict(cls._orig_environ, os.environ) + cls._restore_dict(cls._orig_config, arvados.config.settings()) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''') @@ -268,7 +316,7 @@ if __name__ == "__main__": if args.action == 'start': run(websockets=args.websockets, reuse_server=args.reuse) - if args.auth != None: + if args.auth is not None: authorize_with(args.auth) print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"])) print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))