import sys
import tempfile
import time
+import unittest
import yaml
MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
gobins = [os.path.join(path, 'bin') for path in gopaths]
os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
+if os.path.isdir('tests'):
+ TEST_TMPDIR = 'tests/tmp'
+else:
+ TEST_TMPDIR = 'tmp'
+
def find_server_pid(PID_PATH, wait=10):
now = time.time()
timeout = now + wait
try:
with open(PID_PATH, 'r') as f:
server_pid = int(f.read())
- good_pid = (os.kill(server_pid, 0) == None)
+ good_pid = (os.kill(server_pid, 0) is None)
except IOError:
good_pid = False
except OSError:
with open(PID_PATH, 'r') as f:
server_pid = int(f.read())
while now <= timeout:
- os.kill(server_pid, signal.SIGTERM) == None
+ os.kill(server_pid, signal.SIGTERM)
os.getpgid(server_pid) # throw OSError if no such pid
now = time.time()
time.sleep(0.1)
test_pid = find_server_pid(pid_file, 0)
- if test_pid == None or not reuse_server:
+ if test_pid is None or not reuse_server:
# do not try to run both server variants at once
stop()
subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
+ subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
+ '--pid',
+ os.path.join(os.getcwd(), SERVER_PID_PATH),
+ '-p3000'])
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
+
if websockets:
- os.environ["ARVADOS_WEBSOCKETS"] = "true"
- subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
- '-out', './self-signed.pem',
- '-keyout', './self-signed.key',
- '-days', '3650',
- '-subj', '/CN=localhost'])
+ os.environ["ARVADOS_WEBSOCKETS"] = "ws-only"
subprocess.call(['bundle', 'exec',
'passenger', 'start', '-d', '-p3333',
'--pid-file',
- os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH),
- '--ssl',
- '--ssl-certificate', 'self-signed.pem',
- '--ssl-certificate-key', 'self-signed.key'])
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333"
- else:
- subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
- '--pid',
- os.path.join(os.getcwd(), SERVER_PID_PATH),
- '-p3001'])
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
+ os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH)
+ ])
pid = find_server_pid(SERVER_PID_PATH)
keep_cmd = ["keepstore",
"-volumes={}".format(keep0),
"-listen=:{}".format(25107+n),
- "-pid={}".format("tmp/keep{}.pid".format(n))]
+ "-pid={}".format("{}/keep{}.pid".format(TEST_TMPDIR, n))]
for arg, val in keep_args.iteritems():
keep_cmd.append("{}={}".format(arg, val))
kp0 = subprocess.Popen(keep_cmd)
- with open("tmp/keep{}.pid".format(n), 'w') as f:
+ with open("{}/keep{}.pid".format(TEST_TMPDIR, n), 'w') as f:
f.write(str(kp0.pid))
- with open("tmp/keep{}.volume".format(n), 'w') as f:
+ with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
f.write(keep0)
def run_keep(blob_signing_key=None, enforce_permissions=False):
stop_keep()
- if not os.path.exists("tmp"):
- os.mkdir("tmp")
+ if not os.path.exists(TEST_TMPDIR):
+ os.mkdir(TEST_TMPDIR)
keep_args = {}
if blob_signing_key:
- with open("tmp/keep.blob_signing_key", "w") as f:
+ with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
+ keep_args['--permission-key-file'] = f.name
f.write(blob_signing_key)
- keep_args['--permission-key-file'] = 'tmp/keep.blob_signing_key'
if enforce_permissions:
keep_args['--enforce-permissions'] = 'true'
_start_keep(0, keep_args)
_start_keep(1, keep_args)
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
authorize_with("admin")
for d in api.keep_disks().list().execute()['items']:
api.keep_disks().delete(uuid=d['uuid']).execute()
- s1 = api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25107, "service_type": "disk"} }).execute()
- s2 = api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25108, "service_type": "disk"} }).execute()
+ s1 = api.keep_services().create(body={"keep_service": {
+ "uuid": "zzzzz-bi6l4-5bo5n1iekkjyz6b",
+ "service_host": "localhost",
+ "service_port": 25107,
+ "service_type": "disk"
+ }}).execute()
+ s2 = api.keep_services().create(body={"keep_service": {
+ "uuid": "zzzzz-bi6l4-2nz60e0ksj7vr3s",
+ "service_host": "localhost",
+ "service_port": 25108,
+ "service_type": "disk"
+ }}).execute()
api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute()
api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
def _stop_keep(n):
- kill_server_pid("tmp/keep{}.pid".format(n), 0)
- if os.path.exists("tmp/keep{}.volume".format(n)):
- with open("tmp/keep{}.volume".format(n), 'r') as r:
+ kill_server_pid("{}/keep{}.pid".format(TEST_TMPDIR, n), 0)
+ if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
+ with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
shutil.rmtree(r.read(), True)
- os.unlink("tmp/keep{}.volume".format(n))
- if os.path.exists("tmp/keep.blob_signing_key"):
- os.remove("tmp/keep.blob_signing_key")
+ os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
+ if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
+ os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
def stop_keep():
_stop_keep(0)
def run_keep_proxy(auth):
stop_keep_proxy()
- if not os.path.exists("tmp"):
- os.mkdir("tmp")
+ if not os.path.exists(TEST_TMPDIR):
+ os.mkdir(TEST_TMPDIR)
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
kp0 = subprocess.Popen(["keepproxy",
- "-pid=tmp/keepproxy.pid", "-listen=:{}".format(25101)])
+ "-pid={}/keepproxy.pid".format(TEST_TMPDIR),
+ "-listen=:{}".format(25101)])
authorize_with("admin")
api = arvados.api('v1', cache=False)
api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25101, "service_type": "proxy"} }).execute()
- arvados.config.settings()["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
+ os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
def stop_keep_proxy():
- kill_server_pid("tmp/keepproxy.pid", 0)
+ kill_server_pid(os.path.join(TEST_TMPDIR, "keepproxy.pid"), 0)
def fixture(fix):
'''load a fixture yaml file'''
with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
fix + ".yml")) as f:
- return yaml.load(f.read())
+ yaml_file = f.read()
+ try:
+ trim_index = yaml_file.index("# Test Helper trims the rest of the file")
+ yaml_file = yaml_file[0:trim_index]
+ except ValueError:
+ pass
+ return yaml.load(yaml_file)
def authorize_with(token):
'''token is the symbolic name of the token from the api_client_authorizations fixture'''
arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
+class TestCaseWithServers(unittest.TestCase):
+ """TestCase to start and stop supporting Arvados servers.
+
+ Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
+ class variables as a dictionary of keyword arguments. If you do,
+ setUpClass will start the corresponding servers by passing these
+ keyword arguments to the run, run_keep, and/or run_keep_server
+ functions, respectively. It will also set Arvados environment
+ variables to point to these servers appropriately. If you don't
+ run a Keep or Keep proxy server, setUpClass will set up a
+ temporary directory for Keep local storage, and set it as
+ KEEP_LOCAL_STORE.
+
+ tearDownClass will stop any servers started, and restore the
+ original environment.
+ """
+ MAIN_SERVER = None
+ KEEP_SERVER = None
+ KEEP_PROXY_SERVER = None
+
+ @staticmethod
+ def _restore_dict(src, dest):
+ for key in dest.keys():
+ if key not in src:
+ del dest[key]
+ dest.update(src)
+
+ @classmethod
+ def setUpClass(cls):
+ cls._orig_environ = os.environ.copy()
+ cls._orig_config = arvados.config.settings().copy()
+ cls._cleanup_funcs = []
+ for server_kwargs, start_func, stop_func in (
+ (cls.MAIN_SERVER, run, stop),
+ (cls.KEEP_SERVER, run_keep, stop_keep),
+ (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
+ if server_kwargs is not None:
+ start_func(**server_kwargs)
+ cls._cleanup_funcs.append(stop_func)
+ os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
+ if cls.KEEP_PROXY_SERVER is None:
+ os.environ.pop('ARVADOS_KEEP_PROXY', None)
+ if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
+ cls.local_store = tempfile.mkdtemp()
+ os.environ['KEEP_LOCAL_STORE'] = cls.local_store
+ cls._cleanup_funcs.append(
+ lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
+ else:
+ os.environ.pop('KEEP_LOCAL_STORE', None)
+ arvados.config.initialize()
+
+ @classmethod
+ def tearDownClass(cls):
+ for clean_func in cls._cleanup_funcs:
+ clean_func()
+ cls._restore_dict(cls._orig_environ, os.environ)
+ cls._restore_dict(cls._orig_config, arvados.config.settings())
+
+
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
if args.action == 'start':
run(websockets=args.websockets, reuse_server=args.reuse)
- if args.auth != None:
+ if args.auth is not None:
authorize_with(args.auth)
print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
projects / arvados.git / blobdiff