sdk/python/tests/run_test_server.py

   1 #!/usr/bin/env python
   2
   3 import argparse
   4 import os
   5 import shutil
   6 import signal
   7 import subprocess
   8 import sys
   9 import tempfile
  10 import time
  11 import unittest
  12 import yaml
  13
  14 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
  15 if __name__ == '__main__' and os.path.exists(
  16       os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
  17     # We're being launched to support another test suite.
  18     # Add the Python SDK source to the library path.
  19     sys.path.insert(1, os.path.dirname(MY_DIRNAME))
  20
  21 import arvados.api
  22 import arvados.config
  23
  24 SERVICES_SRC_DIR = os.path.join(MY_DIRNAME, '../../../services')
  25 SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'
  26 WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'
  27 if 'GOPATH' in os.environ:
  28     gopaths = os.environ['GOPATH'].split(':')
  29     gobins = [os.path.join(path, 'bin') for path in gopaths]
  30     os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
  31
  32 if os.path.isdir('tests'):
  33     TEST_TMPDIR = 'tests/tmp'
  34 else:
  35     TEST_TMPDIR = 'tmp'
  36
  37 def find_server_pid(PID_PATH, wait=10):
  38     now = time.time()
  39     timeout = now + wait
  40     good_pid = False
  41     while (not good_pid) and (now <= timeout):
  42         time.sleep(0.2)
  43         try:
  44             with open(PID_PATH, 'r') as f:
  45                 server_pid = int(f.read())
  46             good_pid = (os.kill(server_pid, 0) is None)
  47         except IOError:
  48             good_pid = False
  49         except OSError:
  50             good_pid = False
  51         now = time.time()
  52
  53     if not good_pid:
  54         return None
  55
  56     return server_pid
  57
  58 def kill_server_pid(PID_PATH, wait=10):
  59     try:
  60         now = time.time()
  61         timeout = now + wait
  62         with open(PID_PATH, 'r') as f:
  63             server_pid = int(f.read())
  64         while now <= timeout:
  65             os.kill(server_pid, signal.SIGTERM)
  66             os.getpgid(server_pid) # throw OSError if no such pid
  67             now = time.time()
  68             time.sleep(0.1)
  69     except IOError:
  70         good_pid = False
  71     except OSError:
  72         good_pid = False
  73
  74 def run(websockets=False, reuse_server=False):
  75     cwd = os.getcwd()
  76     os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
  77
  78     if websockets:
  79         pid_file = WEBSOCKETS_SERVER_PID_PATH
  80     else:
  81         pid_file = SERVER_PID_PATH
  82
  83     test_pid = find_server_pid(pid_file, 0)
  84
  85     if test_pid is None or not reuse_server:
  86         # do not try to run both server variants at once
  87         stop()
  88
  89         # delete cached discovery document
  90         shutil.rmtree(arvados.http_cache('discovery'))
  91
  92         # Setup database
  93         os.environ["RAILS_ENV"] = "test"
  94         subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])
  95         subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
  96         subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
  97
  98         subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
  99                          '--pid',
 100                          os.path.join(os.getcwd(), SERVER_PID_PATH),
 101                          '-p3000'])
 102         os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
 103
 104         if websockets:
 105             os.environ["ARVADOS_WEBSOCKETS"] = "ws-only"
 106             subprocess.call(['bundle', 'exec',
 107                              'passenger', 'start', '-d', '-p3333',
 108                              '--pid-file',
 109                              os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH)
 110                          ])
 111
 112         pid = find_server_pid(SERVER_PID_PATH)
 113
 114     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 115     os.environ["ARVADOS_API_TOKEN"] = ""
 116     os.chdir(cwd)
 117
 118 def stop():
 119     cwd = os.getcwd()
 120     os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
 121
 122     kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
 123     kill_server_pid(SERVER_PID_PATH, 0)
 124
 125     try:
 126         os.unlink('self-signed.pem')
 127     except:
 128         pass
 129
 130     try:
 131         os.unlink('self-signed.key')
 132     except:
 133         pass
 134
 135     os.chdir(cwd)
 136
 137 def _start_keep(n, keep_args):
 138     keep0 = tempfile.mkdtemp()
 139     keep_cmd = ["keepstore",
 140                 "-volumes={}".format(keep0),
 141                 "-listen=:{}".format(25107+n),
 142                 "-pid={}".format("{}/keep{}.pid".format(TEST_TMPDIR, n))]
 143
 144     for arg, val in keep_args.iteritems():
 145         keep_cmd.append("{}={}".format(arg, val))
 146
 147     kp0 = subprocess.Popen(keep_cmd)
 148     with open("{}/keep{}.pid".format(TEST_TMPDIR, n), 'w') as f:
 149         f.write(str(kp0.pid))
 150
 151     with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
 152         f.write(keep0)
 153
 154 def run_keep(blob_signing_key=None, enforce_permissions=False):
 155     stop_keep()
 156
 157     if not os.path.exists(TEST_TMPDIR):
 158         os.mkdir(TEST_TMPDIR)
 159
 160     keep_args = {}
 161     if blob_signing_key:
 162         with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
 163             keep_args['--permission-key-file'] = f.name
 164             f.write(blob_signing_key)
 165     if enforce_permissions:
 166         keep_args['--enforce-permissions'] = 'true'
 167
 168     _start_keep(0, keep_args)
 169     _start_keep(1, keep_args)
 170
 171     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
 172     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 173
 174     authorize_with("admin")
 175     api = arvados.api('v1', cache=False)
 176     for d in api.keep_services().list().execute()['items']:
 177         api.keep_services().delete(uuid=d['uuid']).execute()
 178     for d in api.keep_disks().list().execute()['items']:
 179         api.keep_disks().delete(uuid=d['uuid']).execute()
 180
 181     s1 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25107, "service_type": "disk"} }).execute()
 182     s2 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25108, "service_type": "disk"} }).execute()
 183     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute()
 184     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
 185
 186 def _stop_keep(n):
 187     kill_server_pid("{}/keep{}.pid".format(TEST_TMPDIR, n), 0)
 188     if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
 189         with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
 190             shutil.rmtree(r.read(), True)
 191         os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
 192     if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
 193         os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
 194
 195 def stop_keep():
 196     _stop_keep(0)
 197     _stop_keep(1)
 198
 199 def run_keep_proxy(auth):
 200     stop_keep_proxy()
 201
 202     if not os.path.exists(TEST_TMPDIR):
 203         os.mkdir(TEST_TMPDIR)
 204
 205     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
 206     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 207     os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
 208
 209     kp0 = subprocess.Popen(["keepproxy",
 210                             "-pid={}/keepproxy.pid".format(TEST_TMPDIR),
 211                             "-listen=:{}".format(25101)])
 212
 213     authorize_with("admin")
 214     api = arvados.api('v1', cache=False)
 215     api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25101, "service_type": "proxy"} }).execute()
 216
 217     os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
 218
 219 def stop_keep_proxy():
 220     kill_server_pid(os.path.join(TEST_TMPDIR, "keepproxy.pid"), 0)
 221
 222 def fixture(fix):
 223     '''load a fixture yaml file'''
 224     with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
 225                            fix + ".yml")) as f:
 226         yaml_file = f.read()
 227         try:
 228           trim_index = yaml_file.index("# Test Helper trims the rest of the file")
 229           yaml_file = yaml_file[0:trim_index]
 230         except ValueError:
 231           pass
 232         return yaml.load(yaml_file)
 233
 234 def authorize_with(token):
 235     '''token is the symbolic name of the token from the api_client_authorizations fixture'''
 236     arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
 237     arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
 238     arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
 239
 240 class TestCaseWithServers(unittest.TestCase):
 241     """TestCase to start and stop supporting Arvados servers.
 242
 243     Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
 244     class variables as a dictionary of keyword arguments.  If you do,
 245     setUpClass will start the corresponding servers by passing these
 246     keyword arguments to the run, run_keep, and/or run_keep_server
 247     functions, respectively.  It will also set Arvados environment
 248     variables to point to these servers appropriately.  If you don't
 249     run a Keep or Keep proxy server, setUpClass will set up a
 250     temporary directory for Keep local storage, and set it as
 251     KEEP_LOCAL_STORE.
 252
 253     tearDownClass will stop any servers started, and restore the
 254     original environment.
 255     """
 256     MAIN_SERVER = None
 257     KEEP_SERVER = None
 258     KEEP_PROXY_SERVER = None
 259
 260     @staticmethod
 261     def _restore_dict(src, dest):
 262         for key in dest.keys():
 263             if key not in src:
 264                 del dest[key]
 265         dest.update(src)
 266
 267     @classmethod
 268     def setUpClass(cls):
 269         cls._orig_environ = os.environ.copy()
 270         cls._orig_config = arvados.config.settings().copy()
 271         cls._cleanup_funcs = []
 272         for server_kwargs, start_func, stop_func in (
 273               (cls.MAIN_SERVER, run, stop),
 274               (cls.KEEP_SERVER, run_keep, stop_keep),
 275               (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
 276             if server_kwargs is not None:
 277                 start_func(**server_kwargs)
 278                 cls._cleanup_funcs.append(stop_func)
 279         os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
 280         if cls.KEEP_PROXY_SERVER is None:
 281             os.environ.pop('ARVADOS_KEEP_PROXY', None)
 282         if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
 283             cls.local_store = tempfile.mkdtemp()
 284             os.environ['KEEP_LOCAL_STORE'] = cls.local_store
 285             cls._cleanup_funcs.append(
 286                 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
 287         else:
 288             os.environ.pop('KEEP_LOCAL_STORE', None)
 289         arvados.config.initialize()
 290
 291     @classmethod
 292     def tearDownClass(cls):
 293         for clean_func in cls._cleanup_funcs:
 294             clean_func()
 295         cls._restore_dict(cls._orig_environ, os.environ)
 296         cls._restore_dict(cls._orig_config, arvados.config.settings())
 297
 298
 299 if __name__ == "__main__":
 300     parser = argparse.ArgumentParser()
 301     parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
 302     parser.add_argument('--websockets', action='store_true', default=False)
 303     parser.add_argument('--reuse', action='store_true', default=False)
 304     parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')
 305     args = parser.parse_args()
 306
 307     if args.action == 'start':
 308         run(websockets=args.websockets, reuse_server=args.reuse)
 309         if args.auth is not None:
 310             authorize_with(args.auth)
 311             print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
 312             print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
 313             print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))
 314     elif args.action == 'stop':
 315         stop()
 316     elif args.action == 'start_keep':
 317         run_keep()
 318     elif args.action == 'stop_keep':
 319         stop_keep()
 320     elif args.action == 'start_keep_proxy':
 321         run_keep_proxy("admin")
 322     elif args.action == 'stop_keep_proxy':
 323         stop_keep_proxy()
 324     else:
 325         print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))