sdk/python/tests/run_test_server.py

   1 #!/usr/bin/env python
   2
   3 import argparse
   4 import os
   5 import shutil
   6 import signal
   7 import subprocess
   8 import sys
   9 import tempfile
  10 import time
  11 import yaml
  12
  13 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
  14 if __name__ == '__main__' and os.path.exists(
  15       os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
  16     # We're being launched to support another test suite.
  17     # Add the Python SDK source to the library path.
  18     sys.path.insert(1, os.path.dirname(MY_DIRNAME))
  19
  20 import arvados.api
  21 import arvados.config
  22
  23 SERVICES_SRC_DIR = os.path.join(MY_DIRNAME, '../../../services')
  24 SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'
  25 WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'
  26 if 'GOPATH' in os.environ:
  27     gopaths = os.environ['GOPATH'].split(':')
  28     gobins = [os.path.join(path, 'bin') for path in gopaths]
  29     os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
  30
  31 def find_server_pid(PID_PATH, wait=10):
  32     now = time.time()
  33     timeout = now + wait
  34     good_pid = False
  35     while (not good_pid) and (now <= timeout):
  36         time.sleep(0.2)
  37         try:
  38             with open(PID_PATH, 'r') as f:
  39                 server_pid = int(f.read())
  40             good_pid = (os.kill(server_pid, 0) == None)
  41         except IOError:
  42             good_pid = False
  43         except OSError:
  44             good_pid = False
  45         now = time.time()
  46
  47     if not good_pid:
  48         return None
  49
  50     return server_pid
  51
  52 def kill_server_pid(PID_PATH, wait=10):
  53     try:
  54         now = time.time()
  55         timeout = now + wait
  56         with open(PID_PATH, 'r') as f:
  57             server_pid = int(f.read())
  58         while now <= timeout:
  59             os.kill(server_pid, signal.SIGTERM) == None
  60             os.getpgid(server_pid) # throw OSError if no such pid
  61             now = time.time()
  62             time.sleep(0.1)
  63     except IOError:
  64         good_pid = False
  65     except OSError:
  66         good_pid = False
  67
  68 def run(websockets=False, reuse_server=False):
  69     cwd = os.getcwd()
  70     os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
  71
  72     if websockets:
  73         pid_file = WEBSOCKETS_SERVER_PID_PATH
  74     else:
  75         pid_file = SERVER_PID_PATH
  76
  77     test_pid = find_server_pid(pid_file, 0)
  78
  79     if test_pid == None or not reuse_server:
  80         # do not try to run both server variants at once
  81         stop()
  82
  83         # delete cached discovery document
  84         shutil.rmtree(arvados.http_cache('discovery'))
  85
  86         # Setup database
  87         os.environ["RAILS_ENV"] = "test"
  88         subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])
  89         subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
  90         subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
  91
  92         if websockets:
  93             os.environ["ARVADOS_WEBSOCKETS"] = "true"
  94             subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
  95                              '-out', './self-signed.pem',
  96                              '-keyout', './self-signed.key',
  97                              '-days', '3650',
  98                              '-subj', '/CN=localhost'])
  99             subprocess.call(['bundle', 'exec',
 100                              'passenger', 'start', '-d', '-p3333',
 101                              '--pid-file',
 102                              os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH),
 103                              '--ssl',
 104                              '--ssl-certificate', 'self-signed.pem',
 105                              '--ssl-certificate-key', 'self-signed.key'])
 106             os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333"
 107         else:
 108             subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
 109                              '--pid',
 110                              os.path.join(os.getcwd(), SERVER_PID_PATH),
 111                              '-p3001'])
 112             os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 113
 114         pid = find_server_pid(SERVER_PID_PATH)
 115
 116     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 117     os.environ["ARVADOS_API_TOKEN"] = ""
 118     os.chdir(cwd)
 119
 120 def stop():
 121     cwd = os.getcwd()
 122     os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
 123
 124     kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
 125     kill_server_pid(SERVER_PID_PATH, 0)
 126
 127     try:
 128         os.unlink('self-signed.pem')
 129     except:
 130         pass
 131
 132     try:
 133         os.unlink('self-signed.key')
 134     except:
 135         pass
 136
 137     os.chdir(cwd)
 138
 139 def _start_keep(n, keep_args):
 140     keep0 = tempfile.mkdtemp()
 141     keep_cmd = ["keepstore",
 142                 "-volumes={}".format(keep0),
 143                 "-listen=:{}".format(25107+n),
 144                 "-pid={}".format("tmp/keep{}.pid".format(n))]
 145
 146     for arg, val in keep_args.iteritems():
 147         keep_cmd.append("{}={}".format(arg, val))
 148
 149     kp0 = subprocess.Popen(keep_cmd)
 150     with open("tmp/keep{}.pid".format(n), 'w') as f:
 151         f.write(str(kp0.pid))
 152
 153     with open("tmp/keep{}.volume".format(n), 'w') as f:
 154         f.write(keep0)
 155
 156 def run_keep(blob_signing_key=None, enforce_permissions=False):
 157     stop_keep()
 158
 159     if not os.path.exists("tmp"):
 160         os.mkdir("tmp")
 161
 162     keep_args = {}
 163     if blob_signing_key:
 164         with open("tmp/keep.blob_signing_key", "w") as f:
 165             f.write(blob_signing_key)
 166         keep_args['--permission-key-file'] = 'tmp/keep.blob_signing_key'
 167     if enforce_permissions:
 168         keep_args['--enforce-permissions'] = 'true'
 169
 170     _start_keep(0, keep_args)
 171     _start_keep(1, keep_args)
 172
 173     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 174     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 175
 176     authorize_with("admin")
 177     api = arvados.api('v1', cache=False)
 178     for d in api.keep_services().list().execute()['items']:
 179         api.keep_services().delete(uuid=d['uuid']).execute()
 180     for d in api.keep_disks().list().execute()['items']:
 181         api.keep_disks().delete(uuid=d['uuid']).execute()
 182
 183     s1 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25107, "service_type": "disk"} }).execute()
 184     s2 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25108, "service_type": "disk"} }).execute()
 185     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute()
 186     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
 187
 188 def _stop_keep(n):
 189     kill_server_pid("tmp/keep{}.pid".format(n), 0)
 190     if os.path.exists("tmp/keep{}.volume".format(n)):
 191         with open("tmp/keep{}.volume".format(n), 'r') as r:
 192             shutil.rmtree(r.read(), True)
 193         os.unlink("tmp/keep{}.volume".format(n))
 194     if os.path.exists("tmp/keep.blob_signing_key"):
 195         os.remove("tmp/keep.blob_signing_key")
 196
 197 def stop_keep():
 198     _stop_keep(0)
 199     _stop_keep(1)
 200
 201 def run_keep_proxy(auth):
 202     stop_keep_proxy()
 203
 204     if not os.path.exists("tmp"):
 205         os.mkdir("tmp")
 206
 207     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 208     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 209     os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
 210
 211     kp0 = subprocess.Popen(["keepproxy",
 212                             "-pid=tmp/keepproxy.pid", "-listen=:{}".format(25101)])
 213
 214     authorize_with("admin")
 215     api = arvados.api('v1', cache=False)
 216     api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25101, "service_type": "proxy"} }).execute()
 217
 218     arvados.config.settings()["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
 219
 220 def stop_keep_proxy():
 221     kill_server_pid("tmp/keepproxy.pid", 0)
 222
 223 def fixture(fix):
 224     '''load a fixture yaml file'''
 225     with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
 226                            fix + ".yml")) as f:
 227         return yaml.load(f.read())
 228
 229 def authorize_with(token):
 230     '''token is the symbolic name of the token from the api_client_authorizations fixture'''
 231     arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
 232     arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
 233     arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
 234
 235 if __name__ == "__main__":
 236     parser = argparse.ArgumentParser()
 237     parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
 238     parser.add_argument('--websockets', action='store_true', default=False)
 239     parser.add_argument('--reuse', action='store_true', default=False)
 240     parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')
 241     args = parser.parse_args()
 242
 243     if args.action == 'start':
 244         run(websockets=args.websockets, reuse_server=args.reuse)
 245         if args.auth != None:
 246             authorize_with(args.auth)
 247             print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
 248             print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
 249             print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))
 250     elif args.action == 'stop':
 251         stop()
 252     elif args.action == 'start_keep':
 253         run_keep()
 254     elif args.action == 'stop_keep':
 255         stop_keep()
 256     elif args.action == 'start_keep_proxy':
 257         run_keep_proxy("admin")
 258     elif args.action == 'stop_keep_proxy':
 259         stop_keep_proxy()
 260     else:
 261         print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))