sdk/python/tests/run_test_server.py

   1 #!/usr/bin/env python
   2
   3 import argparse
   4 import os
   5 import shutil
   6 import signal
   7 import subprocess
   8 import sys
   9 import tempfile
  10 import time
  11 import yaml
  12
  13 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
  14 if __name__ == '__main__' and os.path.exists(
  15       os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
  16     # We're being launched to support another test suite.
  17     # Add the Python SDK source to the library path.
  18     sys.path.insert(1, os.path.dirname(MY_DIRNAME))
  19
  20 import arvados.api
  21 import arvados.config
  22
  23 ARV_API_SERVER_DIR = '../../../services/api'
  24 KEEP_SERVER_DIR = '../../../services/keep'
  25 SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'
  26 WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'
  27
  28 def find_server_pid(PID_PATH, wait=10):
  29     now = time.time()
  30     timeout = now + wait
  31     good_pid = False
  32     while (not good_pid) and (now <= timeout):
  33         time.sleep(0.2)
  34         try:
  35             with open(PID_PATH, 'r') as f:
  36                 server_pid = int(f.read())
  37             good_pid = (os.kill(server_pid, 0) == None)
  38         except IOError:
  39             good_pid = False
  40         except OSError:
  41             good_pid = False
  42         now = time.time()
  43
  44     if not good_pid:
  45         return None
  46
  47     return server_pid
  48
  49 def kill_server_pid(PID_PATH, wait=10):
  50     try:
  51         now = time.time()
  52         timeout = now + wait
  53         with open(PID_PATH, 'r') as f:
  54             server_pid = int(f.read())
  55         while now <= timeout:
  56             os.kill(server_pid, signal.SIGTERM) == None
  57             os.getpgid(server_pid) # throw OSError if no such pid
  58             now = time.time()
  59             time.sleep(0.1)
  60     except IOError:
  61         good_pid = False
  62     except OSError:
  63         good_pid = False
  64
  65 def run(websockets=False, reuse_server=False):
  66     cwd = os.getcwd()
  67     os.chdir(os.path.join(MY_DIRNAME, ARV_API_SERVER_DIR))
  68
  69     if websockets:
  70         pid_file = WEBSOCKETS_SERVER_PID_PATH
  71     else:
  72         pid_file = SERVER_PID_PATH
  73
  74     test_pid = find_server_pid(pid_file, 0)
  75
  76     if test_pid == None or not reuse_server:
  77         # do not try to run both server variants at once
  78         stop()
  79
  80         # delete cached discovery document
  81         shutil.rmtree(arvados.http_cache('discovery'))
  82
  83         # Setup database
  84         os.environ["RAILS_ENV"] = "test"
  85         subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])
  86         subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
  87         subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
  88
  89         if websockets:
  90             os.environ["ARVADOS_WEBSOCKETS"] = "true"
  91             subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
  92                              '-out', './self-signed.pem',
  93                              '-keyout', './self-signed.key',
  94                              '-days', '3650',
  95                              '-subj', '/CN=localhost'])
  96             subprocess.call(['bundle', 'exec',
  97                              'passenger', 'start', '-d', '-p3333',
  98                              '--pid-file',
  99                              os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH),
 100                              '--ssl',
 101                              '--ssl-certificate', 'self-signed.pem',
 102                              '--ssl-certificate-key', 'self-signed.key'])
 103             os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333"
 104         else:
 105             subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
 106                              '--pid',
 107                              os.path.join(os.getcwd(), SERVER_PID_PATH),
 108                              '-p3001'])
 109             os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 110
 111         pid = find_server_pid(SERVER_PID_PATH)
 112
 113     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 114     os.environ["ARVADOS_API_TOKEN"] = ""
 115     os.chdir(cwd)
 116
 117 def stop():
 118     cwd = os.getcwd()
 119     os.chdir(os.path.join(MY_DIRNAME, ARV_API_SERVER_DIR))
 120
 121     kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
 122     kill_server_pid(SERVER_PID_PATH, 0)
 123
 124     try:
 125         os.unlink('self-signed.pem')
 126     except:
 127         pass
 128
 129     try:
 130         os.unlink('self-signed.key')
 131     except:
 132         pass
 133
 134     os.chdir(cwd)
 135
 136 def _start_keep(n, keep_args):
 137     keep0 = tempfile.mkdtemp()
 138     keep_cmd = ["bin/keep",
 139                 "-volumes={}".format(keep0),
 140                 "-listen=:{}".format(25107+n),
 141                 "-pid={}".format("tmp/keep{}.pid".format(n))]
 142
 143     for arg, val in keep_args.iteritems():
 144         keep_cmd.append("{}={}".format(arg, val))
 145
 146     kp0 = subprocess.Popen(keep_cmd)
 147     with open("tmp/keep{}.pid".format(n), 'w') as f:
 148         f.write(str(kp0.pid))
 149
 150     with open("tmp/keep{}.volume".format(n), 'w') as f:
 151         f.write(keep0)
 152
 153 def run_keep(blob_signing_key=None, enforce_permissions=False):
 154     stop_keep()
 155
 156     cwd = os.getcwd()
 157     os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR))
 158     if os.environ.get('GOPATH') == None:
 159         os.environ["GOPATH"] = os.getcwd()
 160     else:
 161         os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]
 162
 163     subprocess.call(["./go.sh", "install", "keep"])
 164
 165     if not os.path.exists("tmp"):
 166         os.mkdir("tmp")
 167
 168     keep_args = {}
 169     if blob_signing_key:
 170         with open("tmp/keep.blob_signing_key", "w") as f:
 171             f.write(blob_signing_key)
 172         keep_args['--permission-key-file'] = 'tmp/keep.blob_signing_key'
 173     if enforce_permissions:
 174         keep_args['--enforce-permissions'] = 'true'
 175
 176     _start_keep(0, keep_args)
 177     _start_keep(1, keep_args)
 178
 179     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 180     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 181
 182     authorize_with("admin")
 183     api = arvados.api('v1', cache=False)
 184     for d in api.keep_services().list().execute()['items']:
 185         api.keep_services().delete(uuid=d['uuid']).execute()
 186     for d in api.keep_disks().list().execute()['items']:
 187         api.keep_disks().delete(uuid=d['uuid']).execute()
 188
 189     s1 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25107, "service_type": "disk"} }).execute()
 190     s2 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25108, "service_type": "disk"} }).execute()
 191     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute()
 192     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
 193
 194     os.chdir(cwd)
 195
 196 def _stop_keep(n):
 197     kill_server_pid("tmp/keep{}.pid".format(n), 0)
 198     if os.path.exists("tmp/keep{}.volume".format(n)):
 199         with open("tmp/keep{}.volume".format(n), 'r') as r:
 200             shutil.rmtree(r.read(), True)
 201         os.unlink("tmp/keep{}.volume".format(n))
 202     if os.path.exists("tmp/keep.blob_signing_key"):
 203         os.remove("tmp/keep.blob_signing_key")
 204
 205 def stop_keep():
 206     cwd = os.getcwd()
 207     os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR))
 208
 209     _stop_keep(0)
 210     _stop_keep(1)
 211
 212     os.chdir(cwd)
 213
 214 def run_keep_proxy(auth):
 215     stop_keep_proxy()
 216
 217     cwd = os.getcwd()
 218     os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR))
 219     if os.environ.get('GOPATH') == None:
 220         os.environ["GOPATH"] = os.getcwd()
 221     else:
 222         os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]
 223
 224     subprocess.call(["./go.sh", "install", "arvados.org/keepproxy"])
 225
 226     if not os.path.exists("tmp"):
 227         os.mkdir("tmp")
 228
 229     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 230     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 231     os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
 232
 233     kp0 = subprocess.Popen(["bin/keepproxy", "-pid=tmp/keepproxy.pid", "-listen=:{}".format(25101)])
 234
 235     authorize_with("admin")
 236     api = arvados.api('v1', cache=False)
 237     api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25101, "service_type": "proxy"} }).execute()
 238
 239     arvados.config.settings()["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
 240
 241     os.chdir(cwd)
 242
 243 def stop_keep_proxy():
 244     cwd = os.getcwd()
 245     os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR))
 246     kill_server_pid("tmp/keepproxy.pid", 0)
 247     os.chdir(cwd)
 248
 249 def fixture(fix):
 250     '''load a fixture yaml file'''
 251     with open(os.path.join(MY_DIRNAME, ARV_API_SERVER_DIR, "test", "fixtures",
 252                            fix + ".yml")) as f:
 253         return yaml.load(f.read())
 254
 255 def authorize_with(token):
 256     '''token is the symbolic name of the token from the api_client_authorizations fixture'''
 257     arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
 258     arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
 259     arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
 260
 261 if __name__ == "__main__":
 262     parser = argparse.ArgumentParser()
 263     parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
 264     parser.add_argument('--websockets', action='store_true', default=False)
 265     parser.add_argument('--reuse', action='store_true', default=False)
 266     parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')
 267     args = parser.parse_args()
 268
 269     if args.action == 'start':
 270         run(websockets=args.websockets, reuse_server=args.reuse)
 271         if args.auth != None:
 272             authorize_with(args.auth)
 273             print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
 274             print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
 275             print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))
 276     elif args.action == 'stop':
 277         stop()
 278     elif args.action == 'start_keep':
 279         run_keep()
 280     elif args.action == 'stop_keep':
 281         stop_keep()
 282     elif args.action == 'start_keep_proxy':
 283         run_keep_proxy("admin")
 284     elif args.action == 'stop_keep_proxy':
 285         stop_keep_proxy()
 286     else:
 287         print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))