sdk/python/run_test_server.py

   1 import subprocess
   2 import time
   3 import os
   4 import signal
   5 import yaml
   6 import sys
   7 import argparse
   8 import arvados.config
   9 import arvados.api
  10 import shutil
  11 import tempfile
  12
  13 ARV_API_SERVER_DIR = '../../services/api'
  14 KEEP_SERVER_DIR = '../../services/keep'
  15 SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'
  16 WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'
  17
  18 def find_server_pid(PID_PATH, wait=10):
  19     now = time.time()
  20     timeout = now + wait
  21     good_pid = False
  22     while (not good_pid) and (now <= timeout):
  23         time.sleep(0.2)
  24         try:
  25             with open(PID_PATH, 'r') as f:
  26                 server_pid = int(f.read())
  27             good_pid = (os.kill(server_pid, 0) == None)
  28         except IOError:
  29             good_pid = False
  30         except OSError:
  31             good_pid = False
  32         now = time.time()
  33
  34     if not good_pid:
  35         return None
  36
  37     return server_pid
  38
  39 def kill_server_pid(PID_PATH, wait=10):
  40     try:
  41         now = time.time()
  42         timeout = now + wait
  43         with open(PID_PATH, 'r') as f:
  44             server_pid = int(f.read())
  45         while now <= timeout:
  46             os.kill(server_pid, signal.SIGTERM) == None
  47             os.getpgid(server_pid) # throw OSError if no such pid
  48             now = time.time()
  49             time.sleep(0.1)
  50     except IOError:
  51         good_pid = False
  52     except OSError:
  53         good_pid = False
  54
  55 def run(websockets=False, reuse_server=False):
  56     cwd = os.getcwd()
  57     os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
  58
  59     if websockets:
  60         pid_file = WEBSOCKETS_SERVER_PID_PATH
  61     else:
  62         pid_file = SERVER_PID_PATH
  63
  64     test_pid = find_server_pid(pid_file, 0)
  65
  66     if test_pid == None or not reuse_server:
  67         # do not try to run both server variants at once
  68         stop()
  69
  70         # delete cached discovery document
  71         shutil.rmtree(arvados.http_cache('discovery'))
  72
  73         # Setup database
  74         os.environ["RAILS_ENV"] = "test"
  75         subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])
  76         subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
  77         subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
  78
  79         if websockets:
  80             os.environ["ARVADOS_WEBSOCKETS"] = "true"
  81             subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
  82                              '-out', './self-signed.pem',
  83                              '-keyout', './self-signed.key',
  84                              '-days', '3650',
  85                              '-subj', '/CN=localhost'])
  86             subprocess.call(['bundle', 'exec',
  87                              'passenger', 'start', '-d', '-p3333',
  88                              '--pid-file',
  89                              os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH),
  90                              '--ssl',
  91                              '--ssl-certificate', 'self-signed.pem',
  92                              '--ssl-certificate-key', 'self-signed.key'])
  93             os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333"
  94         else:
  95             subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
  96                              '--pid',
  97                              os.path.join(os.getcwd(), SERVER_PID_PATH),
  98                              '-p3001'])
  99             os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 100
 101         pid = find_server_pid(SERVER_PID_PATH)
 102
 103     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 104     os.environ["ARVADOS_API_TOKEN"] = ""
 105     os.chdir(cwd)
 106
 107 def stop():
 108     cwd = os.getcwd()
 109     os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
 110
 111     kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
 112     kill_server_pid(SERVER_PID_PATH, 0)
 113
 114     try:
 115         os.unlink('self-signed.pem')
 116     except:
 117         pass
 118
 119     try:
 120         os.unlink('self-signed.key')
 121     except:
 122         pass
 123
 124     os.chdir(cwd)
 125
 126 def _start_keep(n):
 127     keep0 = tempfile.mkdtemp()
 128     kp0 = subprocess.Popen(["bin/keep", "-volumes={}".format(keep0), "-listen=:{}".format(25107+n)])
 129     with open("tmp/keep{}.pid".format(n), 'w') as f:
 130         f.write(str(kp0.pid))
 131     with open("tmp/keep{}.volume".format(n), 'w') as f:
 132         f.write(keep0)
 133
 134 def run_keep():
 135     stop_keep()
 136
 137     cwd = os.getcwd()
 138     os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
 139     if os.environ.get('GOPATH') == None:
 140         os.environ["GOPATH"] = os.getcwd()
 141     else:
 142         os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]
 143
 144     subprocess.call(["go", "install", "keep"])
 145
 146     if not os.path.exists("tmp"):
 147         os.mkdir("tmp")
 148
 149     _start_keep(0)
 150     _start_keep(1)
 151
 152
 153     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 154     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 155
 156     authorize_with("admin")
 157     api = arvados.api('v1', cache=False)
 158     for d in api.keep_services().list().execute()['items']:
 159         api.keep_services().delete(uuid=d['uuid']).execute()
 160     for d in api.keep_disks().list().execute()['items']:
 161         api.keep_disks().delete(uuid=d['uuid']).execute()
 162
 163     s1 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25107, "service_type": "disk"} }).execute()
 164     s2 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25108, "service_type": "disk"} }).execute()
 165     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute()
 166     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
 167
 168     os.chdir(cwd)
 169
 170 def _stop_keep(n):
 171     kill_server_pid("tmp/keep{}.pid".format(n), 0)
 172     if os.path.exists("tmp/keep{}.volume".format(n)):
 173         with open("tmp/keep{}.volume".format(n), 'r') as r:
 174             shutil.rmtree(r.read(), True)
 175
 176 def stop_keep():
 177     cwd = os.getcwd()
 178     os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
 179
 180     _stop_keep(0)
 181     _stop_keep(1)
 182
 183     shutil.rmtree("tmp", True)
 184
 185     os.chdir(cwd)
 186
 187 def fixture(fix):
 188     '''load a fixture yaml file'''
 189     with open(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR, "test", "fixtures",
 190                            fix + ".yml")) as f:
 191         return yaml.load(f.read())
 192
 193 def authorize_with(token):
 194     '''token is the symbolic name of the token from the api_client_authorizations fixture'''
 195     arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
 196     arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
 197     arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
 198
 199 if __name__ == "__main__":
 200     parser = argparse.ArgumentParser()
 201     parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
 202     parser.add_argument('--websockets', action='store_true', default=False)
 203     parser.add_argument('--reuse', action='store_true', default=False)
 204     parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')
 205     args = parser.parse_args()
 206
 207     if args.action == 'start':
 208         run(websockets=args.websockets, reuse_server=args.reuse)
 209         if args.auth != None:
 210             authorize_with(args.auth)
 211             print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
 212             print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
 213             print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))
 214     elif args.action == 'stop':
 215         stop()
 216     elif args.action == 'start_keep':
 217         run_keep()
 218     elif args.action == 'stop_keep':
 219         stop_keep()
 220     else:
 221         print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))