sdk/python/run_test_server.py

   1 import subprocess
   2 import time
   3 import os
   4 import signal
   5 import yaml
   6 import sys
   7 import argparse
   8 import arvados.config
   9 import arvados.api
  10 import shutil
  11 import tempfile
  12
  13 ARV_API_SERVER_DIR = '../../services/api'
  14 KEEP_SERVER_DIR = '../../services/keep'
  15 SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'
  16 WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'
  17
  18 def find_server_pid(PID_PATH, wait=10):
  19     now = time.time()
  20     timeout = now + wait
  21     good_pid = False
  22     while (not good_pid) and (now <= timeout):
  23         time.sleep(0.2)
  24         try:
  25             with open(PID_PATH, 'r') as f:
  26                 server_pid = int(f.read())
  27             good_pid = (os.kill(server_pid, 0) == None)
  28         except IOError:
  29             good_pid = False
  30         except OSError:
  31             good_pid = False
  32         now = time.time()
  33
  34     if not good_pid:
  35         return None
  36
  37     return server_pid
  38
  39 def kill_server_pid(PID_PATH, wait=10):
  40     try:
  41         now = time.time()
  42         timeout = now + wait
  43         with open(PID_PATH, 'r') as f:
  44             server_pid = int(f.read())
  45         while now <= timeout:
  46             os.kill(server_pid, signal.SIGTERM) == None
  47             os.getpgid(server_pid) # throw OSError if no such pid
  48             now = time.time()
  49             time.sleep(0.1)
  50     except IOError:
  51         good_pid = False
  52     except OSError:
  53         good_pid = False
  54
  55 def run(websockets=False, reuse_server=False):
  56     cwd = os.getcwd()
  57     os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
  58
  59     if websockets:
  60         pid_file = WEBSOCKETS_SERVER_PID_PATH
  61     else:
  62         pid_file = SERVER_PID_PATH
  63
  64     test_pid = find_server_pid(pid_file, 0)
  65
  66     if test_pid == None or not reuse_server:
  67         # do not try to run both server variants at once
  68         stop()
  69
  70         # delete cached discovery document
  71         shutil.rmtree(arvados.http_cache('discovery'))
  72
  73         # Setup database
  74         os.environ["RAILS_ENV"] = "test"
  75         subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])
  76         subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
  77         subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
  78
  79         if websockets:
  80             os.environ["ARVADOS_WEBSOCKETS"] = "true"
  81             subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
  82                              '-out', './self-signed.pem',
  83                              '-keyout', './self-signed.key',
  84                              '-days', '3650',
  85                              '-subj', '/CN=localhost'])
  86             subprocess.call(['bundle', 'exec',
  87                              'passenger', 'start', '-d', '-p3333',
  88                              '--pid-file',
  89                              os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH),
  90                              '--ssl',
  91                              '--ssl-certificate', 'self-signed.pem',
  92                              '--ssl-certificate-key', 'self-signed.key'])
  93             os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333"
  94         else:
  95             subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
  96                              '--pid',
  97                              os.path.join(os.getcwd(), SERVER_PID_PATH),
  98                              '-p3001'])
  99             os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 100
 101         pid = find_server_pid(SERVER_PID_PATH)
 102
 103     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 104     os.environ["ARVADOS_API_TOKEN"] = ""
 105     os.chdir(cwd)
 106
 107 def stop():
 108     cwd = os.getcwd()
 109     os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
 110
 111     kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
 112     kill_server_pid(SERVER_PID_PATH, 0)
 113
 114     try:
 115         os.unlink('self-signed.pem')
 116     except:
 117         pass
 118
 119     try:
 120         os.unlink('self-signed.key')
 121     except:
 122         pass
 123
 124     os.chdir(cwd)
 125
 126 def _start_keep(n, keep_args):
 127     keep0 = tempfile.mkdtemp()
 128     keep_cmd = ["bin/keep",
 129                 "-volumes={}".format(keep0),
 130                 "-listen=:{}".format(25107+n)]
 131
 132     for arg, val in keep_args.iteritems():
 133         keep_cmd.append("{}={}".format(arg, val))
 134
 135     kp0 = subprocess.Popen(keep_cmd)
 136     with open("tmp/keep{}.pid".format(n), 'w') as f:
 137         f.write(str(kp0.pid))
 138     with open("tmp/keep{}.volume".format(n), 'w') as f:
 139         f.write(keep0)
 140
 141 def run_keep(blob_signing_key=None, enforce_permissions=False):
 142     stop_keep()
 143
 144     cwd = os.getcwd()
 145     os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
 146     if os.environ.get('GOPATH') == None:
 147         os.environ["GOPATH"] = os.getcwd()
 148     else:
 149         os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]
 150
 151     subprocess.call(["go", "install", "keep"])
 152
 153     if not os.path.exists("tmp"):
 154         os.mkdir("tmp")
 155
 156     keep_args = {}
 157     if blob_signing_key:
 158         with open("tmp/keep.blob_signing_key", "w") as f:
 159             f.write(blob_signing_key)
 160         keep_args['--permission-key-file'] = 'tmp/keep.blob_signing_key'
 161     if enforce_permissions:
 162         keep_args['--enforce-permissions'] = 'true'
 163
 164     _start_keep(0, keep_args)
 165     _start_keep(1, keep_args)
 166
 167
 168     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 169     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 170
 171     authorize_with("admin")
 172     api = arvados.api('v1', cache=False)
 173     for d in api.keep_services().list().execute()['items']:
 174         api.keep_services().delete(uuid=d['uuid']).execute()
 175     for d in api.keep_disks().list().execute()['items']:
 176         api.keep_disks().delete(uuid=d['uuid']).execute()
 177
 178     s1 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25107, "service_type": "disk"} }).execute()
 179     s2 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25108, "service_type": "disk"} }).execute()
 180     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute()
 181     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
 182
 183     os.chdir(cwd)
 184
 185 def _stop_keep(n):
 186     kill_server_pid("tmp/keep{}.pid".format(n), 0)
 187     if os.path.exists("tmp/keep{}.volume".format(n)):
 188         with open("tmp/keep{}.volume".format(n), 'r') as r:
 189             shutil.rmtree(r.read(), True)
 190     if os.path.exists("tmp/keep.blob_signing_key"):
 191         os.remove("tmp/keep.blob_signing_key")
 192
 193 def stop_keep():
 194     cwd = os.getcwd()
 195     os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
 196
 197     _stop_keep(0)
 198     _stop_keep(1)
 199
 200     shutil.rmtree("tmp", True)
 201
 202     os.chdir(cwd)
 203
 204 def fixture(fix):
 205     '''load a fixture yaml file'''
 206     with open(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR, "test", "fixtures",
 207                            fix + ".yml")) as f:
 208         return yaml.load(f.read())
 209
 210 def authorize_with(token):
 211     '''token is the symbolic name of the token from the api_client_authorizations fixture'''
 212     arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
 213     arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
 214     arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
 215
 216 if __name__ == "__main__":
 217     parser = argparse.ArgumentParser()
 218     parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
 219     parser.add_argument('--websockets', action='store_true', default=False)
 220     parser.add_argument('--reuse', action='store_true', default=False)
 221     parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')
 222     args = parser.parse_args()
 223
 224     if args.action == 'start':
 225         run(websockets=args.websockets, reuse_server=args.reuse)
 226         if args.auth != None:
 227             authorize_with(args.auth)
 228             print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
 229             print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
 230             print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))
 231     elif args.action == 'stop':
 232         stop()
 233     elif args.action == 'start_keep':
 234         run_keep()
 235     elif args.action == 'stop_keep':
 236         stop_keep()
 237     else:
 238         print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))