sdk/python/run_test_server.py

   1 import subprocess
   2 import time
   3 import os
   4 import signal
   5 import yaml
   6 import sys
   7 import argparse
   8 import arvados.config
   9 import arvados.api
  10 import shutil
  11 import tempfile
  12
  13 ARV_API_SERVER_DIR = '../../services/api'
  14 KEEP_SERVER_DIR = '../../services/keep'
  15 SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'
  16 WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'
  17
  18 def find_server_pid(PID_PATH, wait=10):
  19     now = time.time()
  20     timeout = now + wait
  21     good_pid = False
  22     while (not good_pid) and (now <= timeout):
  23         time.sleep(0.2)
  24         try:
  25             with open(PID_PATH, 'r') as f:
  26                 server_pid = int(f.read())
  27             good_pid = (os.kill(server_pid, 0) == None)
  28         except IOError:
  29             good_pid = False
  30         except OSError:
  31             good_pid = False
  32         now = time.time()
  33
  34     if not good_pid:
  35         return None
  36
  37     return server_pid
  38
  39 def kill_server_pid(PID_PATH, wait=10):
  40     try:
  41         now = time.time()
  42         timeout = now + wait
  43         with open(PID_PATH, 'r') as f:
  44             server_pid = int(f.read())
  45         while now <= timeout:
  46             os.kill(server_pid, signal.SIGTERM) == None
  47             os.getpgid(server_pid) # throw OSError if no such pid
  48             now = time.time()
  49             time.sleep(0.1)
  50     except IOError:
  51         good_pid = False
  52     except OSError:
  53         good_pid = False
  54
  55 def run(websockets=False, reuse_server=False):
  56     cwd = os.getcwd()
  57     os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
  58
  59     if websockets:
  60         pid_file = WEBSOCKETS_SERVER_PID_PATH
  61     else:
  62         pid_file = SERVER_PID_PATH
  63
  64     test_pid = find_server_pid(pid_file, 0)
  65
  66     if test_pid == None or not reuse_server:
  67         # do not try to run both server variants at once
  68         stop()
  69
  70         # delete cached discovery document
  71         shutil.rmtree(arvados.http_cache('discovery'))
  72
  73         # Setup database
  74         os.environ["RAILS_ENV"] = "test"
  75         subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])
  76         subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
  77         subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
  78
  79         if websockets:
  80             os.environ["ARVADOS_WEBSOCKETS"] = "true"
  81             subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
  82                              '-out', './self-signed.pem',
  83                              '-keyout', './self-signed.key',
  84                              '-days', '3650',
  85                              '-subj', '/CN=localhost'])
  86             subprocess.call(['bundle', 'exec',
  87                              'passenger', 'start', '-d', '-p3333',
  88                              '--pid-file',
  89                              os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH),
  90                              '--ssl',
  91                              '--ssl-certificate', 'self-signed.pem',
  92                              '--ssl-certificate-key', 'self-signed.key'])
  93             os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333"
  94         else:
  95             subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
  96                              '--pid',
  97                              os.path.join(os.getcwd(), SERVER_PID_PATH),
  98                              '-p3001'])
  99             os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 100
 101         pid = find_server_pid(SERVER_PID_PATH)
 102
 103     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 104     os.environ["ARVADOS_API_TOKEN"] = ""
 105     os.chdir(cwd)
 106
 107 def stop():
 108     cwd = os.getcwd()
 109     os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
 110
 111     kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
 112     kill_server_pid(SERVER_PID_PATH, 0)
 113
 114     try:
 115         os.unlink('self-signed.pem')
 116     except:
 117         pass
 118
 119     try:
 120         os.unlink('self-signed.key')
 121     except:
 122         pass
 123
 124     os.chdir(cwd)
 125
 126 def _start_keep(n, keep_args):
 127     keep0 = tempfile.mkdtemp()
 128     keep_cmd = ["bin/keep",
 129                 "-volumes={}".format(keep0),
 130                 "-listen=:{}".format(25107+n),
 131                 "-pid={}".format("tmp/keep{}.pid".format(n))]
 132
 133     for arg, val in keep_args.iteritems():
 134         keep_cmd.append("{}={}".format(arg, val))
 135
 136     kp0 = subprocess.Popen(keep_cmd)
 137     with open("tmp/keep{}.pid".format(n), 'w') as f:
 138         f.write(str(kp0.pid))
 139
 140     with open("tmp/keep{}.volume".format(n), 'w') as f:
 141         f.write(keep0)
 142
 143 def run_keep(blob_signing_key=None, enforce_permissions=False):
 144     stop_keep()
 145
 146     cwd = os.getcwd()
 147     os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
 148     if os.environ.get('GOPATH') == None:
 149         os.environ["GOPATH"] = os.getcwd()
 150     else:
 151         os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]
 152
 153     subprocess.call(["./go.sh", "install", "keep"])
 154
 155     if not os.path.exists("tmp"):
 156         os.mkdir("tmp")
 157
 158     keep_args = {}
 159     if blob_signing_key:
 160         with open("tmp/keep.blob_signing_key", "w") as f:
 161             f.write(blob_signing_key)
 162         keep_args['--permission-key-file'] = 'tmp/keep.blob_signing_key'
 163     if enforce_permissions:
 164         keep_args['--enforce-permissions'] = 'true'
 165
 166     _start_keep(0, keep_args)
 167     _start_keep(1, keep_args)
 168
 169     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 170     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 171
 172     authorize_with("admin")
 173     api = arvados.api('v1', cache=False)
 174     for d in api.keep_services().list().execute()['items']:
 175         api.keep_services().delete(uuid=d['uuid']).execute()
 176     for d in api.keep_disks().list().execute()['items']:
 177         api.keep_disks().delete(uuid=d['uuid']).execute()
 178
 179     s1 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25107, "service_type": "disk"} }).execute()
 180     s2 = api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25108, "service_type": "disk"} }).execute()
 181     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute()
 182     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
 183
 184     os.chdir(cwd)
 185
 186 def _stop_keep(n):
 187     kill_server_pid("tmp/keep{}.pid".format(n), 0)
 188     if os.path.exists("tmp/keep{}.volume".format(n)):
 189         with open("tmp/keep{}.volume".format(n), 'r') as r:
 190             shutil.rmtree(r.read(), True)
 191         os.unlink("tmp/keep{}.volume".format(n))
 192     if os.path.exists("tmp/keep.blob_signing_key"):
 193         os.remove("tmp/keep.blob_signing_key")
 194
 195 def stop_keep():
 196     cwd = os.getcwd()
 197     os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
 198
 199     _stop_keep(0)
 200     _stop_keep(1)
 201
 202     os.chdir(cwd)
 203
 204 def run_keep_proxy(auth):
 205     stop_keep_proxy()
 206
 207     cwd = os.getcwd()
 208     os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
 209     if os.environ.get('GOPATH') == None:
 210         os.environ["GOPATH"] = os.getcwd()
 211     else:
 212         os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]
 213
 214     subprocess.call(["./go.sh", "install", "arvados.org/keepproxy"])
 215
 216     if not os.path.exists("tmp"):
 217         os.mkdir("tmp")
 218
 219     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
 220     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 221     os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
 222
 223     kp0 = subprocess.Popen(["bin/keepproxy", "-pid=tmp/keepproxy.pid", "-listen=:{}".format(25101)])
 224
 225     authorize_with("admin")
 226     api = arvados.api('v1', cache=False)
 227     api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25101, "service_type": "proxy"} }).execute()
 228
 229     arvados.config.settings()["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
 230
 231     os.chdir(cwd)
 232
 233 def stop_keep_proxy():
 234     cwd = os.getcwd()
 235     os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
 236     kill_server_pid("tmp/keepproxy.pid", 0)
 237     os.chdir(cwd)
 238
 239 def fixture(fix):
 240     '''load a fixture yaml file'''
 241     with open(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR, "test", "fixtures",
 242                            fix + ".yml")) as f:
 243         return yaml.load(f.read())
 244
 245 def authorize_with(token):
 246     '''token is the symbolic name of the token from the api_client_authorizations fixture'''
 247     arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
 248     arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
 249     arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
 250
 251 if __name__ == "__main__":
 252     parser = argparse.ArgumentParser()
 253     parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
 254     parser.add_argument('--websockets', action='store_true', default=False)
 255     parser.add_argument('--reuse', action='store_true', default=False)
 256     parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')
 257     args = parser.parse_args()
 258
 259     if args.action == 'start':
 260         run(websockets=args.websockets, reuse_server=args.reuse)
 261         if args.auth != None:
 262             authorize_with(args.auth)
 263             print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
 264             print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
 265             print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))
 266     elif args.action == 'stop':
 267         stop()
 268     elif args.action == 'start_keep':
 269         run_keep()
 270     elif args.action == 'stop_keep':
 271         stop_keep()
 272     elif args.action == 'start_keep_proxy':
 273         run_keep_proxy("admin")
 274     elif args.action == 'stop_keep_proxy':
 275         stop_keep_proxy()
 276     else:
 277         print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))