Merge branch 'master' into 4823-python-sdk-writable-collection-api
[arvados.git] / sdk / python / tests / run_test_server.py
1 #!/usr/bin/env python
2
3 import argparse
4 import os
5 import shutil
6 import signal
7 import subprocess
8 import sys
9 import tempfile
10 import time
11 import unittest
12 import yaml
13
14 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
15 if __name__ == '__main__' and os.path.exists(
16       os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
17     # We're being launched to support another test suite.
18     # Add the Python SDK source to the library path.
19     sys.path.insert(1, os.path.dirname(MY_DIRNAME))
20
21 import arvados.api
22 import arvados.config
23
24 SERVICES_SRC_DIR = os.path.join(MY_DIRNAME, '../../../services')
25 SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'
26 WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'
27 if 'GOPATH' in os.environ:
28     gopaths = os.environ['GOPATH'].split(':')
29     gobins = [os.path.join(path, 'bin') for path in gopaths]
30     os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
31
32 if os.path.isdir('tests'):
33     TEST_TMPDIR = 'tests/tmp'
34 else:
35     TEST_TMPDIR = 'tmp'
36
37 def find_server_pid(PID_PATH, wait=10):
38     now = time.time()
39     timeout = now + wait
40     good_pid = False
41     while (not good_pid) and (now <= timeout):
42         time.sleep(0.2)
43         try:
44             with open(PID_PATH, 'r') as f:
45                 server_pid = int(f.read())
46             good_pid = (os.kill(server_pid, 0) is None)
47         except IOError:
48             good_pid = False
49         except OSError:
50             good_pid = False
51         now = time.time()
52
53     if not good_pid:
54         return None
55
56     return server_pid
57
58 def kill_server_pid(PID_PATH, wait=10):
59     try:
60         now = time.time()
61         timeout = now + wait
62         with open(PID_PATH, 'r') as f:
63             server_pid = int(f.read())
64         while now <= timeout:
65             os.kill(server_pid, signal.SIGTERM)
66             os.getpgid(server_pid) # throw OSError if no such pid
67             now = time.time()
68             time.sleep(0.1)
69     except IOError:
70         good_pid = False
71     except OSError:
72         good_pid = False
73
74 def run(websockets=False, reuse_server=False):
75     cwd = os.getcwd()
76     os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
77
78     if websockets:
79         pid_file = WEBSOCKETS_SERVER_PID_PATH
80     else:
81         pid_file = SERVER_PID_PATH
82
83     test_pid = find_server_pid(pid_file, 0)
84
85     if test_pid is None or not reuse_server:
86         # do not try to run both server variants at once
87         stop()
88
89         # delete cached discovery document
90         shutil.rmtree(arvados.http_cache('discovery'))
91
92         # Setup database
93         os.environ["RAILS_ENV"] = "test"
94         subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])
95         subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
96         subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
97
98         subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
99                          '--pid',
100                          os.path.join(os.getcwd(), SERVER_PID_PATH),
101                          '-p3000'])
102         os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
103
104         if websockets:
105             os.environ["ARVADOS_WEBSOCKETS"] = "ws-only"
106             subprocess.call(['bundle', 'exec',
107                              'passenger', 'start', '-d', '-p3333',
108                              '--pid-file',
109                              os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH)
110                          ])
111
112         pid = find_server_pid(SERVER_PID_PATH)
113
114     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
115     os.environ["ARVADOS_API_TOKEN"] = ""
116     os.chdir(cwd)
117
118 def stop():
119     cwd = os.getcwd()
120     os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
121
122     kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
123     kill_server_pid(SERVER_PID_PATH, 0)
124
125     try:
126         os.unlink('self-signed.pem')
127     except:
128         pass
129
130     try:
131         os.unlink('self-signed.key')
132     except:
133         pass
134
135     os.chdir(cwd)
136
137 def _start_keep(n, keep_args):
138     keep0 = tempfile.mkdtemp()
139     keep_cmd = ["keepstore",
140                 "-volumes={}".format(keep0),
141                 "-listen=:{}".format(25107+n),
142                 "-pid={}".format("{}/keep{}.pid".format(TEST_TMPDIR, n))]
143
144     for arg, val in keep_args.iteritems():
145         keep_cmd.append("{}={}".format(arg, val))
146
147     kp0 = subprocess.Popen(keep_cmd)
148     with open("{}/keep{}.pid".format(TEST_TMPDIR, n), 'w') as f:
149         f.write(str(kp0.pid))
150
151     with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
152         f.write(keep0)
153
154 def run_keep(blob_signing_key=None, enforce_permissions=False):
155     stop_keep()
156
157     if not os.path.exists(TEST_TMPDIR):
158         os.mkdir(TEST_TMPDIR)
159
160     keep_args = {}
161     if blob_signing_key:
162         with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
163             keep_args['--permission-key-file'] = f.name
164             f.write(blob_signing_key)
165     if enforce_permissions:
166         keep_args['--enforce-permissions'] = 'true'
167
168     _start_keep(0, keep_args)
169     _start_keep(1, keep_args)
170
171     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
172     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
173
174     authorize_with("admin")
175     api = arvados.api('v1', cache=False)
176     for d in api.keep_services().list().execute()['items']:
177         api.keep_services().delete(uuid=d['uuid']).execute()
178     for d in api.keep_disks().list().execute()['items']:
179         api.keep_disks().delete(uuid=d['uuid']).execute()
180
181     s1 = api.keep_services().create(body={"keep_service": {
182                 "uuid": "zzzzz-bi6l4-5bo5n1iekkjyz6b",
183                 "service_host": "localhost",
184                 "service_port": 25107,
185                 "service_type": "disk"
186                 }}).execute()
187     s2 = api.keep_services().create(body={"keep_service": {
188                 "uuid": "zzzzz-bi6l4-2nz60e0ksj7vr3s",
189                 "service_host": "localhost",
190                 "service_port": 25108,
191                 "service_type": "disk"
192                 }}).execute()
193     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute()
194     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
195
196 def _stop_keep(n):
197     kill_server_pid("{}/keep{}.pid".format(TEST_TMPDIR, n), 0)
198     if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
199         with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
200             shutil.rmtree(r.read(), True)
201         os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
202     if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
203         os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
204
205 def stop_keep():
206     _stop_keep(0)
207     _stop_keep(1)
208
209 def run_keep_proxy(auth):
210     stop_keep_proxy()
211
212     if not os.path.exists(TEST_TMPDIR):
213         os.mkdir(TEST_TMPDIR)
214
215     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
216     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
217     os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
218
219     kp0 = subprocess.Popen(["keepproxy",
220                             "-pid={}/keepproxy.pid".format(TEST_TMPDIR),
221                             "-listen=:{}".format(25101)])
222
223     authorize_with("admin")
224     api = arvados.api('v1', cache=False)
225     api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25101, "service_type": "proxy"} }).execute()
226
227     os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
228
229 def stop_keep_proxy():
230     kill_server_pid(os.path.join(TEST_TMPDIR, "keepproxy.pid"), 0)
231
232 def fixture(fix):
233     '''load a fixture yaml file'''
234     with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
235                            fix + ".yml")) as f:
236         yaml_file = f.read()
237         try:
238           trim_index = yaml_file.index("# Test Helper trims the rest of the file")
239           yaml_file = yaml_file[0:trim_index]
240         except ValueError:
241           pass
242         return yaml.load(yaml_file)
243
244 def authorize_with(token):
245     '''token is the symbolic name of the token from the api_client_authorizations fixture'''
246     arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
247     arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
248     arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
249
250 class TestCaseWithServers(unittest.TestCase):
251     """TestCase to start and stop supporting Arvados servers.
252
253     Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
254     class variables as a dictionary of keyword arguments.  If you do,
255     setUpClass will start the corresponding servers by passing these
256     keyword arguments to the run, run_keep, and/or run_keep_server
257     functions, respectively.  It will also set Arvados environment
258     variables to point to these servers appropriately.  If you don't
259     run a Keep or Keep proxy server, setUpClass will set up a
260     temporary directory for Keep local storage, and set it as
261     KEEP_LOCAL_STORE.
262
263     tearDownClass will stop any servers started, and restore the
264     original environment.
265     """
266     MAIN_SERVER = None
267     KEEP_SERVER = None
268     KEEP_PROXY_SERVER = None
269
270     @staticmethod
271     def _restore_dict(src, dest):
272         for key in dest.keys():
273             if key not in src:
274                 del dest[key]
275         dest.update(src)
276
277     @classmethod
278     def setUpClass(cls):
279         cls._orig_environ = os.environ.copy()
280         cls._orig_config = arvados.config.settings().copy()
281         cls._cleanup_funcs = []
282         for server_kwargs, start_func, stop_func in (
283               (cls.MAIN_SERVER, run, stop),
284               (cls.KEEP_SERVER, run_keep, stop_keep),
285               (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
286             if server_kwargs is not None:
287                 start_func(**server_kwargs)
288                 cls._cleanup_funcs.append(stop_func)
289         os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
290         if cls.KEEP_PROXY_SERVER is None:
291             os.environ.pop('ARVADOS_KEEP_PROXY', None)
292         if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
293             cls.local_store = tempfile.mkdtemp()
294             os.environ['KEEP_LOCAL_STORE'] = cls.local_store
295             cls._cleanup_funcs.append(
296                 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
297         else:
298             os.environ.pop('KEEP_LOCAL_STORE', None)
299         arvados.config.initialize()
300
301     @classmethod
302     def tearDownClass(cls):
303         for clean_func in cls._cleanup_funcs:
304             clean_func()
305         cls._restore_dict(cls._orig_environ, os.environ)
306         cls._restore_dict(cls._orig_config, arvados.config.settings())
307
308
309 if __name__ == "__main__":
310     parser = argparse.ArgumentParser()
311     parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
312     parser.add_argument('--websockets', action='store_true', default=False)
313     parser.add_argument('--reuse', action='store_true', default=False)
314     parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')
315     args = parser.parse_args()
316
317     if args.action == 'start':
318         run(websockets=args.websockets, reuse_server=args.reuse)
319         if args.auth is not None:
320             authorize_with(args.auth)
321             print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
322             print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
323             print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))
324     elif args.action == 'stop':
325         stop()
326     elif args.action == 'start_keep':
327         run_keep()
328     elif args.action == 'stop_keep':
329         stop_keep()
330     elif args.action == 'start_keep_proxy':
331         run_keep_proxy("admin")
332     elif args.action == 'stop_keep_proxy':
333         stop_keep_proxy()
334     else:
335         print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))