3 from __future__ import print_function
22 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
23 if __name__ == '__main__' and os.path.exists(
24 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
25 # We're being launched to support another test suite.
26 # Add the Python SDK source to the library path.
27 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
32 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
33 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
34 SERVER_PID_PATH = 'tmp/pids/test-server.pid'
35 if 'GOPATH' in os.environ:
36 gopaths = os.environ['GOPATH'].split(':')
37 gobins = [os.path.join(path, 'bin') for path in gopaths]
38 os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
40 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
41 if not os.path.exists(TEST_TMPDIR):
47 def find_server_pid(PID_PATH, wait=10):
51 while (not good_pid) and (now <= timeout):
54 with open(PID_PATH, 'r') as f:
55 server_pid = int(f.read())
56 good_pid = (os.kill(server_pid, 0) is None)
68 def kill_server_pid(pidfile, wait=10, passenger_root=False):
69 # Must re-import modules in order to work during atexit
76 # First try to shut down nicely
77 restore_cwd = os.getcwd()
78 os.chdir(passenger_root)
80 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
84 with open(pidfile, 'r') as f:
85 server_pid = int(f.read())
87 if not passenger_root or timeout - now < wait / 2:
88 # Half timeout has elapsed. Start sending SIGTERM
89 os.kill(server_pid, signal.SIGTERM)
90 # Raise OSError if process has disappeared
91 os.getpgid(server_pid)
99 def find_available_port():
100 """Return an IPv4 port number that is not in use right now.
102 We assume whoever needs to use the returned port is able to reuse
103 a recently used port without waiting for TIME_WAIT (see
104 SO_REUSEADDR / SO_REUSEPORT).
106 Some opportunity for races here, but it's better than choosing
107 something at random and not checking at all. If all of our servers
108 (hey Passenger) knew that listening on port 0 was a thing, the OS
109 would take care of the races, and this wouldn't be needed at all.
112 sock = socket.socket()
113 sock.bind(('0.0.0.0', 0))
114 port = sock.getsockname()[1]
118 def _wait_until_port_listens(port, timeout=10):
119 """Wait for a process to start listening on the given port.
121 If nothing listens on the port within the specified timeout (given
122 in seconds), print a warning on stderr before returning.
125 subprocess.check_output(['which', 'lsof'])
126 except subprocess.CalledProcessError:
127 print("WARNING: No `lsof` -- cannot wait for port to listen. "+
128 "Sleeping 0.5 and hoping for the best.")
131 deadline = time.time() + timeout
132 while time.time() < deadline:
134 subprocess.check_output(
135 ['lsof', '-t', '-i', 'tcp:'+str(port)])
136 except subprocess.CalledProcessError:
141 "WARNING: Nothing is listening on port {} (waited {} seconds).".
142 format(port, timeout),
145 def run(leave_running_atexit=False):
146 """Ensure an API server is running, and ARVADOS_API_* env vars have
147 admin credentials for it.
149 If ARVADOS_TEST_API_HOST is set, a parent process has started a
150 test server for us to use: we just need to reset() it using the
153 If a previous call to run() started a new server process, and it
154 is still running, we just need to reset() it to fixture state and
157 If neither of those options work out, we'll really start a new
162 # Delete cached discovery document.
163 shutil.rmtree(arvados.http_cache('discovery'))
165 pid_file = os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH)
166 pid_file_ok = find_server_pid(pid_file, 0)
168 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
169 if existing_api_host and pid_file_ok:
170 if existing_api_host == my_api_host:
174 # Fall through to shutdown-and-start case.
177 # Server was provided by parent. Can't recover if it's
181 # Before trying to start up our own server, call stop() to avoid
182 # "Phusion Passenger Standalone is already running on PID 12345".
183 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
184 # we know the server is ours to kill.)
187 restore_cwd = os.getcwd()
188 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
189 os.chdir(api_src_dir)
191 # Either we haven't started a server of our own yet, or it has
192 # died, or we have lost our credentials, or something else is
193 # preventing us from calling reset(). Start a new one.
195 if not os.path.exists('tmp'):
198 if not os.path.exists('tmp/api'):
199 os.makedirs('tmp/api')
201 if not os.path.exists('tmp/logs'):
202 os.makedirs('tmp/logs')
204 if not os.path.exists('tmp/self-signed.pem'):
205 # We assume here that either passenger reports its listening
206 # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
207 # then the certificate won't match the host and reset() will
208 # fail certificate verification. If it reports "localhost",
209 # clients (notably Python SDK's websocket client) might
210 # resolve localhost as ::1 and then fail to connect.
211 subprocess.check_call([
212 'openssl', 'req', '-new', '-x509', '-nodes',
213 '-out', 'tmp/self-signed.pem',
214 '-keyout', 'tmp/self-signed.key',
216 '-subj', '/CN=0.0.0.0'],
219 # Install the git repository fixtures.
220 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
221 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
222 if not os.path.isdir(gitdir):
224 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
226 port = find_available_port()
227 env = os.environ.copy()
228 env['RAILS_ENV'] = 'test'
229 env['ARVADOS_WEBSOCKETS'] = 'yes'
230 env.pop('ARVADOS_TEST_API_HOST', None)
231 env.pop('ARVADOS_API_HOST', None)
232 env.pop('ARVADOS_API_HOST_INSECURE', None)
233 env.pop('ARVADOS_API_TOKEN', None)
234 start_msg = subprocess.check_output(
236 'passenger', 'start', '-d', '-p{}'.format(port),
237 '--pid-file', os.path.join(os.getcwd(), pid_file),
238 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
240 '--ssl-certificate', 'tmp/self-signed.pem',
241 '--ssl-certificate-key', 'tmp/self-signed.key'],
244 if not leave_running_atexit:
245 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
247 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
250 "Passenger did not report endpoint: {}".format(start_msg))
251 my_api_host = match.group(1)
252 os.environ['ARVADOS_API_HOST'] = my_api_host
254 # Make sure the server has written its pid file and started
255 # listening on its TCP port
256 find_server_pid(pid_file)
257 _wait_until_port_listens(port)
260 os.chdir(restore_cwd)
263 """Reset the test server to fixture state.
265 This resets the ARVADOS_TEST_API_HOST provided by a parent process
266 if any, otherwise the server started by run().
268 It also resets ARVADOS_* environment vars to point to the test
269 server with admin credentials.
271 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
272 token = auth_token('admin')
273 httpclient = httplib2.Http(ca_certs=os.path.join(
274 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
276 'https://{}/database/reset'.format(existing_api_host),
278 headers={'Authorization': 'OAuth2 {}'.format(token)})
279 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
280 os.environ['ARVADOS_API_HOST'] = existing_api_host
281 os.environ['ARVADOS_API_TOKEN'] = token
283 def stop(force=False):
284 """Stop the API server, if one is running.
286 If force==False, kill it only if we started it ourselves. (This
287 supports the use case where a Python test suite calls run(), but
288 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
289 process, and the test suite cleans up after itself by calling
290 stop(). In this case the test server provided by the parent
291 process should be left alone.)
293 If force==True, kill it even if we didn't start it
294 ourselves. (This supports the use case in __main__, where "run"
295 and "stop" happen in different processes.)
298 if force or my_api_host is not None:
299 kill_server_pid(os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH))
302 def _start_keep(n, keep_args):
303 keep0 = tempfile.mkdtemp()
304 port = find_available_port()
305 keep_cmd = ["keepstore",
306 "-volume={}".format(keep0),
307 "-listen=:{}".format(port),
308 "-pid="+_pidfile('keep{}'.format(n))]
310 for arg, val in keep_args.iteritems():
311 keep_cmd.append("{}={}".format(arg, val))
313 kp0 = subprocess.Popen(
314 keep_cmd, stdin=open('/dev/null'), stdout=sys.stderr)
315 with open(_pidfile('keep{}'.format(n)), 'w') as f:
316 f.write(str(kp0.pid))
318 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
321 _wait_until_port_listens(port)
325 def run_keep(blob_signing_key=None, enforce_permissions=False):
330 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
331 keep_args['--permission-key-file'] = f.name
332 f.write(blob_signing_key)
333 if enforce_permissions:
334 keep_args['--enforce-permissions'] = 'true'
338 host=os.environ['ARVADOS_API_HOST'],
339 token=os.environ['ARVADOS_API_TOKEN'],
341 for d in api.keep_services().list().execute()['items']:
342 api.keep_services().delete(uuid=d['uuid']).execute()
343 for d in api.keep_disks().list().execute()['items']:
344 api.keep_disks().delete(uuid=d['uuid']).execute()
346 for d in range(0, 2):
347 port = _start_keep(d, keep_args)
348 svc = api.keep_services().create(body={'keep_service': {
349 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
350 'service_host': 'localhost',
351 'service_port': port,
352 'service_type': 'disk',
353 'service_ssl_flag': False,
355 api.keep_disks().create(body={
356 'keep_disk': {'keep_service_uuid': svc['uuid'] }
360 kill_server_pid(_pidfile('keep{}'.format(n)), 0)
361 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
362 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
363 shutil.rmtree(r.read(), True)
364 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
365 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
366 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
372 def run_keep_proxy():
373 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
377 admin_token = auth_token('admin')
378 port = find_available_port()
379 env = os.environ.copy()
380 env['ARVADOS_API_TOKEN'] = admin_token
381 kp = subprocess.Popen(
383 '-pid='+_pidfile('keepproxy'),
384 '-listen=:{}'.format(port)],
385 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
389 host=os.environ['ARVADOS_API_HOST'],
392 for d in api.keep_services().list(
393 filters=[['service_type','=','proxy']]).execute()['items']:
394 api.keep_services().delete(uuid=d['uuid']).execute()
395 api.keep_services().create(body={'keep_service': {
396 'service_host': 'localhost',
397 'service_port': port,
398 'service_type': 'proxy',
399 'service_ssl_flag': False,
401 os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
402 _setport('keepproxy', port)
403 _wait_until_port_listens(port)
405 def stop_keep_proxy():
406 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
408 kill_server_pid(_pidfile('keepproxy'), wait=0)
410 def run_arv_git_httpd():
411 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
415 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
416 gitport = find_available_port()
417 env = os.environ.copy()
418 env.pop('ARVADOS_API_TOKEN', None)
419 agh = subprocess.Popen(
421 '-repo-root='+gitdir+'/test',
422 '-address=:'+str(gitport)],
423 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
424 with open(_pidfile('arv-git-httpd'), 'w') as f:
425 f.write(str(agh.pid))
426 _setport('arv-git-httpd', gitport)
427 _wait_until_port_listens(gitport)
429 def stop_arv_git_httpd():
430 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
432 kill_server_pid(_pidfile('arv-git-httpd'), wait=0)
435 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
438 nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
439 nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
440 nginxconf['GITPORT'] = _getport('arv-git-httpd')
441 nginxconf['GITSSLPORT'] = find_available_port()
442 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
443 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
444 nginxconf['ACCESSLOG'] = os.path.join(TEST_TMPDIR, 'nginx_access_log.fifo')
446 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
447 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
448 with open(conffile, 'w') as f:
451 lambda match: str(nginxconf.get(match.group(1))),
452 open(conftemplatefile).read()))
454 env = os.environ.copy()
455 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
456 os.mkfifo(nginxconf['ACCESSLOG'], 0700)
457 nginx = subprocess.Popen(
459 '-g', 'error_log stderr info;',
460 '-g', 'pid '+_pidfile('nginx')+';',
462 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
463 cat_access = subprocess.Popen(
464 ['cat', nginxconf['ACCESSLOG']],
466 _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
467 _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
470 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
472 kill_server_pid(_pidfile('nginx'), wait=0)
474 def _pidfile(program):
475 return os.path.join(TEST_TMPDIR, program + '.pid')
477 def _portfile(program):
478 return os.path.join(TEST_TMPDIR, program + '.port')
480 def _setport(program, port):
481 with open(_portfile(program), 'w') as f:
484 # Returns 9 if program is not up.
485 def _getport(program):
487 return int(open(_portfile(program)).read())
493 return _cached_config[key]
494 def _load(f, required=True):
495 fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
496 if not required and not os.path.exists(fullpath):
498 return yaml.load(fullpath)
499 cdefault = _load('application.default.yml')
500 csite = _load('application.yml', required=False)
502 for section in [cdefault.get('common',{}), cdefault.get('test',{}),
503 csite.get('common',{}), csite.get('test',{})]:
504 _cached_config.update(section)
505 return _cached_config[key]
508 '''load a fixture yaml file'''
509 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
513 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
514 yaml_file = yaml_file[0:trim_index]
517 return yaml.load(yaml_file)
519 def auth_token(token_name):
520 return fixture("api_client_authorizations")[token_name]["api_token"]
522 def authorize_with(token_name):
523 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
524 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
525 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
526 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
528 class TestCaseWithServers(unittest.TestCase):
529 """TestCase to start and stop supporting Arvados servers.
531 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
532 class variables as a dictionary of keyword arguments. If you do,
533 setUpClass will start the corresponding servers by passing these
534 keyword arguments to the run, run_keep, and/or run_keep_server
535 functions, respectively. It will also set Arvados environment
536 variables to point to these servers appropriately. If you don't
537 run a Keep or Keep proxy server, setUpClass will set up a
538 temporary directory for Keep local storage, and set it as
541 tearDownClass will stop any servers started, and restore the
542 original environment.
546 KEEP_PROXY_SERVER = None
549 def _restore_dict(src, dest):
550 for key in dest.keys():
557 cls._orig_environ = os.environ.copy()
558 cls._orig_config = arvados.config.settings().copy()
559 cls._cleanup_funcs = []
560 os.environ.pop('ARVADOS_KEEP_PROXY', None)
561 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
562 for server_kwargs, start_func, stop_func in (
563 (cls.MAIN_SERVER, run, reset),
564 (cls.KEEP_SERVER, run_keep, stop_keep),
565 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
566 if server_kwargs is not None:
567 start_func(**server_kwargs)
568 cls._cleanup_funcs.append(stop_func)
569 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
570 cls.local_store = tempfile.mkdtemp()
571 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
572 cls._cleanup_funcs.append(
573 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
575 os.environ.pop('KEEP_LOCAL_STORE', None)
576 arvados.config.initialize()
579 def tearDownClass(cls):
580 for clean_func in cls._cleanup_funcs:
582 cls._restore_dict(cls._orig_environ, os.environ)
583 cls._restore_dict(cls._orig_config, arvados.config.settings())
586 if __name__ == "__main__":
589 'start_keep', 'stop_keep',
590 'start_keep_proxy', 'stop_keep_proxy',
591 'start_arv-git-httpd', 'stop_arv-git-httpd',
592 'start_nginx', 'stop_nginx',
594 parser = argparse.ArgumentParser()
595 parser.add_argument('action', type=str, help="one of {}".format(actions))
596 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
597 args = parser.parse_args()
599 if args.action not in actions:
600 print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr)
602 if args.action == 'start':
603 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
604 run(leave_running_atexit=True)
605 host = os.environ['ARVADOS_API_HOST']
606 if args.auth is not None:
607 token = auth_token(args.auth)
608 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
609 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
610 print("export ARVADOS_API_HOST_INSECURE=true")
613 elif args.action == 'stop':
614 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
615 elif args.action == 'start_keep':
617 elif args.action == 'stop_keep':
619 elif args.action == 'start_keep_proxy':
621 elif args.action == 'stop_keep_proxy':
623 elif args.action == 'start_arv-git-httpd':
625 elif args.action == 'stop_arv-git-httpd':
627 elif args.action == 'start_nginx':
629 elif args.action == 'stop_nginx':
632 raise Exception("action recognized but not implemented!?")