3 from __future__ import print_function
22 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
23 if __name__ == '__main__' and os.path.exists(
24 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
25 # We're being launched to support another test suite.
26 # Add the Python SDK source to the library path.
27 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
32 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
33 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
34 SERVER_PID_PATH = 'tmp/pids/test-server.pid'
35 if 'GOPATH' in os.environ:
36 gopaths = os.environ['GOPATH'].split(':')
37 gobins = [os.path.join(path, 'bin') for path in gopaths]
38 os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
40 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
41 if not os.path.exists(TEST_TMPDIR):
47 def find_server_pid(PID_PATH, wait=10):
51 while (not good_pid) and (now <= timeout):
54 with open(PID_PATH, 'r') as f:
55 server_pid = int(f.read())
56 good_pid = (os.kill(server_pid, 0) is None)
68 def kill_server_pid(pidfile, wait=10, passenger_root=False):
69 # Must re-import modules in order to work during atexit
76 # First try to shut down nicely
77 restore_cwd = os.getcwd()
78 os.chdir(passenger_root)
80 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
84 with open(pidfile, 'r') as f:
85 server_pid = int(f.read())
87 if not passenger_root or timeout - now < wait / 2:
88 # Half timeout has elapsed. Start sending SIGTERM
89 os.kill(server_pid, signal.SIGTERM)
90 # Raise OSError if process has disappeared
91 os.getpgid(server_pid)
99 def find_available_port():
100 """Return an IPv4 port number that is not in use right now.
102 We assume whoever needs to use the returned port is able to reuse
103 a recently used port without waiting for TIME_WAIT (see
104 SO_REUSEADDR / SO_REUSEPORT).
106 Some opportunity for races here, but it's better than choosing
107 something at random and not checking at all. If all of our servers
108 (hey Passenger) knew that listening on port 0 was a thing, the OS
109 would take care of the races, and this wouldn't be needed at all.
112 sock = socket.socket()
113 sock.bind(('0.0.0.0', 0))
114 port = sock.getsockname()[1]
118 def _wait_until_port_listens(port, timeout=10):
119 """Wait for a process to start listening on the given port.
121 If nothing listens on the port within the specified timeout (given
122 in seconds), print a warning on stderr before returning.
125 subprocess.check_output(['which', 'lsof'])
126 except subprocess.CalledProcessError:
127 print("WARNING: No `lsof` -- cannot wait for port to listen. "+
128 "Sleeping 0.5 and hoping for the best.")
131 deadline = time.time() + timeout
132 while time.time() < deadline:
134 subprocess.check_output(
135 ['lsof', '-t', '-i', 'tcp:'+str(port)])
136 except subprocess.CalledProcessError:
141 "WARNING: Nothing is listening on port {} (waited {} seconds).".
142 format(port, timeout),
145 def run(leave_running_atexit=False):
146 """Ensure an API server is running, and ARVADOS_API_* env vars have
147 admin credentials for it.
149 If ARVADOS_TEST_API_HOST is set, a parent process has started a
150 test server for us to use: we just need to reset() it using the
153 If a previous call to run() started a new server process, and it
154 is still running, we just need to reset() it to fixture state and
157 If neither of those options work out, we'll really start a new
162 # Delete cached discovery document.
163 shutil.rmtree(arvados.http_cache('discovery'))
165 pid_file = os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH)
166 pid_file_ok = find_server_pid(pid_file, 0)
168 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
169 if existing_api_host and pid_file_ok:
170 if existing_api_host == my_api_host:
174 # Fall through to shutdown-and-start case.
177 # Server was provided by parent. Can't recover if it's
181 # Before trying to start up our own server, call stop() to avoid
182 # "Phusion Passenger Standalone is already running on PID 12345".
183 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
184 # we know the server is ours to kill.)
187 restore_cwd = os.getcwd()
188 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
189 os.chdir(api_src_dir)
191 # Either we haven't started a server of our own yet, or it has
192 # died, or we have lost our credentials, or something else is
193 # preventing us from calling reset(). Start a new one.
195 if not os.path.exists('tmp'):
198 if not os.path.exists('tmp/api'):
199 os.makedirs('tmp/api')
201 if not os.path.exists('tmp/logs'):
202 os.makedirs('tmp/logs')
204 if not os.path.exists('tmp/self-signed.pem'):
205 # We assume here that either passenger reports its listening
206 # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
207 # then the certificate won't match the host and reset() will
208 # fail certificate verification. If it reports "localhost",
209 # clients (notably Python SDK's websocket client) might
210 # resolve localhost as ::1 and then fail to connect.
211 subprocess.check_call([
212 'openssl', 'req', '-new', '-x509', '-nodes',
213 '-out', 'tmp/self-signed.pem',
214 '-keyout', 'tmp/self-signed.key',
216 '-subj', '/CN=0.0.0.0'],
219 # Install the git repository fixtures.
220 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
221 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
222 if not os.path.isdir(gitdir):
224 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
226 port = find_available_port()
227 env = os.environ.copy()
228 env['RAILS_ENV'] = 'test'
229 env['ARVADOS_WEBSOCKETS'] = 'yes'
230 env.pop('ARVADOS_TEST_API_HOST', None)
231 env.pop('ARVADOS_API_HOST', None)
232 env.pop('ARVADOS_API_HOST_INSECURE', None)
233 env.pop('ARVADOS_API_TOKEN', None)
234 start_msg = subprocess.check_output(
236 'passenger', 'start', '-d', '-p{}'.format(port),
237 '--pid-file', os.path.join(os.getcwd(), pid_file),
238 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
240 '--ssl-certificate', 'tmp/self-signed.pem',
241 '--ssl-certificate-key', 'tmp/self-signed.key'],
244 if not leave_running_atexit:
245 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
247 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
250 "Passenger did not report endpoint: {}".format(start_msg))
251 my_api_host = match.group(1)
252 os.environ['ARVADOS_API_HOST'] = my_api_host
254 # Make sure the server has written its pid file and started
255 # listening on its TCP port
256 find_server_pid(pid_file)
257 _wait_until_port_listens(port)
260 os.chdir(restore_cwd)
263 """Reset the test server to fixture state.
265 This resets the ARVADOS_TEST_API_HOST provided by a parent process
266 if any, otherwise the server started by run().
268 It also resets ARVADOS_* environment vars to point to the test
269 server with admin credentials.
271 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
272 token = auth_token('admin')
273 httpclient = httplib2.Http(ca_certs=os.path.join(
274 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
276 'https://{}/database/reset'.format(existing_api_host),
278 headers={'Authorization': 'OAuth2 {}'.format(token)})
279 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
280 os.environ['ARVADOS_API_HOST'] = existing_api_host
281 os.environ['ARVADOS_API_TOKEN'] = token
283 def stop(force=False):
284 """Stop the API server, if one is running.
286 If force==False, kill it only if we started it ourselves. (This
287 supports the use case where a Python test suite calls run(), but
288 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
289 process, and the test suite cleans up after itself by calling
290 stop(). In this case the test server provided by the parent
291 process should be left alone.)
293 If force==True, kill it even if we didn't start it
294 ourselves. (This supports the use case in __main__, where "run"
295 and "stop" happen in different processes.)
298 if force or my_api_host is not None:
299 kill_server_pid(os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH))
302 def _start_keep(n, keep_args):
303 keep0 = tempfile.mkdtemp()
304 port = find_available_port()
305 keep_cmd = ["keepstore",
306 "-volume={}".format(keep0),
307 "-listen=:{}".format(port),
308 "-pid="+_pidfile('keep{}'.format(n))]
310 for arg, val in keep_args.iteritems():
311 keep_cmd.append("{}={}".format(arg, val))
313 logf = open(os.path.join(TEST_TMPDIR, 'keep{}.log'.format(n)), 'a+')
314 kp0 = subprocess.Popen(
315 keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
316 with open(_pidfile('keep{}'.format(n)), 'w') as f:
317 f.write(str(kp0.pid))
319 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
322 _wait_until_port_listens(port)
326 def run_keep(blob_signing_key=None, enforce_permissions=False):
330 if not blob_signing_key:
331 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
332 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
333 keep_args['--blob-signing-key-file'] = f.name
334 f.write(blob_signing_key)
335 if enforce_permissions:
336 keep_args['--enforce-permissions'] = 'true'
337 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
338 keep_args['--data-manager-token-file'] = f.name
339 f.write(os.environ['ARVADOS_API_TOKEN'])
340 keep_args['--never-delete'] = 'false'
344 host=os.environ['ARVADOS_API_HOST'],
345 token=os.environ['ARVADOS_API_TOKEN'],
347 for d in api.keep_services().list().execute()['items']:
348 api.keep_services().delete(uuid=d['uuid']).execute()
349 for d in api.keep_disks().list().execute()['items']:
350 api.keep_disks().delete(uuid=d['uuid']).execute()
352 for d in range(0, 2):
353 port = _start_keep(d, keep_args)
354 svc = api.keep_services().create(body={'keep_service': {
355 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
356 'service_host': 'localhost',
357 'service_port': port,
358 'service_type': 'disk',
359 'service_ssl_flag': False,
361 api.keep_disks().create(body={
362 'keep_disk': {'keep_service_uuid': svc['uuid'] }
366 kill_server_pid(_pidfile('keep{}'.format(n)), 0)
367 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
368 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
369 shutil.rmtree(r.read(), True)
370 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
371 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
372 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
373 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file")):
374 os.remove(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"))
380 def run_keep_proxy():
381 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
385 admin_token = auth_token('admin')
386 port = find_available_port()
387 env = os.environ.copy()
388 env['ARVADOS_API_TOKEN'] = admin_token
389 kp = subprocess.Popen(
391 '-pid='+_pidfile('keepproxy'),
392 '-listen=:{}'.format(port)],
393 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
397 host=os.environ['ARVADOS_API_HOST'],
400 for d in api.keep_services().list(
401 filters=[['service_type','=','proxy']]).execute()['items']:
402 api.keep_services().delete(uuid=d['uuid']).execute()
403 api.keep_services().create(body={'keep_service': {
404 'service_host': 'localhost',
405 'service_port': port,
406 'service_type': 'proxy',
407 'service_ssl_flag': False,
409 os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
410 _setport('keepproxy', port)
411 _wait_until_port_listens(port)
413 def stop_keep_proxy():
414 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
416 kill_server_pid(_pidfile('keepproxy'), wait=0)
418 def run_arv_git_httpd():
419 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
423 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
424 gitport = find_available_port()
425 env = os.environ.copy()
426 env.pop('ARVADOS_API_TOKEN', None)
427 agh = subprocess.Popen(
429 '-repo-root='+gitdir+'/test',
430 '-address=:'+str(gitport)],
431 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
432 with open(_pidfile('arv-git-httpd'), 'w') as f:
433 f.write(str(agh.pid))
434 _setport('arv-git-httpd', gitport)
435 _wait_until_port_listens(gitport)
437 def stop_arv_git_httpd():
438 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
440 kill_server_pid(_pidfile('arv-git-httpd'), wait=0)
443 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
446 nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
447 nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
448 nginxconf['GITPORT'] = _getport('arv-git-httpd')
449 nginxconf['GITSSLPORT'] = find_available_port()
450 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
451 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
453 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
454 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
455 with open(conffile, 'w') as f:
458 lambda match: str(nginxconf.get(match.group(1))),
459 open(conftemplatefile).read()))
461 env = os.environ.copy()
462 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
463 nginx = subprocess.Popen(
465 '-g', 'error_log stderr info;',
466 '-g', 'pid '+_pidfile('nginx')+';',
468 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
469 _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
470 _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
473 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
475 kill_server_pid(_pidfile('nginx'), wait=0)
477 def _pidfile(program):
478 return os.path.join(TEST_TMPDIR, program + '.pid')
480 def _portfile(program):
481 return os.path.join(TEST_TMPDIR, program + '.port')
483 def _setport(program, port):
484 with open(_portfile(program), 'w') as f:
487 # Returns 9 if program is not up.
488 def _getport(program):
490 return int(open(_portfile(program)).read())
496 return _cached_config[key]
497 def _load(f, required=True):
498 fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
499 if not required and not os.path.exists(fullpath):
501 return yaml.load(fullpath)
502 cdefault = _load('application.default.yml')
503 csite = _load('application.yml', required=False)
505 for section in [cdefault.get('common',{}), cdefault.get('test',{}),
506 csite.get('common',{}), csite.get('test',{})]:
507 _cached_config.update(section)
508 return _cached_config[key]
511 '''load a fixture yaml file'''
512 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
516 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
517 yaml_file = yaml_file[0:trim_index]
520 return yaml.load(yaml_file)
522 def auth_token(token_name):
523 return fixture("api_client_authorizations")[token_name]["api_token"]
525 def authorize_with(token_name):
526 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
527 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
528 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
529 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
531 class TestCaseWithServers(unittest.TestCase):
532 """TestCase to start and stop supporting Arvados servers.
534 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
535 class variables as a dictionary of keyword arguments. If you do,
536 setUpClass will start the corresponding servers by passing these
537 keyword arguments to the run, run_keep, and/or run_keep_server
538 functions, respectively. It will also set Arvados environment
539 variables to point to these servers appropriately. If you don't
540 run a Keep or Keep proxy server, setUpClass will set up a
541 temporary directory for Keep local storage, and set it as
544 tearDownClass will stop any servers started, and restore the
545 original environment.
549 KEEP_PROXY_SERVER = None
552 def _restore_dict(src, dest):
553 for key in dest.keys():
560 cls._orig_environ = os.environ.copy()
561 cls._orig_config = arvados.config.settings().copy()
562 cls._cleanup_funcs = []
563 os.environ.pop('ARVADOS_KEEP_PROXY', None)
564 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
565 for server_kwargs, start_func, stop_func in (
566 (cls.MAIN_SERVER, run, reset),
567 (cls.KEEP_SERVER, run_keep, stop_keep),
568 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
569 if server_kwargs is not None:
570 start_func(**server_kwargs)
571 cls._cleanup_funcs.append(stop_func)
572 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
573 cls.local_store = tempfile.mkdtemp()
574 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
575 cls._cleanup_funcs.append(
576 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
578 os.environ.pop('KEEP_LOCAL_STORE', None)
579 arvados.config.initialize()
582 def tearDownClass(cls):
583 for clean_func in cls._cleanup_funcs:
585 cls._restore_dict(cls._orig_environ, os.environ)
586 cls._restore_dict(cls._orig_config, arvados.config.settings())
589 if __name__ == "__main__":
592 'start_keep', 'stop_keep',
593 'start_keep_proxy', 'stop_keep_proxy',
594 'start_arv-git-httpd', 'stop_arv-git-httpd',
595 'start_nginx', 'stop_nginx',
597 parser = argparse.ArgumentParser()
598 parser.add_argument('action', type=str, help="one of {}".format(actions))
599 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
600 args = parser.parse_args()
602 if args.action not in actions:
603 print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr)
605 if args.action == 'start':
606 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
607 run(leave_running_atexit=True)
608 host = os.environ['ARVADOS_API_HOST']
609 if args.auth is not None:
610 token = auth_token(args.auth)
611 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
612 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
613 print("export ARVADOS_API_HOST_INSECURE=true")
616 elif args.action == 'stop':
617 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
618 elif args.action == 'start_keep':
620 elif args.action == 'stop_keep':
622 elif args.action == 'start_keep_proxy':
624 elif args.action == 'stop_keep_proxy':
626 elif args.action == 'start_arv-git-httpd':
628 elif args.action == 'stop_arv-git-httpd':
630 elif args.action == 'start_nginx':
632 elif args.action == 'stop_nginx':
635 raise Exception("action recognized but not implemented!?")