3 from __future__ import print_function
22 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
23 if __name__ == '__main__' and os.path.exists(
24 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
25 # We're being launched to support another test suite.
26 # Add the Python SDK source to the library path.
27 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
32 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
33 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
34 SERVER_PID_PATH = 'tmp/pids/test-server.pid'
35 if 'GOPATH' in os.environ:
36 gopaths = os.environ['GOPATH'].split(':')
37 gobins = [os.path.join(path, 'bin') for path in gopaths]
38 os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
40 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
41 if not os.path.exists(TEST_TMPDIR):
47 def find_server_pid(PID_PATH, wait=10):
51 while (not good_pid) and (now <= timeout):
54 with open(PID_PATH, 'r') as f:
55 server_pid = int(f.read())
56 good_pid = (os.kill(server_pid, 0) is None)
68 def kill_server_pid(pidfile, wait=10, passenger_root=False):
69 # Must re-import modules in order to work during atexit
76 # First try to shut down nicely
77 restore_cwd = os.getcwd()
78 os.chdir(passenger_root)
80 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
84 with open(pidfile, 'r') as f:
85 server_pid = int(f.read())
87 if not passenger_root or timeout - now < wait / 2:
88 # Half timeout has elapsed. Start sending SIGTERM
89 os.kill(server_pid, signal.SIGTERM)
90 # Raise OSError if process has disappeared
91 os.getpgid(server_pid)
99 def find_available_port():
100 """Return an IPv4 port number that is not in use right now.
102 We assume whoever needs to use the returned port is able to reuse
103 a recently used port without waiting for TIME_WAIT (see
104 SO_REUSEADDR / SO_REUSEPORT).
106 Some opportunity for races here, but it's better than choosing
107 something at random and not checking at all. If all of our servers
108 (hey Passenger) knew that listening on port 0 was a thing, the OS
109 would take care of the races, and this wouldn't be needed at all.
112 sock = socket.socket()
113 sock.bind(('0.0.0.0', 0))
114 port = sock.getsockname()[1]
118 def _wait_until_port_listens(port, timeout=10):
119 """Wait for a process to start listening on the given port.
121 If nothing listens on the port within the specified timeout (given
122 in seconds), print a warning on stderr before returning.
125 subprocess.check_output(['which', 'lsof'])
126 except subprocess.CalledProcessError:
127 print("WARNING: No `lsof` -- cannot wait for port to listen. "+
128 "Sleeping 0.5 and hoping for the best.")
131 deadline = time.time() + timeout
132 while time.time() < deadline:
134 subprocess.check_output(
135 ['lsof', '-t', '-i', 'tcp:'+str(port)])
136 except subprocess.CalledProcessError:
141 "WARNING: Nothing is listening on port {} (waited {} seconds).".
142 format(port, timeout),
145 def run(leave_running_atexit=False):
146 """Ensure an API server is running, and ARVADOS_API_* env vars have
147 admin credentials for it.
149 If ARVADOS_TEST_API_HOST is set, a parent process has started a
150 test server for us to use: we just need to reset() it using the
153 If a previous call to run() started a new server process, and it
154 is still running, we just need to reset() it to fixture state and
157 If neither of those options work out, we'll really start a new
162 # Delete cached discovery document.
163 shutil.rmtree(arvados.http_cache('discovery'))
165 pid_file = os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH)
166 pid_file_ok = find_server_pid(pid_file, 0)
168 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
169 if existing_api_host and pid_file_ok:
170 if existing_api_host == my_api_host:
174 # Fall through to shutdown-and-start case.
177 # Server was provided by parent. Can't recover if it's
181 # Before trying to start up our own server, call stop() to avoid
182 # "Phusion Passenger Standalone is already running on PID 12345".
183 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
184 # we know the server is ours to kill.)
187 restore_cwd = os.getcwd()
188 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
189 os.chdir(api_src_dir)
191 # Either we haven't started a server of our own yet, or it has
192 # died, or we have lost our credentials, or something else is
193 # preventing us from calling reset(). Start a new one.
195 if not os.path.exists('tmp'):
198 if not os.path.exists('tmp/api'):
199 os.makedirs('tmp/api')
201 if not os.path.exists('tmp/logs'):
202 os.makedirs('tmp/logs')
204 if not os.path.exists('tmp/self-signed.pem'):
205 # We assume here that either passenger reports its listening
206 # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
207 # then the certificate won't match the host and reset() will
208 # fail certificate verification. If it reports "localhost",
209 # clients (notably Python SDK's websocket client) might
210 # resolve localhost as ::1 and then fail to connect.
211 subprocess.check_call([
212 'openssl', 'req', '-new', '-x509', '-nodes',
213 '-out', 'tmp/self-signed.pem',
214 '-keyout', 'tmp/self-signed.key',
216 '-subj', '/CN=0.0.0.0'],
219 # Install the git repository fixtures.
220 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
221 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
222 if not os.path.isdir(gitdir):
224 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
226 port = find_available_port()
227 env = os.environ.copy()
228 env['RAILS_ENV'] = 'test'
229 env['ARVADOS_WEBSOCKETS'] = 'yes'
230 env.pop('ARVADOS_TEST_API_HOST', None)
231 env.pop('ARVADOS_API_HOST', None)
232 env.pop('ARVADOS_API_HOST_INSECURE', None)
233 env.pop('ARVADOS_API_TOKEN', None)
234 start_msg = subprocess.check_output(
236 'passenger', 'start', '-d', '-p{}'.format(port),
237 '--pid-file', os.path.join(os.getcwd(), pid_file),
238 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
240 '--ssl-certificate', 'tmp/self-signed.pem',
241 '--ssl-certificate-key', 'tmp/self-signed.key'],
244 if not leave_running_atexit:
245 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
247 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
250 "Passenger did not report endpoint: {}".format(start_msg))
251 my_api_host = match.group(1)
252 os.environ['ARVADOS_API_HOST'] = my_api_host
254 # Make sure the server has written its pid file and started
255 # listening on its TCP port
256 find_server_pid(pid_file)
257 _wait_until_port_listens(port)
260 os.chdir(restore_cwd)
263 """Reset the test server to fixture state.
265 This resets the ARVADOS_TEST_API_HOST provided by a parent process
266 if any, otherwise the server started by run().
268 It also resets ARVADOS_* environment vars to point to the test
269 server with admin credentials.
271 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
272 token = auth_token('admin')
273 httpclient = httplib2.Http(ca_certs=os.path.join(
274 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
276 'https://{}/database/reset'.format(existing_api_host),
278 headers={'Authorization': 'OAuth2 {}'.format(token)})
279 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
280 os.environ['ARVADOS_API_HOST'] = existing_api_host
281 os.environ['ARVADOS_API_TOKEN'] = token
283 def stop(force=False):
284 """Stop the API server, if one is running.
286 If force==False, kill it only if we started it ourselves. (This
287 supports the use case where a Python test suite calls run(), but
288 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
289 process, and the test suite cleans up after itself by calling
290 stop(). In this case the test server provided by the parent
291 process should be left alone.)
293 If force==True, kill it even if we didn't start it
294 ourselves. (This supports the use case in __main__, where "run"
295 and "stop" happen in different processes.)
298 if force or my_api_host is not None:
299 kill_server_pid(os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH))
302 def _start_keep(n, keep_args):
303 keep0 = tempfile.mkdtemp()
304 port = find_available_port()
305 keep_cmd = ["keepstore",
306 "-volume={}".format(keep0),
307 "-listen=:{}".format(port),
308 "-pid="+_pidfile('keep{}'.format(n))]
310 for arg, val in keep_args.iteritems():
311 keep_cmd.append("{}={}".format(arg, val))
313 logf = open(os.path.join(TEST_TMPDIR, 'keep{}.log'.format(n)), 'a+')
314 kp0 = subprocess.Popen(
315 keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
316 with open(_pidfile('keep{}'.format(n)), 'w') as f:
317 f.write(str(kp0.pid))
319 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
322 _wait_until_port_listens(port)
326 def run_keep(blob_signing_key=None, enforce_permissions=False):
330 if not blob_signing_key:
331 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
332 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
333 keep_args['-blob-signing-key-file'] = f.name
334 f.write(blob_signing_key)
335 if enforce_permissions:
336 keep_args['-enforce-permissions'] = 'true'
337 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
338 keep_args['-data-manager-token-file'] = f.name
339 f.write(os.environ['ARVADOS_API_TOKEN'])
340 keep_args['-never-delete'] = 'false'
344 host=os.environ['ARVADOS_API_HOST'],
345 token=os.environ['ARVADOS_API_TOKEN'],
347 for d in api.keep_services().list().execute()['items']:
348 api.keep_services().delete(uuid=d['uuid']).execute()
349 for d in api.keep_disks().list().execute()['items']:
350 api.keep_disks().delete(uuid=d['uuid']).execute()
352 for d in range(0, 2):
353 port = _start_keep(d, keep_args)
354 svc = api.keep_services().create(body={'keep_service': {
355 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
356 'service_host': 'localhost',
357 'service_port': port,
358 'service_type': 'disk',
359 'service_ssl_flag': False,
361 api.keep_disks().create(body={
362 'keep_disk': {'keep_service_uuid': svc['uuid'] }
366 kill_server_pid(_pidfile('keep{}'.format(n)), 0)
367 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
368 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
369 shutil.rmtree(r.read(), True)
370 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
371 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
372 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
378 def run_keep_proxy():
379 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
383 admin_token = auth_token('admin')
384 port = find_available_port()
385 env = os.environ.copy()
386 env['ARVADOS_API_TOKEN'] = admin_token
387 kp = subprocess.Popen(
389 '-pid='+_pidfile('keepproxy'),
390 '-listen=:{}'.format(port)],
391 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
395 host=os.environ['ARVADOS_API_HOST'],
398 for d in api.keep_services().list(
399 filters=[['service_type','=','proxy']]).execute()['items']:
400 api.keep_services().delete(uuid=d['uuid']).execute()
401 api.keep_services().create(body={'keep_service': {
402 'service_host': 'localhost',
403 'service_port': port,
404 'service_type': 'proxy',
405 'service_ssl_flag': False,
407 os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
408 _setport('keepproxy', port)
409 _wait_until_port_listens(port)
411 def stop_keep_proxy():
412 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
414 kill_server_pid(_pidfile('keepproxy'), wait=0)
416 def run_arv_git_httpd():
417 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
421 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
422 gitport = find_available_port()
423 env = os.environ.copy()
424 env.pop('ARVADOS_API_TOKEN', None)
425 agh = subprocess.Popen(
427 '-repo-root='+gitdir+'/test',
428 '-address=:'+str(gitport)],
429 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
430 with open(_pidfile('arv-git-httpd'), 'w') as f:
431 f.write(str(agh.pid))
432 _setport('arv-git-httpd', gitport)
433 _wait_until_port_listens(gitport)
435 def stop_arv_git_httpd():
436 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
438 kill_server_pid(_pidfile('arv-git-httpd'), wait=0)
441 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
444 nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
445 nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
446 nginxconf['GITPORT'] = _getport('arv-git-httpd')
447 nginxconf['GITSSLPORT'] = find_available_port()
448 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
449 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
451 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
452 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
453 with open(conffile, 'w') as f:
456 lambda match: str(nginxconf.get(match.group(1))),
457 open(conftemplatefile).read()))
459 env = os.environ.copy()
460 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
461 nginx = subprocess.Popen(
463 '-g', 'error_log stderr info;',
464 '-g', 'pid '+_pidfile('nginx')+';',
466 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
467 _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
468 _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
471 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
473 kill_server_pid(_pidfile('nginx'), wait=0)
475 def _pidfile(program):
476 return os.path.join(TEST_TMPDIR, program + '.pid')
478 def _portfile(program):
479 return os.path.join(TEST_TMPDIR, program + '.port')
481 def _setport(program, port):
482 with open(_portfile(program), 'w') as f:
485 # Returns 9 if program is not up.
486 def _getport(program):
488 return int(open(_portfile(program)).read())
494 return _cached_config[key]
495 def _load(f, required=True):
496 fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
497 if not required and not os.path.exists(fullpath):
499 return yaml.load(fullpath)
500 cdefault = _load('application.default.yml')
501 csite = _load('application.yml', required=False)
503 for section in [cdefault.get('common',{}), cdefault.get('test',{}),
504 csite.get('common',{}), csite.get('test',{})]:
505 _cached_config.update(section)
506 return _cached_config[key]
509 '''load a fixture yaml file'''
510 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
514 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
515 yaml_file = yaml_file[0:trim_index]
518 return yaml.load(yaml_file)
520 def auth_token(token_name):
521 return fixture("api_client_authorizations")[token_name]["api_token"]
523 def authorize_with(token_name):
524 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
525 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
526 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
527 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
529 class TestCaseWithServers(unittest.TestCase):
530 """TestCase to start and stop supporting Arvados servers.
532 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
533 class variables as a dictionary of keyword arguments. If you do,
534 setUpClass will start the corresponding servers by passing these
535 keyword arguments to the run, run_keep, and/or run_keep_server
536 functions, respectively. It will also set Arvados environment
537 variables to point to these servers appropriately. If you don't
538 run a Keep or Keep proxy server, setUpClass will set up a
539 temporary directory for Keep local storage, and set it as
542 tearDownClass will stop any servers started, and restore the
543 original environment.
547 KEEP_PROXY_SERVER = None
550 def _restore_dict(src, dest):
551 for key in dest.keys():
558 cls._orig_environ = os.environ.copy()
559 cls._orig_config = arvados.config.settings().copy()
560 cls._cleanup_funcs = []
561 os.environ.pop('ARVADOS_KEEP_PROXY', None)
562 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
563 for server_kwargs, start_func, stop_func in (
564 (cls.MAIN_SERVER, run, reset),
565 (cls.KEEP_SERVER, run_keep, stop_keep),
566 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
567 if server_kwargs is not None:
568 start_func(**server_kwargs)
569 cls._cleanup_funcs.append(stop_func)
570 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
571 cls.local_store = tempfile.mkdtemp()
572 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
573 cls._cleanup_funcs.append(
574 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
576 os.environ.pop('KEEP_LOCAL_STORE', None)
577 arvados.config.initialize()
580 def tearDownClass(cls):
581 for clean_func in cls._cleanup_funcs:
583 cls._restore_dict(cls._orig_environ, os.environ)
584 cls._restore_dict(cls._orig_config, arvados.config.settings())
587 if __name__ == "__main__":
590 'start_keep', 'stop_keep',
591 'start_keep_proxy', 'stop_keep_proxy',
592 'start_arv-git-httpd', 'stop_arv-git-httpd',
593 'start_nginx', 'stop_nginx',
595 parser = argparse.ArgumentParser()
596 parser.add_argument('action', type=str, help="one of {}".format(actions))
597 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
598 args = parser.parse_args()
600 if args.action not in actions:
601 print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr)
603 if args.action == 'start':
604 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
605 run(leave_running_atexit=True)
606 host = os.environ['ARVADOS_API_HOST']
607 if args.auth is not None:
608 token = auth_token(args.auth)
609 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
610 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
611 print("export ARVADOS_API_HOST_INSECURE=true")
614 elif args.action == 'stop':
615 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
616 elif args.action == 'start_keep':
618 elif args.action == 'stop_keep':
620 elif args.action == 'start_keep_proxy':
622 elif args.action == 'stop_keep_proxy':
624 elif args.action == 'start_arv-git-httpd':
626 elif args.action == 'stop_arv-git-httpd':
628 elif args.action == 'start_nginx':
630 elif args.action == 'stop_nginx':
633 raise Exception("action recognized but not implemented!?")