3 from __future__ import print_function
22 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
23 if __name__ == '__main__' and os.path.exists(
24 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
25 # We're being launched to support another test suite.
26 # Add the Python SDK source to the library path.
27 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
32 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
33 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
34 SERVER_PID_PATH = 'tmp/pids/test-server.pid'
35 if 'GOPATH' in os.environ:
36 gopaths = os.environ['GOPATH'].split(':')
37 gobins = [os.path.join(path, 'bin') for path in gopaths]
38 os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
40 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
41 if not os.path.exists(TEST_TMPDIR):
47 def find_server_pid(PID_PATH, wait=10):
51 while (not good_pid) and (now <= timeout):
54 with open(PID_PATH, 'r') as f:
55 server_pid = int(f.read())
56 good_pid = (os.kill(server_pid, 0) is None)
68 def kill_server_pid(pidfile, wait=10, passenger_root=False):
69 # Must re-import modules in order to work during atexit
76 # First try to shut down nicely
77 restore_cwd = os.getcwd()
78 os.chdir(passenger_root)
80 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
84 with open(pidfile, 'r') as f:
85 server_pid = int(f.read())
87 if not passenger_root or timeout - now < wait / 2:
88 # Half timeout has elapsed. Start sending SIGTERM
89 os.kill(server_pid, signal.SIGTERM)
90 # Raise OSError if process has disappeared
91 os.getpgid(server_pid)
99 def find_available_port():
100 """Return an IPv4 port number that is not in use right now.
102 We assume whoever needs to use the returned port is able to reuse
103 a recently used port without waiting for TIME_WAIT (see
104 SO_REUSEADDR / SO_REUSEPORT).
106 Some opportunity for races here, but it's better than choosing
107 something at random and not checking at all. If all of our servers
108 (hey Passenger) knew that listening on port 0 was a thing, the OS
109 would take care of the races, and this wouldn't be needed at all.
112 sock = socket.socket()
113 sock.bind(('0.0.0.0', 0))
114 port = sock.getsockname()[1]
118 def _wait_until_port_listens(port, timeout=10):
119 """Wait for a process to start listening on the given port.
121 If nothing listens on the port within the specified timeout (given
122 in seconds), print a warning on stderr before returning.
125 subprocess.check_output(['which', 'lsof'])
126 except subprocess.CalledProcessError:
127 print("WARNING: No `lsof` -- cannot wait for port to listen. "+
128 "Sleeping 0.5 and hoping for the best.")
131 deadline = time.time() + timeout
132 while time.time() < deadline:
134 subprocess.check_output(
135 ['lsof', '-t', '-i', 'tcp:'+str(port)])
136 except subprocess.CalledProcessError:
141 "WARNING: Nothing is listening on port {} (waited {} seconds).".
142 format(port, timeout),
145 def run(leave_running_atexit=False):
146 """Ensure an API server is running, and ARVADOS_API_* env vars have
147 admin credentials for it.
149 If ARVADOS_TEST_API_HOST is set, a parent process has started a
150 test server for us to use: we just need to reset() it using the
153 If a previous call to run() started a new server process, and it
154 is still running, we just need to reset() it to fixture state and
157 If neither of those options work out, we'll really start a new
162 # Delete cached discovery document.
163 shutil.rmtree(arvados.http_cache('discovery'))
165 pid_file = os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH)
166 pid_file_ok = find_server_pid(pid_file, 0)
168 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
169 if existing_api_host and pid_file_ok:
170 if existing_api_host == my_api_host:
174 # Fall through to shutdown-and-start case.
177 # Server was provided by parent. Can't recover if it's
181 # Before trying to start up our own server, call stop() to avoid
182 # "Phusion Passenger Standalone is already running on PID 12345".
183 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
184 # we know the server is ours to kill.)
187 restore_cwd = os.getcwd()
188 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
189 os.chdir(api_src_dir)
191 # Either we haven't started a server of our own yet, or it has
192 # died, or we have lost our credentials, or something else is
193 # preventing us from calling reset(). Start a new one.
195 if not os.path.exists('tmp'):
198 if not os.path.exists('tmp/api'):
199 os.makedirs('tmp/api')
201 if not os.path.exists('tmp/logs'):
202 os.makedirs('tmp/logs')
204 if not os.path.exists('tmp/self-signed.pem'):
205 # We assume here that either passenger reports its listening
206 # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
207 # then the certificate won't match the host and reset() will
208 # fail certificate verification. If it reports "localhost",
209 # clients (notably Python SDK's websocket client) might
210 # resolve localhost as ::1 and then fail to connect.
211 subprocess.check_call([
212 'openssl', 'req', '-new', '-x509', '-nodes',
213 '-out', 'tmp/self-signed.pem',
214 '-keyout', 'tmp/self-signed.key',
216 '-subj', '/CN=0.0.0.0'],
219 # Install the git repository fixtures.
220 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
221 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
222 if not os.path.isdir(gitdir):
224 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
226 port = find_available_port()
227 env = os.environ.copy()
228 env['RAILS_ENV'] = 'test'
229 env['ARVADOS_WEBSOCKETS'] = 'yes'
230 env.pop('ARVADOS_TEST_API_HOST', None)
231 env.pop('ARVADOS_API_HOST', None)
232 env.pop('ARVADOS_API_HOST_INSECURE', None)
233 env.pop('ARVADOS_API_TOKEN', None)
234 start_msg = subprocess.check_output(
236 'passenger', 'start', '-d', '-p{}'.format(port),
237 '--pid-file', os.path.join(os.getcwd(), pid_file),
238 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
240 '--ssl-certificate', 'tmp/self-signed.pem',
241 '--ssl-certificate-key', 'tmp/self-signed.key'],
244 if not leave_running_atexit:
245 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
247 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
250 "Passenger did not report endpoint: {}".format(start_msg))
251 my_api_host = match.group(1)
252 os.environ['ARVADOS_API_HOST'] = my_api_host
254 # Make sure the server has written its pid file and started
255 # listening on its TCP port
256 find_server_pid(pid_file)
257 _wait_until_port_listens(port)
260 os.chdir(restore_cwd)
263 """Reset the test server to fixture state.
265 This resets the ARVADOS_TEST_API_HOST provided by a parent process
266 if any, otherwise the server started by run().
268 It also resets ARVADOS_* environment vars to point to the test
269 server with admin credentials.
271 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
272 token = auth_token('admin')
273 httpclient = httplib2.Http(ca_certs=os.path.join(
274 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
276 'https://{}/database/reset'.format(existing_api_host),
278 headers={'Authorization': 'OAuth2 {}'.format(token)})
279 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
280 os.environ['ARVADOS_API_HOST'] = existing_api_host
281 os.environ['ARVADOS_API_TOKEN'] = token
283 def stop(force=False):
284 """Stop the API server, if one is running.
286 If force==False, kill it only if we started it ourselves. (This
287 supports the use case where a Python test suite calls run(), but
288 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
289 process, and the test suite cleans up after itself by calling
290 stop(). In this case the test server provided by the parent
291 process should be left alone.)
293 If force==True, kill it even if we didn't start it
294 ourselves. (This supports the use case in __main__, where "run"
295 and "stop" happen in different processes.)
298 if force or my_api_host is not None:
299 kill_server_pid(os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH))
302 def _start_keep(n, keep_args):
303 keep0 = tempfile.mkdtemp()
304 port = find_available_port()
305 keep_cmd = ["keepstore",
306 "-volume={}".format(keep0),
307 "-listen=:{}".format(port),
308 "-pid="+_pidfile('keep{}'.format(n))]
310 for arg, val in keep_args.iteritems():
311 keep_cmd.append("{}={}".format(arg, val))
313 logf = open(os.path.join(TEST_TMPDIR, 'keep{}.log'.format(n)), 'a+')
314 kp0 = subprocess.Popen(
315 keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
316 with open(_pidfile('keep{}'.format(n)), 'w') as f:
317 f.write(str(kp0.pid))
319 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
322 _wait_until_port_listens(port)
326 def run_keep(blob_signing_key=None, enforce_permissions=False):
327 if args.keep_existing is None:
331 if not blob_signing_key:
332 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
333 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
334 keep_args['-blob-signing-key-file'] = f.name
335 f.write(blob_signing_key)
336 if enforce_permissions:
337 keep_args['-enforce-permissions'] = 'true'
338 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
339 keep_args['-data-manager-token-file'] = f.name
340 f.write(os.environ['ARVADOS_API_TOKEN'])
341 keep_args['-never-delete'] = 'false'
345 host=os.environ['ARVADOS_API_HOST'],
346 token=os.environ['ARVADOS_API_TOKEN'],
349 for d in api.keep_services().list().execute()['items']:
350 api.keep_services().delete(uuid=d['uuid']).execute()
351 for d in api.keep_disks().list().execute()['items']:
352 api.keep_disks().delete(uuid=d['uuid']).execute()
355 if args.keep_existing is not None:
357 for d in range(start_index, start_index+2):
358 port = _start_keep(d, keep_args)
359 svc = api.keep_services().create(body={'keep_service': {
360 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
361 'service_host': 'localhost',
362 'service_port': port,
363 'service_type': 'disk',
364 'service_ssl_flag': False,
366 api.keep_disks().create(body={
367 'keep_disk': {'keep_service_uuid': svc['uuid'] }
371 kill_server_pid(_pidfile('keep{}'.format(n)), 0)
372 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
373 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
374 shutil.rmtree(r.read(), True)
375 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
376 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
377 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
382 # We may have created 2 additional keep servers when keep_existing is used
386 def run_keep_proxy():
387 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
391 admin_token = auth_token('admin')
392 port = find_available_port()
393 env = os.environ.copy()
394 env['ARVADOS_API_TOKEN'] = admin_token
395 kp = subprocess.Popen(
397 '-pid='+_pidfile('keepproxy'),
398 '-listen=:{}'.format(port)],
399 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
403 host=os.environ['ARVADOS_API_HOST'],
406 for d in api.keep_services().list(
407 filters=[['service_type','=','proxy']]).execute()['items']:
408 api.keep_services().delete(uuid=d['uuid']).execute()
409 api.keep_services().create(body={'keep_service': {
410 'service_host': 'localhost',
411 'service_port': port,
412 'service_type': 'proxy',
413 'service_ssl_flag': False,
415 os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
416 _setport('keepproxy', port)
417 _wait_until_port_listens(port)
419 def stop_keep_proxy():
420 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
422 kill_server_pid(_pidfile('keepproxy'), wait=0)
424 def run_arv_git_httpd():
425 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
429 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
430 gitport = find_available_port()
431 env = os.environ.copy()
432 env.pop('ARVADOS_API_TOKEN', None)
433 agh = subprocess.Popen(
435 '-repo-root='+gitdir+'/test',
436 '-address=:'+str(gitport)],
437 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
438 with open(_pidfile('arv-git-httpd'), 'w') as f:
439 f.write(str(agh.pid))
440 _setport('arv-git-httpd', gitport)
441 _wait_until_port_listens(gitport)
443 def stop_arv_git_httpd():
444 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
446 kill_server_pid(_pidfile('arv-git-httpd'), wait=0)
449 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
452 nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
453 nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
454 nginxconf['GITPORT'] = _getport('arv-git-httpd')
455 nginxconf['GITSSLPORT'] = find_available_port()
456 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
457 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
459 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
460 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
461 with open(conffile, 'w') as f:
464 lambda match: str(nginxconf.get(match.group(1))),
465 open(conftemplatefile).read()))
467 env = os.environ.copy()
468 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
469 nginx = subprocess.Popen(
471 '-g', 'error_log stderr info;',
472 '-g', 'pid '+_pidfile('nginx')+';',
474 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
475 _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
476 _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
479 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
481 kill_server_pid(_pidfile('nginx'), wait=0)
483 def _pidfile(program):
484 return os.path.join(TEST_TMPDIR, program + '.pid')
486 def _portfile(program):
487 return os.path.join(TEST_TMPDIR, program + '.port')
489 def _setport(program, port):
490 with open(_portfile(program), 'w') as f:
493 # Returns 9 if program is not up.
494 def _getport(program):
496 return int(open(_portfile(program)).read())
502 return _cached_config[key]
503 def _load(f, required=True):
504 fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
505 if not required and not os.path.exists(fullpath):
507 return yaml.load(fullpath)
508 cdefault = _load('application.default.yml')
509 csite = _load('application.yml', required=False)
511 for section in [cdefault.get('common',{}), cdefault.get('test',{}),
512 csite.get('common',{}), csite.get('test',{})]:
513 _cached_config.update(section)
514 return _cached_config[key]
517 '''load a fixture yaml file'''
518 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
522 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
523 yaml_file = yaml_file[0:trim_index]
526 return yaml.load(yaml_file)
528 def auth_token(token_name):
529 return fixture("api_client_authorizations")[token_name]["api_token"]
531 def authorize_with(token_name):
532 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
533 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
534 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
535 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
537 class TestCaseWithServers(unittest.TestCase):
538 """TestCase to start and stop supporting Arvados servers.
540 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
541 class variables as a dictionary of keyword arguments. If you do,
542 setUpClass will start the corresponding servers by passing these
543 keyword arguments to the run, run_keep, and/or run_keep_server
544 functions, respectively. It will also set Arvados environment
545 variables to point to these servers appropriately. If you don't
546 run a Keep or Keep proxy server, setUpClass will set up a
547 temporary directory for Keep local storage, and set it as
550 tearDownClass will stop any servers started, and restore the
551 original environment.
555 KEEP_PROXY_SERVER = None
558 def _restore_dict(src, dest):
559 for key in dest.keys():
566 cls._orig_environ = os.environ.copy()
567 cls._orig_config = arvados.config.settings().copy()
568 cls._cleanup_funcs = []
569 os.environ.pop('ARVADOS_KEEP_PROXY', None)
570 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
571 for server_kwargs, start_func, stop_func in (
572 (cls.MAIN_SERVER, run, reset),
573 (cls.KEEP_SERVER, run_keep, stop_keep),
574 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
575 if server_kwargs is not None:
576 start_func(**server_kwargs)
577 cls._cleanup_funcs.append(stop_func)
578 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
579 cls.local_store = tempfile.mkdtemp()
580 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
581 cls._cleanup_funcs.append(
582 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
584 os.environ.pop('KEEP_LOCAL_STORE', None)
585 arvados.config.initialize()
588 def tearDownClass(cls):
589 for clean_func in cls._cleanup_funcs:
591 cls._restore_dict(cls._orig_environ, os.environ)
592 cls._restore_dict(cls._orig_config, arvados.config.settings())
595 if __name__ == "__main__":
598 'start_keep', 'stop_keep',
599 'start_keep_proxy', 'stop_keep_proxy',
600 'start_arv-git-httpd', 'stop_arv-git-httpd',
601 'start_nginx', 'stop_nginx',
603 parser = argparse.ArgumentParser()
604 parser.add_argument('action', type=str, help="one of {}".format(actions))
605 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
606 parser.add_argument('--keep_existing', type=str, help="Used to add additional keep servers, without terminating existing servers")
607 args = parser.parse_args()
609 if args.action not in actions:
610 print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr)
612 if args.action == 'start':
613 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
614 run(leave_running_atexit=True)
615 host = os.environ['ARVADOS_API_HOST']
616 if args.auth is not None:
617 token = auth_token(args.auth)
618 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
619 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
620 print("export ARVADOS_API_HOST_INSECURE=true")
623 elif args.action == 'stop':
624 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
625 elif args.action == 'start_keep':
627 elif args.action == 'stop_keep':
629 elif args.action == 'start_keep_proxy':
631 elif args.action == 'stop_keep_proxy':
633 elif args.action == 'start_arv-git-httpd':
635 elif args.action == 'stop_arv-git-httpd':
637 elif args.action == 'start_nginx':
639 elif args.action == 'stop_nginx':
642 raise Exception("action recognized but not implemented!?")