3 from __future__ import print_function
23 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
24 if __name__ == '__main__' and os.path.exists(
25 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
26 # We're being launched to support another test suite.
27 # Add the Python SDK source to the library path.
28 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
33 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
34 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
35 SERVER_PID_PATH = 'tmp/pids/test-server.pid'
36 if 'GOPATH' in os.environ:
37 gopaths = os.environ['GOPATH'].split(':')
38 gobins = [os.path.join(path, 'bin') for path in gopaths]
39 os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
41 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
42 if not os.path.exists(TEST_TMPDIR):
48 def find_server_pid(PID_PATH, wait=10):
52 while (not good_pid) and (now <= timeout):
55 with open(PID_PATH, 'r') as f:
56 server_pid = int(f.read())
57 good_pid = (os.kill(server_pid, 0) is None)
58 except EnvironmentError:
67 def kill_server_pid(pidfile, wait=10, passenger_root=False):
68 # Must re-import modules in order to work during atexit
75 # First try to shut down nicely
76 restore_cwd = os.getcwd()
77 os.chdir(passenger_root)
79 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
83 with open(pidfile, 'r') as f:
84 server_pid = int(f.read())
86 if not passenger_root or timeout - now < wait / 2:
87 # Half timeout has elapsed. Start sending SIGTERM
88 os.kill(server_pid, signal.SIGTERM)
89 # Raise OSError if process has disappeared
90 os.getpgid(server_pid)
93 except EnvironmentError:
96 def find_available_port():
97 """Return an IPv4 port number that is not in use right now.
99 We assume whoever needs to use the returned port is able to reuse
100 a recently used port without waiting for TIME_WAIT (see
101 SO_REUSEADDR / SO_REUSEPORT).
103 Some opportunity for races here, but it's better than choosing
104 something at random and not checking at all. If all of our servers
105 (hey Passenger) knew that listening on port 0 was a thing, the OS
106 would take care of the races, and this wouldn't be needed at all.
109 sock = socket.socket()
110 sock.bind(('0.0.0.0', 0))
111 port = sock.getsockname()[1]
115 def _wait_until_port_listens(port, timeout=10):
116 """Wait for a process to start listening on the given port.
118 If nothing listens on the port within the specified timeout (given
119 in seconds), print a warning on stderr before returning.
122 subprocess.check_output(['which', 'lsof'])
123 except subprocess.CalledProcessError:
124 print("WARNING: No `lsof` -- cannot wait for port to listen. "+
125 "Sleeping 0.5 and hoping for the best.")
128 deadline = time.time() + timeout
129 while time.time() < deadline:
131 subprocess.check_output(
132 ['lsof', '-t', '-i', 'tcp:'+str(port)])
133 except subprocess.CalledProcessError:
138 "WARNING: Nothing is listening on port {} (waited {} seconds).".
139 format(port, timeout),
142 def run(leave_running_atexit=False):
143 """Ensure an API server is running, and ARVADOS_API_* env vars have
144 admin credentials for it.
146 If ARVADOS_TEST_API_HOST is set, a parent process has started a
147 test server for us to use: we just need to reset() it using the
150 If a previous call to run() started a new server process, and it
151 is still running, we just need to reset() it to fixture state and
154 If neither of those options work out, we'll really start a new
159 # Delete cached discovery document.
160 shutil.rmtree(arvados.http_cache('discovery'))
162 pid_file = os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH)
163 pid_file_ok = find_server_pid(pid_file, 0)
165 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
166 if existing_api_host and pid_file_ok:
167 if existing_api_host == my_api_host:
171 # Fall through to shutdown-and-start case.
174 # Server was provided by parent. Can't recover if it's
178 # Before trying to start up our own server, call stop() to avoid
179 # "Phusion Passenger Standalone is already running on PID 12345".
180 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
181 # we know the server is ours to kill.)
184 restore_cwd = os.getcwd()
185 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
186 os.chdir(api_src_dir)
188 # Either we haven't started a server of our own yet, or it has
189 # died, or we have lost our credentials, or something else is
190 # preventing us from calling reset(). Start a new one.
192 if not os.path.exists('tmp'):
195 if not os.path.exists('tmp/api'):
196 os.makedirs('tmp/api')
198 if not os.path.exists('tmp/logs'):
199 os.makedirs('tmp/logs')
201 if not os.path.exists('tmp/self-signed.pem'):
202 # We assume here that either passenger reports its listening
203 # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
204 # then the certificate won't match the host and reset() will
205 # fail certificate verification. If it reports "localhost",
206 # clients (notably Python SDK's websocket client) might
207 # resolve localhost as ::1 and then fail to connect.
208 subprocess.check_call([
209 'openssl', 'req', '-new', '-x509', '-nodes',
210 '-out', 'tmp/self-signed.pem',
211 '-keyout', 'tmp/self-signed.key',
213 '-subj', '/CN=0.0.0.0'],
216 # Install the git repository fixtures.
217 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
218 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
219 if not os.path.isdir(gitdir):
221 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
223 port = find_available_port()
224 env = os.environ.copy()
225 env['RAILS_ENV'] = 'test'
226 env['ARVADOS_WEBSOCKETS'] = 'yes'
227 env.pop('ARVADOS_TEST_API_HOST', None)
228 env.pop('ARVADOS_API_HOST', None)
229 env.pop('ARVADOS_API_HOST_INSECURE', None)
230 env.pop('ARVADOS_API_TOKEN', None)
231 start_msg = subprocess.check_output(
233 'passenger', 'start', '-d', '-p{}'.format(port),
234 '--pid-file', os.path.join(os.getcwd(), pid_file),
235 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
237 '--ssl-certificate', 'tmp/self-signed.pem',
238 '--ssl-certificate-key', 'tmp/self-signed.key'],
241 if not leave_running_atexit:
242 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
244 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
247 "Passenger did not report endpoint: {}".format(start_msg))
248 my_api_host = match.group(1)
249 os.environ['ARVADOS_API_HOST'] = my_api_host
251 # Make sure the server has written its pid file and started
252 # listening on its TCP port
253 find_server_pid(pid_file)
254 _wait_until_port_listens(port)
257 os.chdir(restore_cwd)
260 """Reset the test server to fixture state.
262 This resets the ARVADOS_TEST_API_HOST provided by a parent process
263 if any, otherwise the server started by run().
265 It also resets ARVADOS_* environment vars to point to the test
266 server with admin credentials.
268 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
269 token = auth_token('admin')
270 httpclient = httplib2.Http(ca_certs=os.path.join(
271 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
273 'https://{}/database/reset'.format(existing_api_host),
275 headers={'Authorization': 'OAuth2 {}'.format(token)})
276 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
277 os.environ['ARVADOS_API_HOST'] = existing_api_host
278 os.environ['ARVADOS_API_TOKEN'] = token
280 def stop(force=False):
281 """Stop the API server, if one is running.
283 If force==False, kill it only if we started it ourselves. (This
284 supports the use case where a Python test suite calls run(), but
285 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
286 process, and the test suite cleans up after itself by calling
287 stop(). In this case the test server provided by the parent
288 process should be left alone.)
290 If force==True, kill it even if we didn't start it
291 ourselves. (This supports the use case in __main__, where "run"
292 and "stop" happen in different processes.)
295 if force or my_api_host is not None:
296 kill_server_pid(os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH))
299 def _start_keep(n, keep_args):
300 keep0 = tempfile.mkdtemp()
301 port = find_available_port()
302 keep_cmd = ["keepstore",
303 "-volume={}".format(keep0),
304 "-listen=:{}".format(port),
305 "-pid="+_pidfile('keep{}'.format(n))]
307 for arg, val in keep_args.iteritems():
308 keep_cmd.append("{}={}".format(arg, val))
310 logf = open(os.path.join(TEST_TMPDIR, 'keep{}.log'.format(n)), 'a+')
311 kp0 = subprocess.Popen(
312 keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
313 with open(_pidfile('keep{}'.format(n)), 'w') as f:
314 f.write(str(kp0.pid))
316 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
319 _wait_until_port_listens(port)
323 def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
324 stop_keep(num_servers)
327 if not blob_signing_key:
328 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
329 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
330 keep_args['-blob-signing-key-file'] = f.name
331 f.write(blob_signing_key)
332 if enforce_permissions:
333 keep_args['-enforce-permissions'] = 'true'
334 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
335 keep_args['-data-manager-token-file'] = f.name
336 f.write(os.environ['ARVADOS_API_TOKEN'])
337 keep_args['-never-delete'] = 'false'
341 host=os.environ['ARVADOS_API_HOST'],
342 token=os.environ['ARVADOS_API_TOKEN'],
345 for d in api.keep_services().list().execute()['items']:
346 api.keep_services().delete(uuid=d['uuid']).execute()
347 for d in api.keep_disks().list().execute()['items']:
348 api.keep_disks().delete(uuid=d['uuid']).execute()
350 for d in range(0, num_servers):
351 port = _start_keep(d, keep_args)
352 svc = api.keep_services().create(body={'keep_service': {
353 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
354 'service_host': 'localhost',
355 'service_port': port,
356 'service_type': 'disk',
357 'service_ssl_flag': False,
359 api.keep_disks().create(body={
360 'keep_disk': {'keep_service_uuid': svc['uuid'] }
364 kill_server_pid(_pidfile('keep{}'.format(n)), 0)
365 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
366 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
367 shutil.rmtree(r.read(), True)
368 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
369 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
370 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
372 def stop_keep(num_servers=2):
373 for n in range(0, num_servers):
376 def run_keep_proxy():
377 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
381 admin_token = auth_token('admin')
382 port = find_available_port()
383 env = os.environ.copy()
384 env['ARVADOS_API_TOKEN'] = admin_token
385 kp = subprocess.Popen(
387 '-pid='+_pidfile('keepproxy'),
388 '-listen=:{}'.format(port)],
389 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
393 host=os.environ['ARVADOS_API_HOST'],
396 for d in api.keep_services().list(
397 filters=[['service_type','=','proxy']]).execute()['items']:
398 api.keep_services().delete(uuid=d['uuid']).execute()
399 api.keep_services().create(body={'keep_service': {
400 'service_host': 'localhost',
401 'service_port': port,
402 'service_type': 'proxy',
403 'service_ssl_flag': False,
405 os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
406 _setport('keepproxy', port)
407 _wait_until_port_listens(port)
409 def stop_keep_proxy():
410 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
412 kill_server_pid(_pidfile('keepproxy'), wait=0)
414 def run_arv_git_httpd():
415 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
419 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
420 gitport = find_available_port()
421 env = os.environ.copy()
422 env.pop('ARVADOS_API_TOKEN', None)
423 agh = subprocess.Popen(
425 '-repo-root='+gitdir+'/test',
426 '-address=:'+str(gitport)],
427 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
428 with open(_pidfile('arv-git-httpd'), 'w') as f:
429 f.write(str(agh.pid))
430 _setport('arv-git-httpd', gitport)
431 _wait_until_port_listens(gitport)
433 def stop_arv_git_httpd():
434 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
436 kill_server_pid(_pidfile('arv-git-httpd'), wait=0)
439 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
442 nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
443 nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
444 nginxconf['GITPORT'] = _getport('arv-git-httpd')
445 nginxconf['GITSSLPORT'] = find_available_port()
446 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
447 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
448 nginxconf['ACCESSLOG'] = os.path.join(TEST_TMPDIR, 'nginx_access_log.fifo')
450 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
451 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
452 with open(conffile, 'w') as f:
455 lambda match: str(nginxconf.get(match.group(1))),
456 open(conftemplatefile).read()))
458 env = os.environ.copy()
459 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
462 os.remove(nginxconf['ACCESSLOG'])
463 except OSError as error:
464 if error.errno != errno.ENOENT:
467 os.mkfifo(nginxconf['ACCESSLOG'], 0700)
468 nginx = subprocess.Popen(
470 '-g', 'error_log stderr info;',
471 '-g', 'pid '+_pidfile('nginx')+';',
473 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
474 cat_access = subprocess.Popen(
475 ['cat', nginxconf['ACCESSLOG']],
477 _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
478 _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
481 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
483 kill_server_pid(_pidfile('nginx'), wait=0)
485 def _pidfile(program):
486 return os.path.join(TEST_TMPDIR, program + '.pid')
488 def _portfile(program):
489 return os.path.join(TEST_TMPDIR, program + '.port')
491 def _setport(program, port):
492 with open(_portfile(program), 'w') as f:
495 # Returns 9 if program is not up.
496 def _getport(program):
498 return int(open(_portfile(program)).read())
504 return _cached_config[key]
505 def _load(f, required=True):
506 fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
507 if not required and not os.path.exists(fullpath):
509 return yaml.load(fullpath)
510 cdefault = _load('application.default.yml')
511 csite = _load('application.yml', required=False)
513 for section in [cdefault.get('common',{}), cdefault.get('test',{}),
514 csite.get('common',{}), csite.get('test',{})]:
515 _cached_config.update(section)
516 return _cached_config[key]
519 '''load a fixture yaml file'''
520 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
524 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
525 yaml_file = yaml_file[0:trim_index]
528 return yaml.load(yaml_file)
530 def auth_token(token_name):
531 return fixture("api_client_authorizations")[token_name]["api_token"]
533 def authorize_with(token_name):
534 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
535 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
536 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
537 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
539 class TestCaseWithServers(unittest.TestCase):
540 """TestCase to start and stop supporting Arvados servers.
542 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
543 class variables as a dictionary of keyword arguments. If you do,
544 setUpClass will start the corresponding servers by passing these
545 keyword arguments to the run, run_keep, and/or run_keep_server
546 functions, respectively. It will also set Arvados environment
547 variables to point to these servers appropriately. If you don't
548 run a Keep or Keep proxy server, setUpClass will set up a
549 temporary directory for Keep local storage, and set it as
552 tearDownClass will stop any servers started, and restore the
553 original environment.
557 KEEP_PROXY_SERVER = None
560 def _restore_dict(src, dest):
561 for key in dest.keys():
568 cls._orig_environ = os.environ.copy()
569 cls._orig_config = arvados.config.settings().copy()
570 cls._cleanup_funcs = []
571 os.environ.pop('ARVADOS_KEEP_PROXY', None)
572 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
573 for server_kwargs, start_func, stop_func in (
574 (cls.MAIN_SERVER, run, reset),
575 (cls.KEEP_SERVER, run_keep, stop_keep),
576 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
577 if server_kwargs is not None:
578 start_func(**server_kwargs)
579 cls._cleanup_funcs.append(stop_func)
580 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
581 cls.local_store = tempfile.mkdtemp()
582 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
583 cls._cleanup_funcs.append(
584 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
586 os.environ.pop('KEEP_LOCAL_STORE', None)
587 arvados.config.initialize()
590 def tearDownClass(cls):
591 for clean_func in cls._cleanup_funcs:
593 cls._restore_dict(cls._orig_environ, os.environ)
594 cls._restore_dict(cls._orig_config, arvados.config.settings())
597 if __name__ == "__main__":
600 'start_keep', 'stop_keep',
601 'start_keep_proxy', 'stop_keep_proxy',
602 'start_arv-git-httpd', 'stop_arv-git-httpd',
603 'start_nginx', 'stop_nginx',
605 parser = argparse.ArgumentParser()
606 parser.add_argument('action', type=str, help="one of {}".format(actions))
607 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
608 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
609 parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions")
611 args = parser.parse_args()
613 if args.action not in actions:
614 print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr)
616 if args.action == 'start':
617 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
618 run(leave_running_atexit=True)
619 host = os.environ['ARVADOS_API_HOST']
620 if args.auth is not None:
621 token = auth_token(args.auth)
622 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
623 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
624 print("export ARVADOS_API_HOST_INSECURE=true")
627 elif args.action == 'stop':
628 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
629 elif args.action == 'start_keep':
630 run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
631 elif args.action == 'stop_keep':
633 elif args.action == 'start_keep_proxy':
635 elif args.action == 'stop_keep_proxy':
637 elif args.action == 'start_arv-git-httpd':
639 elif args.action == 'stop_arv-git-httpd':
641 elif args.action == 'start_nginx':
643 elif args.action == 'stop_nginx':
646 raise Exception("action recognized but not implemented!?")