1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import print_function
6 from __future__ import division
7 from builtins import str
8 from builtins import range
29 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
30 if __name__ == '__main__' and os.path.exists(
31 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
32 # We're being launched to support another test suite.
33 # Add the Python SDK source to the library path.
34 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
39 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
40 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
41 if 'GOPATH' in os.environ:
42 # Add all GOPATH bin dirs to PATH -- but insert them after the
43 # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
44 # command, not the golang.org/x/tools/cmd/bundle command.
45 gopaths = os.environ['GOPATH'].split(':')
46 addbins = [os.path.join(path, 'bin') for path in gopaths]
48 for path in os.environ['PATH'].split(':'):
50 if os.path.exists(os.path.join(path, 'bundle')):
54 os.environ['PATH'] = ':'.join(newbins)
56 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
57 if not os.path.exists(TEST_TMPDIR):
62 _cached_db_config = {}
64 def find_server_pid(PID_PATH, wait=10):
68 while (not good_pid) and (now <= timeout):
71 with open(PID_PATH, 'r') as f:
72 server_pid = int(f.read())
73 good_pid = (os.kill(server_pid, 0) is None)
74 except EnvironmentError:
83 def kill_server_pid(pidfile, wait=10, passenger_root=False):
84 # Must re-import modules in order to work during atexit
95 # First try to shut down nicely
96 restore_cwd = os.getcwd()
97 os.chdir(passenger_root)
99 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
100 os.chdir(restore_cwd)
101 # Use up to half of the +wait+ period waiting for "passenger
102 # stop" to work. If the process hasn't exited by then, start
103 # sending TERM signals.
107 while now <= deadline and server_pid is None:
109 with open(pidfile, 'r') as f:
110 server_pid = int(f.read())
112 # No pidfile = nothing to kill.
114 except ValueError as error:
115 # Pidfile exists, but we can't parse it. Perhaps the
116 # server has created the file but hasn't written its PID
118 print("Parse error reading pidfile {}: {}".format(pidfile, error),
123 while now <= deadline:
125 exited, _ = os.waitpid(server_pid, os.WNOHANG)
127 _remove_pidfile(pidfile)
130 # already exited, or isn't our child process
134 os.kill(server_pid, signal.SIGTERM)
135 print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
137 except OSError as error:
138 if error.errno == errno.ESRCH:
139 # Thrown by os.getpgid() or os.kill() if the process
140 # does not exist, i.e., our work here is done.
141 _remove_pidfile(pidfile)
147 print("Server PID {} ({}) did not exit, giving up after {}s".
148 format(server_pid, pidfile, wait),
151 def _remove_pidfile(pidfile):
155 if os.path.lexists(pidfile):
158 def find_available_port():
159 """Return an IPv4 port number that is not in use right now.
161 We assume whoever needs to use the returned port is able to reuse
162 a recently used port without waiting for TIME_WAIT (see
163 SO_REUSEADDR / SO_REUSEPORT).
165 Some opportunity for races here, but it's better than choosing
166 something at random and not checking at all. If all of our servers
167 (hey Passenger) knew that listening on port 0 was a thing, the OS
168 would take care of the races, and this wouldn't be needed at all.
171 sock = socket.socket()
172 sock.bind(('0.0.0.0', 0))
173 port = sock.getsockname()[1]
177 def _wait_until_port_listens(port, timeout=10, warn=True):
178 """Wait for a process to start listening on the given port.
180 If nothing listens on the port within the specified timeout (given
181 in seconds), print a warning on stderr before returning.
184 subprocess.check_output(['which', 'netstat'])
185 except subprocess.CalledProcessError:
186 print("WARNING: No `netstat` -- cannot wait for port to listen. "+
187 "Sleeping 0.5 and hoping for the best.",
191 deadline = time.time() + timeout
192 while time.time() < deadline:
193 if re.search(r'\ntcp.*:'+str(port)+' .* LISTEN *\n', subprocess.check_output(['netstat', '-Wln']).decode()):
198 "WARNING: Nothing is listening on port {} (waited {} seconds).".
199 format(port, timeout),
203 def _logfilename(label):
204 """Set up a labelled log file, and return a path to write logs to.
206 Normally, the returned path is {tmpdir}/{label}.log.
208 In debug mode, logs are also written to stderr, with [label]
209 prepended to each line. The returned path is a FIFO.
211 +label+ should contain only alphanumerics: it is also used as part
212 of the FIFO filename.
215 logfilename = os.path.join(TEST_TMPDIR, label+'.log')
216 if not os.environ.get('ARVADOS_DEBUG', ''):
218 fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
221 except OSError as error:
222 if error.errno != errno.ENOENT:
224 os.mkfifo(fifo, 0o700)
225 stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
226 # open(fifo, 'r') would block waiting for someone to open the fifo
227 # for writing, so we need a separate cat process to open it for
229 cat = subprocess.Popen(
230 stdbuf+['cat', fifo],
231 stdin=open('/dev/null'),
232 stdout=subprocess.PIPE)
233 tee = subprocess.Popen(
234 stdbuf+['tee', '-a', logfilename],
236 stdout=subprocess.PIPE)
238 stdbuf+['sed', '-e', 's/^/['+label+'] /'],
243 def run(leave_running_atexit=False):
244 """Ensure an API server is running, and ARVADOS_API_* env vars have
245 admin credentials for it.
247 If ARVADOS_TEST_API_HOST is set, a parent process has started a
248 test server for us to use: we just need to reset() it using the
251 If a previous call to run() started a new server process, and it
252 is still running, we just need to reset() it to fixture state and
255 If neither of those options work out, we'll really start a new
260 # Delete cached discovery documents.
262 # This will clear cached docs that belong to other processes (like
263 # concurrent test suites) even if they're still running. They should
264 # be able to tolerate that.
265 for fn in glob.glob(os.path.join(
266 str(arvados.http_cache('discovery')),
267 '*,arvados,v1,rest,*')):
270 pid_file = _pidfile('api')
271 pid_file_ok = find_server_pid(pid_file, 0)
273 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
274 if existing_api_host and pid_file_ok:
275 if existing_api_host == my_api_host:
279 # Fall through to shutdown-and-start case.
282 # Server was provided by parent. Can't recover if it's
286 # Before trying to start up our own server, call stop() to avoid
287 # "Phusion Passenger Standalone is already running on PID 12345".
288 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
289 # we know the server is ours to kill.)
292 restore_cwd = os.getcwd()
293 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
294 os.chdir(api_src_dir)
296 # Either we haven't started a server of our own yet, or it has
297 # died, or we have lost our credentials, or something else is
298 # preventing us from calling reset(). Start a new one.
300 if not os.path.exists('tmp'):
303 if not os.path.exists('tmp/api'):
304 os.makedirs('tmp/api')
306 if not os.path.exists('tmp/logs'):
307 os.makedirs('tmp/logs')
309 # Install the git repository fixtures.
310 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
311 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
312 if not os.path.isdir(gitdir):
314 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
316 # The nginx proxy isn't listening here yet, but we need to choose
317 # the wss:// port now so we can write the API server config file.
318 wss_port = find_available_port()
319 _setport('wss', wss_port)
321 port = find_available_port()
322 env = os.environ.copy()
323 env['RAILS_ENV'] = 'test'
324 env['ARVADOS_TEST_WSS_PORT'] = str(wss_port)
325 env.pop('ARVADOS_WEBSOCKETS', None)
326 env.pop('ARVADOS_TEST_API_HOST', None)
327 env.pop('ARVADOS_API_HOST', None)
328 env.pop('ARVADOS_API_HOST_INSECURE', None)
329 env.pop('ARVADOS_API_TOKEN', None)
330 start_msg = subprocess.check_output(
332 'passenger', 'start', '-d', '-p{}'.format(port),
333 '--pid-file', pid_file,
334 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
336 '--ssl-certificate', 'tmp/self-signed.pem',
337 '--ssl-certificate-key', 'tmp/self-signed.key'],
340 if not leave_running_atexit:
341 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
343 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
346 "Passenger did not report endpoint: {}".format(start_msg))
347 my_api_host = match.group(1)
348 os.environ['ARVADOS_API_HOST'] = my_api_host
350 # Make sure the server has written its pid file and started
351 # listening on its TCP port
352 find_server_pid(pid_file)
353 _wait_until_port_listens(port)
356 os.chdir(restore_cwd)
359 """Reset the test server to fixture state.
361 This resets the ARVADOS_TEST_API_HOST provided by a parent process
362 if any, otherwise the server started by run().
364 It also resets ARVADOS_* environment vars to point to the test
365 server with admin credentials.
367 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
368 token = auth_token('admin')
369 httpclient = httplib2.Http(ca_certs=os.path.join(
370 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
372 'https://{}/database/reset'.format(existing_api_host),
374 headers={'Authorization': 'OAuth2 {}'.format(token)})
375 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
376 os.environ['ARVADOS_API_TOKEN'] = token
377 if _wait_until_port_listens(_getport('controller-ssl'), timeout=0.5, warn=False):
378 os.environ['ARVADOS_API_HOST'] = '0.0.0.0:'+str(_getport('controller-ssl'))
380 os.environ['ARVADOS_API_HOST'] = existing_api_host
382 def stop(force=False):
383 """Stop the API server, if one is running.
385 If force==False, kill it only if we started it ourselves. (This
386 supports the use case where a Python test suite calls run(), but
387 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
388 process, and the test suite cleans up after itself by calling
389 stop(). In this case the test server provided by the parent
390 process should be left alone.)
392 If force==True, kill it even if we didn't start it
393 ourselves. (This supports the use case in __main__, where "run"
394 and "stop" happen in different processes.)
397 if force or my_api_host is not None:
398 kill_server_pid(_pidfile('api'))
401 def run_controller():
402 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
405 rails_api_port = int(string.split(os.environ.get('ARVADOS_TEST_API_HOST', my_api_host), ':')[-1])
406 port = find_available_port()
407 conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
408 with open(conf, 'w') as f:
412 ManagementToken: e687950a23c3a9bceec28c6223a06c79
427 "http://localhost:{controllerport}": {{}}
430 "https://localhost:{railsport}": {{}}
432 dbhost=_dbconfig('host'),
433 dbname=_dbconfig('database'),
434 dbuser=_dbconfig('username'),
435 dbpass=_dbconfig('password'),
437 railsport=rails_api_port,
439 logf = open(_logfilename('controller'), 'a')
440 controller = subprocess.Popen(
441 ["arvados-server", "controller", "-config", conf],
442 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
443 with open(_pidfile('controller'), 'w') as f:
444 f.write(str(controller.pid))
445 _wait_until_port_listens(port)
446 _setport('controller', port)
449 def stop_controller():
450 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
452 kill_server_pid(_pidfile('controller'))
455 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
458 port = find_available_port()
459 conf = os.path.join(TEST_TMPDIR, 'ws.yml')
460 with open(conf, 'w') as f:
473 """.format(os.environ['ARVADOS_API_HOST'],
475 ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
477 _dbconfig('database'),
478 _dbconfig('username'),
479 _dbconfig('password')))
480 logf = open(_logfilename('ws'), 'a')
481 ws = subprocess.Popen(
482 ["ws", "-config", conf],
483 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
484 with open(_pidfile('ws'), 'w') as f:
486 _wait_until_port_listens(port)
491 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
493 kill_server_pid(_pidfile('ws'))
495 def _start_keep(n, keep_args):
496 keep0 = tempfile.mkdtemp()
497 port = find_available_port()
498 keep_cmd = ["keepstore",
499 "-volume={}".format(keep0),
500 "-listen=:{}".format(port),
501 "-pid="+_pidfile('keep{}'.format(n))]
503 for arg, val in keep_args.items():
504 keep_cmd.append("{}={}".format(arg, val))
506 with open(_logfilename('keep{}'.format(n)), 'a') as logf:
507 with open('/dev/null') as _stdin:
508 kp0 = subprocess.Popen(
509 keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
511 with open(_pidfile('keep{}'.format(n)), 'w') as f:
512 f.write(str(kp0.pid))
514 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
517 _wait_until_port_listens(port)
521 def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
522 stop_keep(num_servers)
525 if not blob_signing_key:
526 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
527 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
528 keep_args['-blob-signing-key-file'] = f.name
529 f.write(blob_signing_key)
530 keep_args['-enforce-permissions'] = str(enforce_permissions).lower()
531 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
532 keep_args['-data-manager-token-file'] = f.name
533 f.write(auth_token('data_manager'))
534 keep_args['-never-delete'] = 'false'
538 host=os.environ['ARVADOS_API_HOST'],
539 token=os.environ['ARVADOS_API_TOKEN'],
542 for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']:
543 api.keep_services().delete(uuid=d['uuid']).execute()
544 for d in api.keep_disks().list().execute()['items']:
545 api.keep_disks().delete(uuid=d['uuid']).execute()
547 for d in range(0, num_servers):
548 port = _start_keep(d, keep_args)
549 svc = api.keep_services().create(body={'keep_service': {
550 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
551 'service_host': 'localhost',
552 'service_port': port,
553 'service_type': 'disk',
554 'service_ssl_flag': False,
556 api.keep_disks().create(body={
557 'keep_disk': {'keep_service_uuid': svc['uuid'] }
560 # If keepproxy and/or keep-web is running, send SIGHUP to make
561 # them discover the new keepstore services.
562 for svc in ('keepproxy', 'keep-web'):
563 pidfile = _pidfile('keepproxy')
564 if os.path.exists(pidfile):
566 with open(pidfile) as pid:
567 os.kill(int(pid.read()), signal.SIGHUP)
572 kill_server_pid(_pidfile('keep{}'.format(n)))
573 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
574 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
575 shutil.rmtree(r.read(), True)
576 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
577 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
578 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
580 def stop_keep(num_servers=2):
581 for n in range(0, num_servers):
584 def run_keep_proxy():
585 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
586 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(_getport('keepproxy'))
590 port = find_available_port()
591 env = os.environ.copy()
592 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
593 logf = open(_logfilename('keepproxy'), 'a')
594 kp = subprocess.Popen(
596 '-pid='+_pidfile('keepproxy'),
597 '-listen=:{}'.format(port)],
598 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
602 host=os.environ['ARVADOS_API_HOST'],
603 token=auth_token('admin'),
605 for d in api.keep_services().list(
606 filters=[['service_type','=','proxy']]).execute()['items']:
607 api.keep_services().delete(uuid=d['uuid']).execute()
608 api.keep_services().create(body={'keep_service': {
609 'service_host': 'localhost',
610 'service_port': port,
611 'service_type': 'proxy',
612 'service_ssl_flag': False,
614 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
615 _setport('keepproxy', port)
616 _wait_until_port_listens(port)
618 def stop_keep_proxy():
619 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
621 kill_server_pid(_pidfile('keepproxy'))
623 def run_arv_git_httpd():
624 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
628 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
629 gitport = find_available_port()
630 env = os.environ.copy()
631 env.pop('ARVADOS_API_TOKEN', None)
632 logf = open(_logfilename('arv-git-httpd'), 'a')
633 agh = subprocess.Popen(
635 '-repo-root='+gitdir+'/test',
636 '-management-token=e687950a23c3a9bceec28c6223a06c79',
637 '-address=:'+str(gitport)],
638 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
639 with open(_pidfile('arv-git-httpd'), 'w') as f:
640 f.write(str(agh.pid))
641 _setport('arv-git-httpd', gitport)
642 _wait_until_port_listens(gitport)
644 def stop_arv_git_httpd():
645 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
647 kill_server_pid(_pidfile('arv-git-httpd'))
650 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
654 keepwebport = find_available_port()
655 env = os.environ.copy()
656 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
657 logf = open(_logfilename('keep-web'), 'a')
658 keepweb = subprocess.Popen(
661 '-attachment-only-host=download',
662 '-management-token=e687950a23c3a9bceec28c6223a06c79',
663 '-listen=:'+str(keepwebport)],
664 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
665 with open(_pidfile('keep-web'), 'w') as f:
666 f.write(str(keepweb.pid))
667 _setport('keep-web', keepwebport)
668 _wait_until_port_listens(keepwebport)
671 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
673 kill_server_pid(_pidfile('keep-web'))
676 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
680 nginxconf['CONTROLLERPORT'] = _getport('controller')
681 nginxconf['CONTROLLERSSLPORT'] = find_available_port()
682 nginxconf['KEEPWEBPORT'] = _getport('keep-web')
683 nginxconf['KEEPWEBDLSSLPORT'] = find_available_port()
684 nginxconf['KEEPWEBSSLPORT'] = find_available_port()
685 nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
686 nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
687 nginxconf['GITPORT'] = _getport('arv-git-httpd')
688 nginxconf['GITSSLPORT'] = find_available_port()
689 nginxconf['WSPORT'] = _getport('ws')
690 nginxconf['WSSPORT'] = _getport('wss')
691 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
692 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
693 nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
694 nginxconf['ERRORLOG'] = _logfilename('nginx_error')
695 nginxconf['TMPDIR'] = TEST_TMPDIR
697 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
698 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
699 with open(conffile, 'w') as f:
702 lambda match: str(nginxconf.get(match.group(1))),
703 open(conftemplatefile).read()))
705 env = os.environ.copy()
706 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
708 nginx = subprocess.Popen(
710 '-g', 'error_log stderr info;',
711 '-g', 'pid '+_pidfile('nginx')+';',
713 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
714 _setport('controller-ssl', nginxconf['CONTROLLERSSLPORT'])
715 _setport('keep-web-dl-ssl', nginxconf['KEEPWEBDLSSLPORT'])
716 _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
717 _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
718 _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
721 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
723 kill_server_pid(_pidfile('nginx'))
725 def _pidfile(program):
726 return os.path.join(TEST_TMPDIR, program + '.pid')
728 def _portfile(program):
729 return os.path.join(TEST_TMPDIR, program + '.port')
731 def _setport(program, port):
732 with open(_portfile(program), 'w') as f:
735 # Returns 9 if program is not up.
736 def _getport(program):
738 with open(_portfile(program)) as prog:
739 return int(prog.read())
744 global _cached_db_config
745 if not _cached_db_config:
746 _cached_db_config = yaml.safe_load(open(os.path.join(
747 SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))
748 return _cached_db_config['test'][key]
751 global _cached_config
753 return _cached_config[key]
754 def _load(f, required=True):
755 fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
756 if not required and not os.path.exists(fullpath):
758 return yaml.safe_load(fullpath)
759 cdefault = _load('application.default.yml')
760 csite = _load('application.yml', required=False)
762 for section in [cdefault.get('common',{}), cdefault.get('test',{}),
763 csite.get('common',{}), csite.get('test',{})]:
764 _cached_config.update(section)
765 return _cached_config[key]
768 '''load a fixture yaml file'''
769 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
773 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
774 yaml_file = yaml_file[0:trim_index]
777 return yaml.safe_load(yaml_file)
779 def auth_token(token_name):
780 return fixture("api_client_authorizations")[token_name]["api_token"]
782 def authorize_with(token_name):
783 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
784 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
785 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
786 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
788 class TestCaseWithServers(unittest.TestCase):
789 """TestCase to start and stop supporting Arvados servers.
791 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
792 class variables as a dictionary of keyword arguments. If you do,
793 setUpClass will start the corresponding servers by passing these
794 keyword arguments to the run, run_keep, and/or run_keep_server
795 functions, respectively. It will also set Arvados environment
796 variables to point to these servers appropriately. If you don't
797 run a Keep or Keep proxy server, setUpClass will set up a
798 temporary directory for Keep local storage, and set it as
801 tearDownClass will stop any servers started, and restore the
802 original environment.
807 KEEP_PROXY_SERVER = None
808 KEEP_WEB_SERVER = None
811 def _restore_dict(src, dest):
812 for key in list(dest.keys()):
819 cls._orig_environ = os.environ.copy()
820 cls._orig_config = arvados.config.settings().copy()
821 cls._cleanup_funcs = []
822 os.environ.pop('ARVADOS_KEEP_SERVICES', None)
823 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
824 for server_kwargs, start_func, stop_func in (
825 (cls.MAIN_SERVER, run, reset),
826 (cls.WS_SERVER, run_ws, stop_ws),
827 (cls.KEEP_SERVER, run_keep, stop_keep),
828 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
829 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
830 if server_kwargs is not None:
831 start_func(**server_kwargs)
832 cls._cleanup_funcs.append(stop_func)
833 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
834 cls.local_store = tempfile.mkdtemp()
835 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
836 cls._cleanup_funcs.append(
837 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
839 os.environ.pop('KEEP_LOCAL_STORE', None)
840 arvados.config.initialize()
843 def tearDownClass(cls):
844 for clean_func in cls._cleanup_funcs:
846 cls._restore_dict(cls._orig_environ, os.environ)
847 cls._restore_dict(cls._orig_config, arvados.config.settings())
850 if __name__ == "__main__":
853 'start_ws', 'stop_ws',
854 'start_controller', 'stop_controller',
855 'start_keep', 'stop_keep',
856 'start_keep_proxy', 'stop_keep_proxy',
857 'start_keep-web', 'stop_keep-web',
858 'start_arv-git-httpd', 'stop_arv-git-httpd',
859 'start_nginx', 'stop_nginx',
861 parser = argparse.ArgumentParser()
862 parser.add_argument('action', type=str, help="one of {}".format(actions))
863 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
864 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
865 parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions")
867 args = parser.parse_args()
869 if args.action not in actions:
870 print("Unrecognized action '{}'. Actions are: {}.".
871 format(args.action, actions),
874 if args.action == 'start':
875 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
876 run(leave_running_atexit=True)
877 host = os.environ['ARVADOS_API_HOST']
878 if args.auth is not None:
879 token = auth_token(args.auth)
880 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
881 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
882 print("export ARVADOS_API_HOST_INSECURE=true")
885 elif args.action == 'stop':
886 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
887 elif args.action == 'start_ws':
889 elif args.action == 'stop_ws':
891 elif args.action == 'start_controller':
893 elif args.action == 'stop_controller':
895 elif args.action == 'start_keep':
896 run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
897 elif args.action == 'stop_keep':
898 stop_keep(num_servers=args.num_keep_servers)
899 elif args.action == 'start_keep_proxy':
901 elif args.action == 'stop_keep_proxy':
903 elif args.action == 'start_arv-git-httpd':
905 elif args.action == 'stop_arv-git-httpd':
907 elif args.action == 'start_keep-web':
909 elif args.action == 'stop_keep-web':
911 elif args.action == 'start_nginx':
913 print("export ARVADOS_API_HOST=0.0.0.0:{}".format(_getport('controller-ssl')))
914 elif args.action == 'stop_nginx':
917 raise Exception("action recognized but not implemented!?")