1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import print_function
6 from __future__ import division
7 from builtins import str
8 from builtins import range
29 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
30 if __name__ == '__main__' and os.path.exists(
31 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
32 # We're being launched to support another test suite.
33 # Add the Python SDK source to the library path.
34 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
39 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
40 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
41 if 'GOPATH' in os.environ:
42 # Add all GOPATH bin dirs to PATH -- but insert them after the
43 # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
44 # command, not the golang.org/x/tools/cmd/bundle command.
45 gopaths = os.environ['GOPATH'].split(':')
46 addbins = [os.path.join(path, 'bin') for path in gopaths]
48 for path in os.environ['PATH'].split(':'):
50 if os.path.exists(os.path.join(path, 'bundle')):
54 os.environ['PATH'] = ':'.join(newbins)
56 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
57 if not os.path.exists(TEST_TMPDIR):
62 _cached_db_config = {}
64 def find_server_pid(PID_PATH, wait=10):
68 while (not good_pid) and (now <= timeout):
71 with open(PID_PATH, 'r') as f:
72 server_pid = int(f.read())
73 good_pid = (os.kill(server_pid, 0) is None)
74 except EnvironmentError:
83 def kill_server_pid(pidfile, wait=10, passenger_root=False):
84 # Must re-import modules in order to work during atexit
95 # First try to shut down nicely
96 restore_cwd = os.getcwd()
97 os.chdir(passenger_root)
99 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
100 os.chdir(restore_cwd)
101 # Use up to half of the +wait+ period waiting for "passenger
102 # stop" to work. If the process hasn't exited by then, start
103 # sending TERM signals.
107 while now <= deadline and server_pid is None:
109 with open(pidfile, 'r') as f:
110 server_pid = int(f.read())
112 # No pidfile = nothing to kill.
114 except ValueError as error:
115 # Pidfile exists, but we can't parse it. Perhaps the
116 # server has created the file but hasn't written its PID
118 print("Parse error reading pidfile {}: {}".format(pidfile, error),
123 while now <= deadline:
125 exited, _ = os.waitpid(server_pid, os.WNOHANG)
127 _remove_pidfile(pidfile)
130 # already exited, or isn't our child process
134 os.kill(server_pid, signal.SIGTERM)
135 print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
137 except OSError as error:
138 if error.errno == errno.ESRCH:
139 # Thrown by os.getpgid() or os.kill() if the process
140 # does not exist, i.e., our work here is done.
141 _remove_pidfile(pidfile)
147 print("Server PID {} ({}) did not exit, giving up after {}s".
148 format(server_pid, pidfile, wait),
151 def _remove_pidfile(pidfile):
155 if os.path.lexists(pidfile):
158 def find_available_port():
159 """Return an IPv4 port number that is not in use right now.
161 We assume whoever needs to use the returned port is able to reuse
162 a recently used port without waiting for TIME_WAIT (see
163 SO_REUSEADDR / SO_REUSEPORT).
165 Some opportunity for races here, but it's better than choosing
166 something at random and not checking at all. If all of our servers
167 (hey Passenger) knew that listening on port 0 was a thing, the OS
168 would take care of the races, and this wouldn't be needed at all.
171 sock = socket.socket()
172 sock.bind(('0.0.0.0', 0))
173 port = sock.getsockname()[1]
177 def _wait_until_port_listens(port, timeout=10, warn=True):
178 """Wait for a process to start listening on the given port.
180 If nothing listens on the port within the specified timeout (given
181 in seconds), print a warning on stderr before returning.
184 subprocess.check_output(['which', 'lsof'])
185 except subprocess.CalledProcessError:
186 print("WARNING: No `lsof` -- cannot wait for port to listen. "+
187 "Sleeping 0.5 and hoping for the best.",
191 deadline = time.time() + timeout
192 while time.time() < deadline:
194 subprocess.check_output(
195 ['lsof', '-t', '-i', 'tcp:'+str(port)])
196 except subprocess.CalledProcessError:
202 "WARNING: Nothing is listening on port {} (waited {} seconds).".
203 format(port, timeout),
207 def _logfilename(label):
208 """Set up a labelled log file, and return a path to write logs to.
210 Normally, the returned path is {tmpdir}/{label}.log.
212 In debug mode, logs are also written to stderr, with [label]
213 prepended to each line. The returned path is a FIFO.
215 +label+ should contain only alphanumerics: it is also used as part
216 of the FIFO filename.
219 logfilename = os.path.join(TEST_TMPDIR, label+'.log')
220 if not os.environ.get('ARVADOS_DEBUG', ''):
222 fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
225 except OSError as error:
226 if error.errno != errno.ENOENT:
228 os.mkfifo(fifo, 0o700)
229 stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
230 # open(fifo, 'r') would block waiting for someone to open the fifo
231 # for writing, so we need a separate cat process to open it for
233 cat = subprocess.Popen(
234 stdbuf+['cat', fifo],
235 stdin=open('/dev/null'),
236 stdout=subprocess.PIPE)
237 tee = subprocess.Popen(
238 stdbuf+['tee', '-a', logfilename],
240 stdout=subprocess.PIPE)
242 stdbuf+['sed', '-e', 's/^/['+label+'] /'],
247 def run(leave_running_atexit=False):
248 """Ensure an API server is running, and ARVADOS_API_* env vars have
249 admin credentials for it.
251 If ARVADOS_TEST_API_HOST is set, a parent process has started a
252 test server for us to use: we just need to reset() it using the
255 If a previous call to run() started a new server process, and it
256 is still running, we just need to reset() it to fixture state and
259 If neither of those options work out, we'll really start a new
264 # Delete cached discovery documents.
266 # This will clear cached docs that belong to other processes (like
267 # concurrent test suites) even if they're still running. They should
268 # be able to tolerate that.
269 for fn in glob.glob(os.path.join(
270 str(arvados.http_cache('discovery')),
271 '*,arvados,v1,rest,*')):
274 pid_file = _pidfile('api')
275 pid_file_ok = find_server_pid(pid_file, 0)
277 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
278 if existing_api_host and pid_file_ok:
279 if existing_api_host == my_api_host:
283 # Fall through to shutdown-and-start case.
286 # Server was provided by parent. Can't recover if it's
290 # Before trying to start up our own server, call stop() to avoid
291 # "Phusion Passenger Standalone is already running on PID 12345".
292 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
293 # we know the server is ours to kill.)
296 restore_cwd = os.getcwd()
297 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
298 os.chdir(api_src_dir)
300 # Either we haven't started a server of our own yet, or it has
301 # died, or we have lost our credentials, or something else is
302 # preventing us from calling reset(). Start a new one.
304 if not os.path.exists('tmp'):
307 if not os.path.exists('tmp/api'):
308 os.makedirs('tmp/api')
310 if not os.path.exists('tmp/logs'):
311 os.makedirs('tmp/logs')
313 # Install the git repository fixtures.
314 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
315 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
316 if not os.path.isdir(gitdir):
318 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
320 # The nginx proxy isn't listening here yet, but we need to choose
321 # the wss:// port now so we can write the API server config file.
322 wss_port = find_available_port()
323 _setport('wss', wss_port)
325 port = find_available_port()
326 env = os.environ.copy()
327 env['RAILS_ENV'] = 'test'
328 env['ARVADOS_TEST_WSS_PORT'] = str(wss_port)
329 env.pop('ARVADOS_WEBSOCKETS', None)
330 env.pop('ARVADOS_TEST_API_HOST', None)
331 env.pop('ARVADOS_API_HOST', None)
332 env.pop('ARVADOS_API_HOST_INSECURE', None)
333 env.pop('ARVADOS_API_TOKEN', None)
334 start_msg = subprocess.check_output(
336 'passenger', 'start', '-d', '-p{}'.format(port),
337 '--pid-file', pid_file,
338 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
340 '--ssl-certificate', 'tmp/self-signed.pem',
341 '--ssl-certificate-key', 'tmp/self-signed.key'],
344 if not leave_running_atexit:
345 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
347 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
350 "Passenger did not report endpoint: {}".format(start_msg))
351 my_api_host = match.group(1)
352 os.environ['ARVADOS_API_HOST'] = my_api_host
354 # Make sure the server has written its pid file and started
355 # listening on its TCP port
356 find_server_pid(pid_file)
357 _wait_until_port_listens(port)
360 os.chdir(restore_cwd)
363 """Reset the test server to fixture state.
365 This resets the ARVADOS_TEST_API_HOST provided by a parent process
366 if any, otherwise the server started by run().
368 It also resets ARVADOS_* environment vars to point to the test
369 server with admin credentials.
371 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
372 token = auth_token('admin')
373 httpclient = httplib2.Http(ca_certs=os.path.join(
374 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
376 'https://{}/database/reset'.format(existing_api_host),
378 headers={'Authorization': 'OAuth2 {}'.format(token)})
379 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
380 os.environ['ARVADOS_API_TOKEN'] = token
381 if _wait_until_port_listens(_getport('controller-ssl'), timeout=0.5, warn=False):
382 os.environ['ARVADOS_API_HOST'] = '0.0.0.0:'+str(_getport('controller-ssl'))
384 os.environ['ARVADOS_API_HOST'] = existing_api_host
386 def stop(force=False):
387 """Stop the API server, if one is running.
389 If force==False, kill it only if we started it ourselves. (This
390 supports the use case where a Python test suite calls run(), but
391 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
392 process, and the test suite cleans up after itself by calling
393 stop(). In this case the test server provided by the parent
394 process should be left alone.)
396 If force==True, kill it even if we didn't start it
397 ourselves. (This supports the use case in __main__, where "run"
398 and "stop" happen in different processes.)
401 if force or my_api_host is not None:
402 kill_server_pid(_pidfile('api'))
405 def run_controller():
406 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
409 rails_api_port = int(string.split(os.environ.get('ARVADOS_TEST_API_HOST', my_api_host), ':')[-1])
410 port = find_available_port()
411 conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
412 with open(conf, 'w') as f:
416 EnableBetaController14287: {beta14287}
417 ManagementToken: e687950a23c3a9bceec28c6223a06c79
422 HTTPRequestTimeout: 30s
435 "http://localhost:{controllerport}": {{}}
438 "https://localhost:{railsport}": {{}}
440 beta14287=('true' if '14287' in os.environ.get('ARVADOS_EXPERIMENTAL', '') else 'false'),
441 loglevel=('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
442 dbhost=_dbconfig('host'),
443 dbname=_dbconfig('database'),
444 dbuser=_dbconfig('username'),
445 dbpass=_dbconfig('password'),
447 railsport=rails_api_port,
449 logf = open(_logfilename('controller'), 'a')
450 controller = subprocess.Popen(
451 ["arvados-server", "controller", "-config", conf],
452 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
453 with open(_pidfile('controller'), 'w') as f:
454 f.write(str(controller.pid))
455 _wait_until_port_listens(port)
456 _setport('controller', port)
459 def stop_controller():
460 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
462 kill_server_pid(_pidfile('controller'))
465 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
468 port = find_available_port()
469 conf = os.path.join(TEST_TMPDIR, 'ws.yml')
470 with open(conf, 'w') as f:
483 """.format(os.environ['ARVADOS_API_HOST'],
485 ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
487 _dbconfig('database'),
488 _dbconfig('username'),
489 _dbconfig('password')))
490 logf = open(_logfilename('ws'), 'a')
491 ws = subprocess.Popen(
492 ["ws", "-config", conf],
493 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
494 with open(_pidfile('ws'), 'w') as f:
496 _wait_until_port_listens(port)
501 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
503 kill_server_pid(_pidfile('ws'))
505 def _start_keep(n, keep_args):
506 keep0 = tempfile.mkdtemp()
507 port = find_available_port()
508 keep_cmd = ["keepstore",
509 "-volume={}".format(keep0),
510 "-listen=:{}".format(port),
511 "-pid="+_pidfile('keep{}'.format(n))]
513 for arg, val in keep_args.items():
514 keep_cmd.append("{}={}".format(arg, val))
516 logf = open(_logfilename('keep{}'.format(n)), 'a')
517 kp0 = subprocess.Popen(
518 keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
520 with open(_pidfile('keep{}'.format(n)), 'w') as f:
521 f.write(str(kp0.pid))
523 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
526 _wait_until_port_listens(port)
530 def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
531 stop_keep(num_servers)
534 if not blob_signing_key:
535 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
536 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
537 keep_args['-blob-signing-key-file'] = f.name
538 f.write(blob_signing_key)
539 keep_args['-enforce-permissions'] = str(enforce_permissions).lower()
540 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
541 keep_args['-data-manager-token-file'] = f.name
542 f.write(auth_token('data_manager'))
543 keep_args['-never-delete'] = 'false'
547 host=os.environ['ARVADOS_API_HOST'],
548 token=os.environ['ARVADOS_API_TOKEN'],
551 for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']:
552 api.keep_services().delete(uuid=d['uuid']).execute()
553 for d in api.keep_disks().list().execute()['items']:
554 api.keep_disks().delete(uuid=d['uuid']).execute()
556 for d in range(0, num_servers):
557 port = _start_keep(d, keep_args)
558 svc = api.keep_services().create(body={'keep_service': {
559 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
560 'service_host': 'localhost',
561 'service_port': port,
562 'service_type': 'disk',
563 'service_ssl_flag': False,
565 api.keep_disks().create(body={
566 'keep_disk': {'keep_service_uuid': svc['uuid'] }
569 # If keepproxy and/or keep-web is running, send SIGHUP to make
570 # them discover the new keepstore services.
571 for svc in ('keepproxy', 'keep-web'):
572 pidfile = _pidfile('keepproxy')
573 if os.path.exists(pidfile):
575 os.kill(int(open(pidfile).read()), signal.SIGHUP)
580 kill_server_pid(_pidfile('keep{}'.format(n)))
581 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
582 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
583 shutil.rmtree(r.read(), True)
584 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
585 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
586 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
588 def stop_keep(num_servers=2):
589 for n in range(0, num_servers):
592 def run_keep_proxy():
593 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
594 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(_getport('keepproxy'))
598 port = find_available_port()
599 env = os.environ.copy()
600 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
601 logf = open(_logfilename('keepproxy'), 'a')
602 kp = subprocess.Popen(
604 '-pid='+_pidfile('keepproxy'),
605 '-listen=:{}'.format(port)],
606 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
610 host=os.environ['ARVADOS_API_HOST'],
611 token=auth_token('admin'),
613 for d in api.keep_services().list(
614 filters=[['service_type','=','proxy']]).execute()['items']:
615 api.keep_services().delete(uuid=d['uuid']).execute()
616 api.keep_services().create(body={'keep_service': {
617 'service_host': 'localhost',
618 'service_port': port,
619 'service_type': 'proxy',
620 'service_ssl_flag': False,
622 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
623 _setport('keepproxy', port)
624 _wait_until_port_listens(port)
626 def stop_keep_proxy():
627 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
629 kill_server_pid(_pidfile('keepproxy'))
631 def run_arv_git_httpd():
632 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
636 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
637 gitport = find_available_port()
638 env = os.environ.copy()
639 env.pop('ARVADOS_API_TOKEN', None)
640 logf = open(_logfilename('arv-git-httpd'), 'a')
641 agh = subprocess.Popen(
643 '-repo-root='+gitdir+'/test',
644 '-management-token=e687950a23c3a9bceec28c6223a06c79',
645 '-address=:'+str(gitport)],
646 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
647 with open(_pidfile('arv-git-httpd'), 'w') as f:
648 f.write(str(agh.pid))
649 _setport('arv-git-httpd', gitport)
650 _wait_until_port_listens(gitport)
652 def stop_arv_git_httpd():
653 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
655 kill_server_pid(_pidfile('arv-git-httpd'))
658 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
662 keepwebport = find_available_port()
663 env = os.environ.copy()
664 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
665 logf = open(_logfilename('keep-web'), 'a')
666 keepweb = subprocess.Popen(
669 '-attachment-only-host=download',
670 '-management-token=e687950a23c3a9bceec28c6223a06c79',
671 '-listen=:'+str(keepwebport)],
672 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
673 with open(_pidfile('keep-web'), 'w') as f:
674 f.write(str(keepweb.pid))
675 _setport('keep-web', keepwebport)
676 _wait_until_port_listens(keepwebport)
679 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
681 kill_server_pid(_pidfile('keep-web'))
684 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
688 nginxconf['CONTROLLERPORT'] = _getport('controller')
689 nginxconf['CONTROLLERSSLPORT'] = find_available_port()
690 nginxconf['KEEPWEBPORT'] = _getport('keep-web')
691 nginxconf['KEEPWEBDLSSLPORT'] = find_available_port()
692 nginxconf['KEEPWEBSSLPORT'] = find_available_port()
693 nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
694 nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
695 nginxconf['GITPORT'] = _getport('arv-git-httpd')
696 nginxconf['GITSSLPORT'] = find_available_port()
697 nginxconf['WSPORT'] = _getport('ws')
698 nginxconf['WSSPORT'] = _getport('wss')
699 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
700 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
701 nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
702 nginxconf['ERRORLOG'] = _logfilename('nginx_error')
703 nginxconf['TMPDIR'] = TEST_TMPDIR
705 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
706 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
707 with open(conffile, 'w') as f:
710 lambda match: str(nginxconf.get(match.group(1))),
711 open(conftemplatefile).read()))
713 env = os.environ.copy()
714 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
716 nginx = subprocess.Popen(
718 '-g', 'error_log stderr info;',
719 '-g', 'pid '+_pidfile('nginx')+';',
721 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
722 _setport('controller-ssl', nginxconf['CONTROLLERSSLPORT'])
723 _setport('keep-web-dl-ssl', nginxconf['KEEPWEBDLSSLPORT'])
724 _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
725 _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
726 _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
729 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
731 kill_server_pid(_pidfile('nginx'))
733 def _pidfile(program):
734 return os.path.join(TEST_TMPDIR, program + '.pid')
736 def _portfile(program):
737 return os.path.join(TEST_TMPDIR, program + '.port')
739 def _setport(program, port):
740 with open(_portfile(program), 'w') as f:
743 # Returns 9 if program is not up.
744 def _getport(program):
746 return int(open(_portfile(program)).read())
751 global _cached_db_config
752 if not _cached_db_config:
753 _cached_db_config = yaml.safe_load(open(os.path.join(
754 SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))
755 return _cached_db_config['test'][key]
758 global _cached_config
760 return _cached_config[key]
761 def _load(f, required=True):
762 fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
763 if not required and not os.path.exists(fullpath):
765 return yaml.safe_load(fullpath)
766 cdefault = _load('application.default.yml')
767 csite = _load('application.yml', required=False)
769 for section in [cdefault.get('common',{}), cdefault.get('test',{}),
770 csite.get('common',{}), csite.get('test',{})]:
771 _cached_config.update(section)
772 return _cached_config[key]
775 '''load a fixture yaml file'''
776 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
780 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
781 yaml_file = yaml_file[0:trim_index]
784 return yaml.safe_load(yaml_file)
786 def auth_token(token_name):
787 return fixture("api_client_authorizations")[token_name]["api_token"]
789 def authorize_with(token_name):
790 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
791 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
792 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
793 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
795 class TestCaseWithServers(unittest.TestCase):
796 """TestCase to start and stop supporting Arvados servers.
798 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
799 class variables as a dictionary of keyword arguments. If you do,
800 setUpClass will start the corresponding servers by passing these
801 keyword arguments to the run, run_keep, and/or run_keep_server
802 functions, respectively. It will also set Arvados environment
803 variables to point to these servers appropriately. If you don't
804 run a Keep or Keep proxy server, setUpClass will set up a
805 temporary directory for Keep local storage, and set it as
808 tearDownClass will stop any servers started, and restore the
809 original environment.
814 KEEP_PROXY_SERVER = None
815 KEEP_WEB_SERVER = None
818 def _restore_dict(src, dest):
819 for key in list(dest.keys()):
826 cls._orig_environ = os.environ.copy()
827 cls._orig_config = arvados.config.settings().copy()
828 cls._cleanup_funcs = []
829 os.environ.pop('ARVADOS_KEEP_SERVICES', None)
830 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
831 for server_kwargs, start_func, stop_func in (
832 (cls.MAIN_SERVER, run, reset),
833 (cls.WS_SERVER, run_ws, stop_ws),
834 (cls.KEEP_SERVER, run_keep, stop_keep),
835 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
836 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
837 if server_kwargs is not None:
838 start_func(**server_kwargs)
839 cls._cleanup_funcs.append(stop_func)
840 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
841 cls.local_store = tempfile.mkdtemp()
842 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
843 cls._cleanup_funcs.append(
844 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
846 os.environ.pop('KEEP_LOCAL_STORE', None)
847 arvados.config.initialize()
850 def tearDownClass(cls):
851 for clean_func in cls._cleanup_funcs:
853 cls._restore_dict(cls._orig_environ, os.environ)
854 cls._restore_dict(cls._orig_config, arvados.config.settings())
857 if __name__ == "__main__":
860 'start_ws', 'stop_ws',
861 'start_controller', 'stop_controller',
862 'start_keep', 'stop_keep',
863 'start_keep_proxy', 'stop_keep_proxy',
864 'start_keep-web', 'stop_keep-web',
865 'start_arv-git-httpd', 'stop_arv-git-httpd',
866 'start_nginx', 'stop_nginx',
868 parser = argparse.ArgumentParser()
869 parser.add_argument('action', type=str, help="one of {}".format(actions))
870 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
871 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
872 parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions")
874 args = parser.parse_args()
876 if args.action not in actions:
877 print("Unrecognized action '{}'. Actions are: {}.".
878 format(args.action, actions),
881 if args.action == 'start':
882 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
883 run(leave_running_atexit=True)
884 host = os.environ['ARVADOS_API_HOST']
885 if args.auth is not None:
886 token = auth_token(args.auth)
887 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
888 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
889 print("export ARVADOS_API_HOST_INSECURE=true")
892 elif args.action == 'stop':
893 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
894 elif args.action == 'start_ws':
896 elif args.action == 'stop_ws':
898 elif args.action == 'start_controller':
900 elif args.action == 'stop_controller':
902 elif args.action == 'start_keep':
903 run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
904 elif args.action == 'stop_keep':
905 stop_keep(num_servers=args.num_keep_servers)
906 elif args.action == 'start_keep_proxy':
908 elif args.action == 'stop_keep_proxy':
910 elif args.action == 'start_arv-git-httpd':
912 elif args.action == 'stop_arv-git-httpd':
914 elif args.action == 'start_keep-web':
916 elif args.action == 'stop_keep-web':
918 elif args.action == 'start_nginx':
920 print("export ARVADOS_API_HOST=0.0.0.0:{}".format(_getport('controller-ssl')))
921 elif args.action == 'stop_nginx':
924 raise Exception("action recognized but not implemented!?")