1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import print_function
6 from __future__ import division
7 from builtins import str
8 from builtins import range
29 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
30 if __name__ == '__main__' and os.path.exists(
31 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
32 # We're being launched to support another test suite.
33 # Add the Python SDK source to the library path.
34 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
39 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
40 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
41 if 'GOPATH' in os.environ:
42 # Add all GOPATH bin dirs to PATH -- but insert them after the
43 # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
44 # command, not the golang.org/x/tools/cmd/bundle command.
45 gopaths = os.environ['GOPATH'].split(':')
46 addbins = [os.path.join(path, 'bin') for path in gopaths]
48 for path in os.environ['PATH'].split(':'):
50 if os.path.exists(os.path.join(path, 'bundle')):
54 os.environ['PATH'] = ':'.join(newbins)
56 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
57 if not os.path.exists(TEST_TMPDIR):
62 _cached_db_config = {}
64 def find_server_pid(PID_PATH, wait=10):
68 while (not good_pid) and (now <= timeout):
71 with open(PID_PATH, 'r') as f:
72 server_pid = int(f.read())
73 good_pid = (os.kill(server_pid, 0) is None)
74 except EnvironmentError:
83 def kill_server_pid(pidfile, wait=10, passenger_root=False):
84 # Must re-import modules in order to work during atexit
95 # First try to shut down nicely
96 restore_cwd = os.getcwd()
97 os.chdir(passenger_root)
99 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
100 os.chdir(restore_cwd)
101 # Use up to half of the +wait+ period waiting for "passenger
102 # stop" to work. If the process hasn't exited by then, start
103 # sending TERM signals.
107 while now <= deadline and server_pid is None:
109 with open(pidfile, 'r') as f:
110 server_pid = int(f.read())
112 # No pidfile = nothing to kill.
114 except ValueError as error:
115 # Pidfile exists, but we can't parse it. Perhaps the
116 # server has created the file but hasn't written its PID
118 print("Parse error reading pidfile {}: {}".format(pidfile, error),
123 while now <= deadline:
125 exited, _ = os.waitpid(server_pid, os.WNOHANG)
127 _remove_pidfile(pidfile)
130 # already exited, or isn't our child process
134 os.kill(server_pid, signal.SIGTERM)
135 print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
137 except OSError as error:
138 if error.errno == errno.ESRCH:
139 # Thrown by os.getpgid() or os.kill() if the process
140 # does not exist, i.e., our work here is done.
141 _remove_pidfile(pidfile)
147 print("Server PID {} ({}) did not exit, giving up after {}s".
148 format(server_pid, pidfile, wait),
151 def _remove_pidfile(pidfile):
155 if os.path.lexists(pidfile):
158 def find_available_port():
159 """Return an IPv4 port number that is not in use right now.
161 We assume whoever needs to use the returned port is able to reuse
162 a recently used port without waiting for TIME_WAIT (see
163 SO_REUSEADDR / SO_REUSEPORT).
165 Some opportunity for races here, but it's better than choosing
166 something at random and not checking at all. If all of our servers
167 (hey Passenger) knew that listening on port 0 was a thing, the OS
168 would take care of the races, and this wouldn't be needed at all.
171 sock = socket.socket()
172 sock.bind(('0.0.0.0', 0))
173 port = sock.getsockname()[1]
177 def _wait_until_port_listens(port, timeout=10, warn=True):
178 """Wait for a process to start listening on the given port.
180 If nothing listens on the port within the specified timeout (given
181 in seconds), print a warning on stderr before returning.
184 subprocess.check_output(['which', 'netstat'])
185 except subprocess.CalledProcessError:
186 print("WARNING: No `netstat` -- cannot wait for port to listen. "+
187 "Sleeping 0.5 and hoping for the best.",
191 deadline = time.time() + timeout
192 while time.time() < deadline:
193 if re.search(r'\ntcp.*:'+str(port)+' .* LISTEN *\n', str(subprocess.check_output(['netstat', '-an']))):
198 "WARNING: Nothing is listening on port {} (waited {} seconds).".
199 format(port, timeout),
203 def _logfilename(label):
204 """Set up a labelled log file, and return a path to write logs to.
206 Normally, the returned path is {tmpdir}/{label}.log.
208 In debug mode, logs are also written to stderr, with [label]
209 prepended to each line. The returned path is a FIFO.
211 +label+ should contain only alphanumerics: it is also used as part
212 of the FIFO filename.
215 logfilename = os.path.join(TEST_TMPDIR, label+'.log')
216 if not os.environ.get('ARVADOS_DEBUG', ''):
218 fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
221 except OSError as error:
222 if error.errno != errno.ENOENT:
224 os.mkfifo(fifo, 0o700)
225 stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
226 # open(fifo, 'r') would block waiting for someone to open the fifo
227 # for writing, so we need a separate cat process to open it for
229 cat = subprocess.Popen(
230 stdbuf+['cat', fifo],
231 stdin=open('/dev/null'),
232 stdout=subprocess.PIPE)
233 tee = subprocess.Popen(
234 stdbuf+['tee', '-a', logfilename],
236 stdout=subprocess.PIPE)
238 stdbuf+['sed', '-e', 's/^/['+label+'] /'],
243 def run(leave_running_atexit=False):
244 """Ensure an API server is running, and ARVADOS_API_* env vars have
245 admin credentials for it.
247 If ARVADOS_TEST_API_HOST is set, a parent process has started a
248 test server for us to use: we just need to reset() it using the
251 If a previous call to run() started a new server process, and it
252 is still running, we just need to reset() it to fixture state and
255 If neither of those options work out, we'll really start a new
260 # Delete cached discovery documents.
262 # This will clear cached docs that belong to other processes (like
263 # concurrent test suites) even if they're still running. They should
264 # be able to tolerate that.
265 for fn in glob.glob(os.path.join(
266 str(arvados.http_cache('discovery')),
267 '*,arvados,v1,rest,*')):
270 pid_file = _pidfile('api')
271 pid_file_ok = find_server_pid(pid_file, 0)
273 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
274 if existing_api_host and pid_file_ok:
275 if existing_api_host == my_api_host:
279 # Fall through to shutdown-and-start case.
282 # Server was provided by parent. Can't recover if it's
286 # Before trying to start up our own server, call stop() to avoid
287 # "Phusion Passenger Standalone is already running on PID 12345".
288 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
289 # we know the server is ours to kill.)
292 restore_cwd = os.getcwd()
293 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
294 os.chdir(api_src_dir)
296 # Either we haven't started a server of our own yet, or it has
297 # died, or we have lost our credentials, or something else is
298 # preventing us from calling reset(). Start a new one.
300 if not os.path.exists('tmp'):
303 if not os.path.exists('tmp/api'):
304 os.makedirs('tmp/api')
306 if not os.path.exists('tmp/logs'):
307 os.makedirs('tmp/logs')
309 # Install the git repository fixtures.
310 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
311 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
312 if not os.path.isdir(gitdir):
314 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
316 # The nginx proxy isn't listening here yet, but we need to choose
317 # the wss:// port now so we can write the API server config file.
318 wss_port = find_available_port()
319 _setport('wss', wss_port)
321 port = find_available_port()
322 env = os.environ.copy()
323 env['RAILS_ENV'] = 'test'
324 env['ARVADOS_TEST_WSS_PORT'] = str(wss_port)
325 env.pop('ARVADOS_WEBSOCKETS', None)
326 env.pop('ARVADOS_TEST_API_HOST', None)
327 env.pop('ARVADOS_API_HOST', None)
328 env.pop('ARVADOS_API_HOST_INSECURE', None)
329 env.pop('ARVADOS_API_TOKEN', None)
330 start_msg = subprocess.check_output(
332 'passenger', 'start', '-d', '-p{}'.format(port),
333 '--pid-file', pid_file,
334 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
336 '--ssl-certificate', 'tmp/self-signed.pem',
337 '--ssl-certificate-key', 'tmp/self-signed.key'],
340 if not leave_running_atexit:
341 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
343 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
346 "Passenger did not report endpoint: {}".format(start_msg))
347 my_api_host = match.group(1)
348 os.environ['ARVADOS_API_HOST'] = my_api_host
350 # Make sure the server has written its pid file and started
351 # listening on its TCP port
352 find_server_pid(pid_file)
353 _wait_until_port_listens(port)
356 os.chdir(restore_cwd)
359 """Reset the test server to fixture state.
361 This resets the ARVADOS_TEST_API_HOST provided by a parent process
362 if any, otherwise the server started by run().
364 It also resets ARVADOS_* environment vars to point to the test
365 server with admin credentials.
367 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
368 token = auth_token('admin')
369 httpclient = httplib2.Http(ca_certs=os.path.join(
370 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
372 'https://{}/database/reset'.format(existing_api_host),
374 headers={'Authorization': 'OAuth2 {}'.format(token)})
375 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
376 os.environ['ARVADOS_API_TOKEN'] = token
377 if _wait_until_port_listens(_getport('controller-ssl'), timeout=0.5, warn=False):
378 os.environ['ARVADOS_API_HOST'] = '0.0.0.0:'+str(_getport('controller-ssl'))
380 os.environ['ARVADOS_API_HOST'] = existing_api_host
382 def stop(force=False):
383 """Stop the API server, if one is running.
385 If force==False, kill it only if we started it ourselves. (This
386 supports the use case where a Python test suite calls run(), but
387 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
388 process, and the test suite cleans up after itself by calling
389 stop(). In this case the test server provided by the parent
390 process should be left alone.)
392 If force==True, kill it even if we didn't start it
393 ourselves. (This supports the use case in __main__, where "run"
394 and "stop" happen in different processes.)
397 if force or my_api_host is not None:
398 kill_server_pid(_pidfile('api'))
401 def run_controller():
402 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
405 rails_api_port = int(string.split(os.environ.get('ARVADOS_TEST_API_HOST', my_api_host), ':')[-1])
406 port = find_available_port()
407 conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
408 with open(conf, 'w') as f:
412 ManagementToken: e687950a23c3a9bceec28c6223a06c79
427 "http://localhost:{controllerport}": {{}}
430 "https://localhost:{railsport}": {{}}
432 dbhost=_dbconfig('host'),
433 dbname=_dbconfig('database'),
434 dbuser=_dbconfig('username'),
435 dbpass=_dbconfig('password'),
437 railsport=rails_api_port,
439 logf = open(_logfilename('controller'), 'a')
440 controller = subprocess.Popen(
441 ["arvados-server", "controller", "-config", conf],
442 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
443 with open(_pidfile('controller'), 'w') as f:
444 f.write(str(controller.pid))
445 _wait_until_port_listens(port)
446 _setport('controller', port)
449 def stop_controller():
450 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
452 kill_server_pid(_pidfile('controller'))
455 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
458 port = find_available_port()
459 conf = os.path.join(TEST_TMPDIR, 'ws.yml')
460 with open(conf, 'w') as f:
473 """.format(os.environ['ARVADOS_API_HOST'],
475 ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
477 _dbconfig('database'),
478 _dbconfig('username'),
479 _dbconfig('password')))
480 logf = open(_logfilename('ws'), 'a')
481 ws = subprocess.Popen(
482 ["ws", "-config", conf],
483 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
484 with open(_pidfile('ws'), 'w') as f:
486 _wait_until_port_listens(port)
491 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
493 kill_server_pid(_pidfile('ws'))
495 def _start_keep(n, keep_args):
496 keep0 = tempfile.mkdtemp()
497 port = find_available_port()
498 keep_cmd = ["keepstore",
499 "-volume={}".format(keep0),
500 "-listen=:{}".format(port),
501 "-pid="+_pidfile('keep{}'.format(n))]
503 for arg, val in keep_args.items():
504 keep_cmd.append("{}={}".format(arg, val))
506 logf = open(_logfilename('keep{}'.format(n)), 'a')
507 kp0 = subprocess.Popen(
508 keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
510 with open(_pidfile('keep{}'.format(n)), 'w') as f:
511 f.write(str(kp0.pid))
513 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
516 _wait_until_port_listens(port)
520 def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
521 stop_keep(num_servers)
524 if not blob_signing_key:
525 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
526 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
527 keep_args['-blob-signing-key-file'] = f.name
528 f.write(blob_signing_key)
529 keep_args['-enforce-permissions'] = str(enforce_permissions).lower()
530 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
531 keep_args['-data-manager-token-file'] = f.name
532 f.write(auth_token('data_manager'))
533 keep_args['-never-delete'] = 'false'
537 host=os.environ['ARVADOS_API_HOST'],
538 token=os.environ['ARVADOS_API_TOKEN'],
541 for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']:
542 api.keep_services().delete(uuid=d['uuid']).execute()
543 for d in api.keep_disks().list().execute()['items']:
544 api.keep_disks().delete(uuid=d['uuid']).execute()
546 for d in range(0, num_servers):
547 port = _start_keep(d, keep_args)
548 svc = api.keep_services().create(body={'keep_service': {
549 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
550 'service_host': 'localhost',
551 'service_port': port,
552 'service_type': 'disk',
553 'service_ssl_flag': False,
555 api.keep_disks().create(body={
556 'keep_disk': {'keep_service_uuid': svc['uuid'] }
559 # If keepproxy and/or keep-web is running, send SIGHUP to make
560 # them discover the new keepstore services.
561 for svc in ('keepproxy', 'keep-web'):
562 pidfile = _pidfile('keepproxy')
563 if os.path.exists(pidfile):
565 os.kill(int(open(pidfile).read()), signal.SIGHUP)
570 kill_server_pid(_pidfile('keep{}'.format(n)))
571 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
572 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
573 shutil.rmtree(r.read(), True)
574 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
575 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
576 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
578 def stop_keep(num_servers=2):
579 for n in range(0, num_servers):
582 def run_keep_proxy():
583 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
584 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(_getport('keepproxy'))
588 port = find_available_port()
589 env = os.environ.copy()
590 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
591 logf = open(_logfilename('keepproxy'), 'a')
592 kp = subprocess.Popen(
594 '-pid='+_pidfile('keepproxy'),
595 '-listen=:{}'.format(port)],
596 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
600 host=os.environ['ARVADOS_API_HOST'],
601 token=auth_token('admin'),
603 for d in api.keep_services().list(
604 filters=[['service_type','=','proxy']]).execute()['items']:
605 api.keep_services().delete(uuid=d['uuid']).execute()
606 api.keep_services().create(body={'keep_service': {
607 'service_host': 'localhost',
608 'service_port': port,
609 'service_type': 'proxy',
610 'service_ssl_flag': False,
612 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
613 _setport('keepproxy', port)
614 _wait_until_port_listens(port)
616 def stop_keep_proxy():
617 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
619 kill_server_pid(_pidfile('keepproxy'))
621 def run_arv_git_httpd():
622 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
626 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
627 gitport = find_available_port()
628 env = os.environ.copy()
629 env.pop('ARVADOS_API_TOKEN', None)
630 logf = open(_logfilename('arv-git-httpd'), 'a')
631 agh = subprocess.Popen(
633 '-repo-root='+gitdir+'/test',
634 '-management-token=e687950a23c3a9bceec28c6223a06c79',
635 '-address=:'+str(gitport)],
636 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
637 with open(_pidfile('arv-git-httpd'), 'w') as f:
638 f.write(str(agh.pid))
639 _setport('arv-git-httpd', gitport)
640 _wait_until_port_listens(gitport)
642 def stop_arv_git_httpd():
643 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
645 kill_server_pid(_pidfile('arv-git-httpd'))
648 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
652 keepwebport = find_available_port()
653 env = os.environ.copy()
654 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
655 logf = open(_logfilename('keep-web'), 'a')
656 keepweb = subprocess.Popen(
659 '-attachment-only-host=download',
660 '-management-token=e687950a23c3a9bceec28c6223a06c79',
661 '-listen=:'+str(keepwebport)],
662 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
663 with open(_pidfile('keep-web'), 'w') as f:
664 f.write(str(keepweb.pid))
665 _setport('keep-web', keepwebport)
666 _wait_until_port_listens(keepwebport)
669 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
671 kill_server_pid(_pidfile('keep-web'))
674 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
678 nginxconf['CONTROLLERPORT'] = _getport('controller')
679 nginxconf['CONTROLLERSSLPORT'] = find_available_port()
680 nginxconf['KEEPWEBPORT'] = _getport('keep-web')
681 nginxconf['KEEPWEBDLSSLPORT'] = find_available_port()
682 nginxconf['KEEPWEBSSLPORT'] = find_available_port()
683 nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
684 nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
685 nginxconf['GITPORT'] = _getport('arv-git-httpd')
686 nginxconf['GITSSLPORT'] = find_available_port()
687 nginxconf['WSPORT'] = _getport('ws')
688 nginxconf['WSSPORT'] = _getport('wss')
689 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
690 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
691 nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
692 nginxconf['ERRORLOG'] = _logfilename('nginx_error')
693 nginxconf['TMPDIR'] = TEST_TMPDIR
695 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
696 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
697 with open(conffile, 'w') as f:
700 lambda match: str(nginxconf.get(match.group(1))),
701 open(conftemplatefile).read()))
703 env = os.environ.copy()
704 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
706 nginx = subprocess.Popen(
708 '-g', 'error_log stderr info;',
709 '-g', 'pid '+_pidfile('nginx')+';',
711 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
712 _setport('controller-ssl', nginxconf['CONTROLLERSSLPORT'])
713 _setport('keep-web-dl-ssl', nginxconf['KEEPWEBDLSSLPORT'])
714 _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
715 _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
716 _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
719 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
721 kill_server_pid(_pidfile('nginx'))
723 def _pidfile(program):
724 return os.path.join(TEST_TMPDIR, program + '.pid')
726 def _portfile(program):
727 return os.path.join(TEST_TMPDIR, program + '.port')
729 def _setport(program, port):
730 with open(_portfile(program), 'w') as f:
733 # Returns 9 if program is not up.
734 def _getport(program):
736 return int(open(_portfile(program)).read())
741 global _cached_db_config
742 if not _cached_db_config:
743 _cached_db_config = yaml.safe_load(open(os.path.join(
744 SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))
745 return _cached_db_config['test'][key]
748 global _cached_config
750 return _cached_config[key]
751 def _load(f, required=True):
752 fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
753 if not required and not os.path.exists(fullpath):
755 return yaml.safe_load(fullpath)
756 cdefault = _load('application.default.yml')
757 csite = _load('application.yml', required=False)
759 for section in [cdefault.get('common',{}), cdefault.get('test',{}),
760 csite.get('common',{}), csite.get('test',{})]:
761 _cached_config.update(section)
762 return _cached_config[key]
765 '''load a fixture yaml file'''
766 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
770 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
771 yaml_file = yaml_file[0:trim_index]
774 return yaml.safe_load(yaml_file)
776 def auth_token(token_name):
777 return fixture("api_client_authorizations")[token_name]["api_token"]
779 def authorize_with(token_name):
780 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
781 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
782 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
783 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
785 class TestCaseWithServers(unittest.TestCase):
786 """TestCase to start and stop supporting Arvados servers.
788 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
789 class variables as a dictionary of keyword arguments. If you do,
790 setUpClass will start the corresponding servers by passing these
791 keyword arguments to the run, run_keep, and/or run_keep_server
792 functions, respectively. It will also set Arvados environment
793 variables to point to these servers appropriately. If you don't
794 run a Keep or Keep proxy server, setUpClass will set up a
795 temporary directory for Keep local storage, and set it as
798 tearDownClass will stop any servers started, and restore the
799 original environment.
804 KEEP_PROXY_SERVER = None
805 KEEP_WEB_SERVER = None
808 def _restore_dict(src, dest):
809 for key in list(dest.keys()):
816 cls._orig_environ = os.environ.copy()
817 cls._orig_config = arvados.config.settings().copy()
818 cls._cleanup_funcs = []
819 os.environ.pop('ARVADOS_KEEP_SERVICES', None)
820 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
821 for server_kwargs, start_func, stop_func in (
822 (cls.MAIN_SERVER, run, reset),
823 (cls.WS_SERVER, run_ws, stop_ws),
824 (cls.KEEP_SERVER, run_keep, stop_keep),
825 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
826 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
827 if server_kwargs is not None:
828 start_func(**server_kwargs)
829 cls._cleanup_funcs.append(stop_func)
830 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
831 cls.local_store = tempfile.mkdtemp()
832 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
833 cls._cleanup_funcs.append(
834 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
836 os.environ.pop('KEEP_LOCAL_STORE', None)
837 arvados.config.initialize()
840 def tearDownClass(cls):
841 for clean_func in cls._cleanup_funcs:
843 cls._restore_dict(cls._orig_environ, os.environ)
844 cls._restore_dict(cls._orig_config, arvados.config.settings())
847 if __name__ == "__main__":
850 'start_ws', 'stop_ws',
851 'start_controller', 'stop_controller',
852 'start_keep', 'stop_keep',
853 'start_keep_proxy', 'stop_keep_proxy',
854 'start_keep-web', 'stop_keep-web',
855 'start_arv-git-httpd', 'stop_arv-git-httpd',
856 'start_nginx', 'stop_nginx',
858 parser = argparse.ArgumentParser()
859 parser.add_argument('action', type=str, help="one of {}".format(actions))
860 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
861 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
862 parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions")
864 args = parser.parse_args()
866 if args.action not in actions:
867 print("Unrecognized action '{}'. Actions are: {}.".
868 format(args.action, actions),
871 if args.action == 'start':
872 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
873 run(leave_running_atexit=True)
874 host = os.environ['ARVADOS_API_HOST']
875 if args.auth is not None:
876 token = auth_token(args.auth)
877 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
878 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
879 print("export ARVADOS_API_HOST_INSECURE=true")
882 elif args.action == 'stop':
883 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
884 elif args.action == 'start_ws':
886 elif args.action == 'stop_ws':
888 elif args.action == 'start_controller':
890 elif args.action == 'stop_controller':
892 elif args.action == 'start_keep':
893 run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
894 elif args.action == 'stop_keep':
895 stop_keep(num_servers=args.num_keep_servers)
896 elif args.action == 'start_keep_proxy':
898 elif args.action == 'stop_keep_proxy':
900 elif args.action == 'start_arv-git-httpd':
902 elif args.action == 'stop_arv-git-httpd':
904 elif args.action == 'start_keep-web':
906 elif args.action == 'stop_keep-web':
908 elif args.action == 'start_nginx':
910 print("export ARVADOS_API_HOST=0.0.0.0:{}".format(_getport('controller-ssl')))
911 elif args.action == 'stop_nginx':
914 raise Exception("action recognized but not implemented!?")