1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import print_function
6 from __future__ import division
7 from builtins import str
8 from builtins import range
29 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
30 if __name__ == '__main__' and os.path.exists(
31 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
32 # We're being launched to support another test suite.
33 # Add the Python SDK source to the library path.
34 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
39 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
40 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
41 if 'GOPATH' in os.environ:
42 # Add all GOPATH bin dirs to PATH -- but insert them after the
43 # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
44 # command, not the golang.org/x/tools/cmd/bundle command.
45 gopaths = os.environ['GOPATH'].split(':')
46 addbins = [os.path.join(path, 'bin') for path in gopaths]
48 for path in os.environ['PATH'].split(':'):
50 if os.path.exists(os.path.join(path, 'bundle')):
54 os.environ['PATH'] = ':'.join(newbins)
56 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
57 if not os.path.exists(TEST_TMPDIR):
62 _cached_db_config = {}
64 def find_server_pid(PID_PATH, wait=10):
68 while (not good_pid) and (now <= timeout):
71 with open(PID_PATH, 'r') as f:
72 server_pid = int(f.read())
73 good_pid = (os.kill(server_pid, 0) is None)
74 except EnvironmentError:
83 def kill_server_pid(pidfile, wait=10, passenger_root=False):
84 # Must re-import modules in order to work during atexit
95 # First try to shut down nicely
96 restore_cwd = os.getcwd()
97 os.chdir(passenger_root)
99 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
100 os.chdir(restore_cwd)
101 # Use up to half of the +wait+ period waiting for "passenger
102 # stop" to work. If the process hasn't exited by then, start
103 # sending TERM signals.
107 while now <= deadline and server_pid is None:
109 with open(pidfile, 'r') as f:
110 server_pid = int(f.read())
112 # No pidfile = nothing to kill.
114 except ValueError as error:
115 # Pidfile exists, but we can't parse it. Perhaps the
116 # server has created the file but hasn't written its PID
118 print("Parse error reading pidfile {}: {}".format(pidfile, error),
123 while now <= deadline:
125 exited, _ = os.waitpid(server_pid, os.WNOHANG)
127 _remove_pidfile(pidfile)
130 # already exited, or isn't our child process
134 os.kill(server_pid, signal.SIGTERM)
135 print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
137 except OSError as error:
138 if error.errno == errno.ESRCH:
139 # Thrown by os.getpgid() or os.kill() if the process
140 # does not exist, i.e., our work here is done.
141 _remove_pidfile(pidfile)
147 print("Server PID {} ({}) did not exit, giving up after {}s".
148 format(server_pid, pidfile, wait),
151 def _remove_pidfile(pidfile):
155 if os.path.lexists(pidfile):
158 def find_available_port():
159 """Return an IPv4 port number that is not in use right now.
161 We assume whoever needs to use the returned port is able to reuse
162 a recently used port without waiting for TIME_WAIT (see
163 SO_REUSEADDR / SO_REUSEPORT).
165 Some opportunity for races here, but it's better than choosing
166 something at random and not checking at all. If all of our servers
167 (hey Passenger) knew that listening on port 0 was a thing, the OS
168 would take care of the races, and this wouldn't be needed at all.
171 sock = socket.socket()
172 sock.bind(('0.0.0.0', 0))
173 port = sock.getsockname()[1]
177 def _wait_until_port_listens(port, timeout=10, warn=True):
178 """Wait for a process to start listening on the given port.
180 If nothing listens on the port within the specified timeout (given
181 in seconds), print a warning on stderr before returning.
184 subprocess.check_output(['which', 'lsof'])
185 except subprocess.CalledProcessError:
186 print("WARNING: No `lsof` -- cannot wait for port to listen. "+
187 "Sleeping 0.5 and hoping for the best.",
191 deadline = time.time() + timeout
192 while time.time() < deadline:
194 subprocess.check_output(
195 ['lsof', '-t', '-i', 'tcp:'+str(port)])
196 except subprocess.CalledProcessError:
202 "WARNING: Nothing is listening on port {} (waited {} seconds).".
203 format(port, timeout),
207 def _logfilename(label):
208 """Set up a labelled log file, and return a path to write logs to.
210 Normally, the returned path is {tmpdir}/{label}.log.
212 In debug mode, logs are also written to stderr, with [label]
213 prepended to each line. The returned path is a FIFO.
215 +label+ should contain only alphanumerics: it is also used as part
216 of the FIFO filename.
219 logfilename = os.path.join(TEST_TMPDIR, label+'.log')
220 if not os.environ.get('ARVADOS_DEBUG', ''):
222 fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
225 except OSError as error:
226 if error.errno != errno.ENOENT:
228 os.mkfifo(fifo, 0o700)
229 stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
230 # open(fifo, 'r') would block waiting for someone to open the fifo
231 # for writing, so we need a separate cat process to open it for
233 cat = subprocess.Popen(
234 stdbuf+['cat', fifo],
235 stdin=open('/dev/null'),
236 stdout=subprocess.PIPE)
237 tee = subprocess.Popen(
238 stdbuf+['tee', '-a', logfilename],
240 stdout=subprocess.PIPE)
242 stdbuf+['sed', '-e', 's/^/['+label+'] /'],
247 def run(leave_running_atexit=False):
248 """Ensure an API server is running, and ARVADOS_API_* env vars have
249 admin credentials for it.
251 If ARVADOS_TEST_API_HOST is set, a parent process has started a
252 test server for us to use: we just need to reset() it using the
255 If a previous call to run() started a new server process, and it
256 is still running, we just need to reset() it to fixture state and
259 If neither of those options work out, we'll really start a new
264 # Delete cached discovery documents.
266 # This will clear cached docs that belong to other processes (like
267 # concurrent test suites) even if they're still running. They should
268 # be able to tolerate that.
269 for fn in glob.glob(os.path.join(
270 str(arvados.http_cache('discovery')),
271 '*,arvados,v1,rest,*')):
274 pid_file = _pidfile('api')
275 pid_file_ok = find_server_pid(pid_file, 0)
277 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
278 if existing_api_host and pid_file_ok:
279 if existing_api_host == my_api_host:
283 # Fall through to shutdown-and-start case.
286 # Server was provided by parent. Can't recover if it's
290 # Before trying to start up our own server, call stop() to avoid
291 # "Phusion Passenger Standalone is already running on PID 12345".
292 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
293 # we know the server is ours to kill.)
296 restore_cwd = os.getcwd()
297 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
298 os.chdir(api_src_dir)
300 # Either we haven't started a server of our own yet, or it has
301 # died, or we have lost our credentials, or something else is
302 # preventing us from calling reset(). Start a new one.
304 if not os.path.exists('tmp'):
307 if not os.path.exists('tmp/api'):
308 os.makedirs('tmp/api')
310 if not os.path.exists('tmp/logs'):
311 os.makedirs('tmp/logs')
313 # Install the git repository fixtures.
314 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
315 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
316 if not os.path.isdir(gitdir):
318 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
320 # The nginx proxy isn't listening here yet, but we need to choose
321 # the wss:// port now so we can write the API server config file.
322 wss_port = find_available_port()
323 _setport('wss', wss_port)
325 port = find_available_port()
326 env = os.environ.copy()
327 env['RAILS_ENV'] = 'test'
328 env['ARVADOS_TEST_WSS_PORT'] = str(wss_port)
329 env.pop('ARVADOS_WEBSOCKETS', None)
330 env.pop('ARVADOS_TEST_API_HOST', None)
331 env.pop('ARVADOS_API_HOST', None)
332 env.pop('ARVADOS_API_HOST_INSECURE', None)
333 env.pop('ARVADOS_API_TOKEN', None)
334 start_msg = subprocess.check_output(
336 'passenger', 'start', '-d', '-p{}'.format(port),
337 '--pid-file', pid_file,
338 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
340 '--ssl-certificate', 'tmp/self-signed.pem',
341 '--ssl-certificate-key', 'tmp/self-signed.key'],
344 if not leave_running_atexit:
345 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
347 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
350 "Passenger did not report endpoint: {}".format(start_msg))
351 my_api_host = match.group(1)
352 os.environ['ARVADOS_API_HOST'] = my_api_host
354 # Make sure the server has written its pid file and started
355 # listening on its TCP port
356 find_server_pid(pid_file)
357 _wait_until_port_listens(port)
360 os.chdir(restore_cwd)
363 """Reset the test server to fixture state.
365 This resets the ARVADOS_TEST_API_HOST provided by a parent process
366 if any, otherwise the server started by run().
368 It also resets ARVADOS_* environment vars to point to the test
369 server with admin credentials.
371 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
372 token = auth_token('admin')
373 httpclient = httplib2.Http(ca_certs=os.path.join(
374 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
376 'https://{}/database/reset'.format(existing_api_host),
378 headers={'Authorization': 'OAuth2 {}'.format(token)})
379 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
380 os.environ['ARVADOS_API_TOKEN'] = token
381 if _wait_until_port_listens(_getport('controller-ssl'), timeout=0.5, warn=False):
382 os.environ['ARVADOS_API_HOST'] = '0.0.0.0:'+str(_getport('controller-ssl'))
384 os.environ['ARVADOS_API_HOST'] = existing_api_host
386 def stop(force=False):
387 """Stop the API server, if one is running.
389 If force==False, kill it only if we started it ourselves. (This
390 supports the use case where a Python test suite calls run(), but
391 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
392 process, and the test suite cleans up after itself by calling
393 stop(). In this case the test server provided by the parent
394 process should be left alone.)
396 If force==True, kill it even if we didn't start it
397 ourselves. (This supports the use case in __main__, where "run"
398 and "stop" happen in different processes.)
401 if force or my_api_host is not None:
402 kill_server_pid(_pidfile('api'))
405 def run_controller():
406 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
409 rails_api_port = int(string.split(os.environ.get('ARVADOS_TEST_API_HOST', my_api_host), ':')[-1])
410 port = find_available_port()
411 conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
412 with open(conf, 'w') as f:
416 ManagementToken: e687950a23c3a9bceec28c6223a06c79
428 "arvados-controller":
430 "arvados-api-server":
436 _dbconfig('database'),
437 _dbconfig('username'),
438 _dbconfig('password'),
442 logf = open(_logfilename('controller'), 'a')
443 controller = subprocess.Popen(
444 ["arvados-server", "controller", "-config", conf],
445 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
446 with open(_pidfile('controller'), 'w') as f:
447 f.write(str(controller.pid))
448 _wait_until_port_listens(port)
449 _setport('controller', port)
452 def stop_controller():
453 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
455 kill_server_pid(_pidfile('controller'))
458 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
461 port = find_available_port()
462 conf = os.path.join(TEST_TMPDIR, 'ws.yml')
463 with open(conf, 'w') as f:
476 """.format(os.environ['ARVADOS_API_HOST'],
478 ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
480 _dbconfig('database'),
481 _dbconfig('username'),
482 _dbconfig('password')))
483 logf = open(_logfilename('ws'), 'a')
484 ws = subprocess.Popen(
485 ["ws", "-config", conf],
486 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
487 with open(_pidfile('ws'), 'w') as f:
489 _wait_until_port_listens(port)
494 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
496 kill_server_pid(_pidfile('ws'))
498 def _start_keep(n, keep_args):
499 keep0 = tempfile.mkdtemp()
500 port = find_available_port()
501 keep_cmd = ["keepstore",
502 "-volume={}".format(keep0),
503 "-listen=:{}".format(port),
504 "-pid="+_pidfile('keep{}'.format(n))]
506 for arg, val in keep_args.items():
507 keep_cmd.append("{}={}".format(arg, val))
509 logf = open(_logfilename('keep{}'.format(n)), 'a')
510 kp0 = subprocess.Popen(
511 keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
513 with open(_pidfile('keep{}'.format(n)), 'w') as f:
514 f.write(str(kp0.pid))
516 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
519 _wait_until_port_listens(port)
523 def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
524 stop_keep(num_servers)
527 if not blob_signing_key:
528 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
529 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
530 keep_args['-blob-signing-key-file'] = f.name
531 f.write(blob_signing_key)
532 keep_args['-enforce-permissions'] = str(enforce_permissions).lower()
533 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
534 keep_args['-data-manager-token-file'] = f.name
535 f.write(auth_token('data_manager'))
536 keep_args['-never-delete'] = 'false'
540 host=os.environ['ARVADOS_API_HOST'],
541 token=os.environ['ARVADOS_API_TOKEN'],
544 for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']:
545 api.keep_services().delete(uuid=d['uuid']).execute()
546 for d in api.keep_disks().list().execute()['items']:
547 api.keep_disks().delete(uuid=d['uuid']).execute()
549 for d in range(0, num_servers):
550 port = _start_keep(d, keep_args)
551 svc = api.keep_services().create(body={'keep_service': {
552 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
553 'service_host': 'localhost',
554 'service_port': port,
555 'service_type': 'disk',
556 'service_ssl_flag': False,
558 api.keep_disks().create(body={
559 'keep_disk': {'keep_service_uuid': svc['uuid'] }
562 # If keepproxy and/or keep-web is running, send SIGHUP to make
563 # them discover the new keepstore services.
564 for svc in ('keepproxy', 'keep-web'):
565 pidfile = _pidfile('keepproxy')
566 if os.path.exists(pidfile):
568 os.kill(int(open(pidfile).read()), signal.SIGHUP)
573 kill_server_pid(_pidfile('keep{}'.format(n)))
574 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
575 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
576 shutil.rmtree(r.read(), True)
577 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
578 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
579 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
581 def stop_keep(num_servers=2):
582 for n in range(0, num_servers):
585 def run_keep_proxy():
586 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
587 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(_getport('keepproxy'))
591 port = find_available_port()
592 env = os.environ.copy()
593 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
594 logf = open(_logfilename('keepproxy'), 'a')
595 kp = subprocess.Popen(
597 '-pid='+_pidfile('keepproxy'),
598 '-listen=:{}'.format(port)],
599 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
603 host=os.environ['ARVADOS_API_HOST'],
604 token=auth_token('admin'),
606 for d in api.keep_services().list(
607 filters=[['service_type','=','proxy']]).execute()['items']:
608 api.keep_services().delete(uuid=d['uuid']).execute()
609 api.keep_services().create(body={'keep_service': {
610 'service_host': 'localhost',
611 'service_port': port,
612 'service_type': 'proxy',
613 'service_ssl_flag': False,
615 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
616 _setport('keepproxy', port)
617 _wait_until_port_listens(port)
619 def stop_keep_proxy():
620 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
622 kill_server_pid(_pidfile('keepproxy'))
624 def run_arv_git_httpd():
625 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
629 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
630 gitport = find_available_port()
631 env = os.environ.copy()
632 env.pop('ARVADOS_API_TOKEN', None)
633 logf = open(_logfilename('arv-git-httpd'), 'a')
634 agh = subprocess.Popen(
636 '-repo-root='+gitdir+'/test',
637 '-management-token=e687950a23c3a9bceec28c6223a06c79',
638 '-address=:'+str(gitport)],
639 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
640 with open(_pidfile('arv-git-httpd'), 'w') as f:
641 f.write(str(agh.pid))
642 _setport('arv-git-httpd', gitport)
643 _wait_until_port_listens(gitport)
645 def stop_arv_git_httpd():
646 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
648 kill_server_pid(_pidfile('arv-git-httpd'))
651 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
655 keepwebport = find_available_port()
656 env = os.environ.copy()
657 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
658 logf = open(_logfilename('keep-web'), 'a')
659 keepweb = subprocess.Popen(
662 '-attachment-only-host=download',
663 '-management-token=e687950a23c3a9bceec28c6223a06c79',
664 '-listen=:'+str(keepwebport)],
665 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
666 with open(_pidfile('keep-web'), 'w') as f:
667 f.write(str(keepweb.pid))
668 _setport('keep-web', keepwebport)
669 _wait_until_port_listens(keepwebport)
672 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
674 kill_server_pid(_pidfile('keep-web'))
677 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
681 nginxconf['CONTROLLERPORT'] = _getport('controller')
682 nginxconf['CONTROLLERSSLPORT'] = find_available_port()
683 nginxconf['KEEPWEBPORT'] = _getport('keep-web')
684 nginxconf['KEEPWEBDLSSLPORT'] = find_available_port()
685 nginxconf['KEEPWEBSSLPORT'] = find_available_port()
686 nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
687 nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
688 nginxconf['GITPORT'] = _getport('arv-git-httpd')
689 nginxconf['GITSSLPORT'] = find_available_port()
690 nginxconf['WSPORT'] = _getport('ws')
691 nginxconf['WSSPORT'] = _getport('wss')
692 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
693 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
694 nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
695 nginxconf['ERRORLOG'] = _logfilename('nginx_error')
696 nginxconf['TMPDIR'] = TEST_TMPDIR
698 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
699 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
700 with open(conffile, 'w') as f:
703 lambda match: str(nginxconf.get(match.group(1))),
704 open(conftemplatefile).read()))
706 env = os.environ.copy()
707 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
709 nginx = subprocess.Popen(
711 '-g', 'error_log stderr info;',
712 '-g', 'pid '+_pidfile('nginx')+';',
714 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
715 _setport('controller-ssl', nginxconf['CONTROLLERSSLPORT'])
716 _setport('keep-web-dl-ssl', nginxconf['KEEPWEBDLSSLPORT'])
717 _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
718 _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
719 _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
722 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
724 kill_server_pid(_pidfile('nginx'))
726 def _pidfile(program):
727 return os.path.join(TEST_TMPDIR, program + '.pid')
729 def _portfile(program):
730 return os.path.join(TEST_TMPDIR, program + '.port')
732 def _setport(program, port):
733 with open(_portfile(program), 'w') as f:
736 # Returns 9 if program is not up.
737 def _getport(program):
739 return int(open(_portfile(program)).read())
744 global _cached_db_config
745 if not _cached_db_config:
746 _cached_db_config = yaml.safe_load(open(os.path.join(
747 SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))
748 return _cached_db_config['test'][key]
751 global _cached_config
753 return _cached_config[key]
754 def _load(f, required=True):
755 fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
756 if not required and not os.path.exists(fullpath):
758 return yaml.safe_load(fullpath)
759 cdefault = _load('application.default.yml')
760 csite = _load('application.yml', required=False)
762 for section in [cdefault.get('common',{}), cdefault.get('test',{}),
763 csite.get('common',{}), csite.get('test',{})]:
764 _cached_config.update(section)
765 return _cached_config[key]
768 '''load a fixture yaml file'''
769 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
773 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
774 yaml_file = yaml_file[0:trim_index]
777 return yaml.safe_load(yaml_file)
779 def auth_token(token_name):
780 return fixture("api_client_authorizations")[token_name]["api_token"]
782 def authorize_with(token_name):
783 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
784 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
785 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
786 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
788 class TestCaseWithServers(unittest.TestCase):
789 """TestCase to start and stop supporting Arvados servers.
791 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
792 class variables as a dictionary of keyword arguments. If you do,
793 setUpClass will start the corresponding servers by passing these
794 keyword arguments to the run, run_keep, and/or run_keep_server
795 functions, respectively. It will also set Arvados environment
796 variables to point to these servers appropriately. If you don't
797 run a Keep or Keep proxy server, setUpClass will set up a
798 temporary directory for Keep local storage, and set it as
801 tearDownClass will stop any servers started, and restore the
802 original environment.
807 KEEP_PROXY_SERVER = None
808 KEEP_WEB_SERVER = None
811 def _restore_dict(src, dest):
812 for key in list(dest.keys()):
819 cls._orig_environ = os.environ.copy()
820 cls._orig_config = arvados.config.settings().copy()
821 cls._cleanup_funcs = []
822 os.environ.pop('ARVADOS_KEEP_SERVICES', None)
823 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
824 for server_kwargs, start_func, stop_func in (
825 (cls.MAIN_SERVER, run, reset),
826 (cls.WS_SERVER, run_ws, stop_ws),
827 (cls.KEEP_SERVER, run_keep, stop_keep),
828 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
829 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
830 if server_kwargs is not None:
831 start_func(**server_kwargs)
832 cls._cleanup_funcs.append(stop_func)
833 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
834 cls.local_store = tempfile.mkdtemp()
835 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
836 cls._cleanup_funcs.append(
837 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
839 os.environ.pop('KEEP_LOCAL_STORE', None)
840 arvados.config.initialize()
843 def tearDownClass(cls):
844 for clean_func in cls._cleanup_funcs:
846 cls._restore_dict(cls._orig_environ, os.environ)
847 cls._restore_dict(cls._orig_config, arvados.config.settings())
850 if __name__ == "__main__":
853 'start_ws', 'stop_ws',
854 'start_controller', 'stop_controller',
855 'start_keep', 'stop_keep',
856 'start_keep_proxy', 'stop_keep_proxy',
857 'start_keep-web', 'stop_keep-web',
858 'start_arv-git-httpd', 'stop_arv-git-httpd',
859 'start_nginx', 'stop_nginx',
861 parser = argparse.ArgumentParser()
862 parser.add_argument('action', type=str, help="one of {}".format(actions))
863 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
864 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
865 parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions")
867 args = parser.parse_args()
869 if args.action not in actions:
870 print("Unrecognized action '{}'. Actions are: {}.".
871 format(args.action, actions),
874 if args.action == 'start':
875 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
876 run(leave_running_atexit=True)
877 host = os.environ['ARVADOS_API_HOST']
878 if args.auth is not None:
879 token = auth_token(args.auth)
880 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
881 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
882 print("export ARVADOS_API_HOST_INSECURE=true")
885 elif args.action == 'stop':
886 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
887 elif args.action == 'start_ws':
889 elif args.action == 'stop_ws':
891 elif args.action == 'start_controller':
893 elif args.action == 'stop_controller':
895 elif args.action == 'start_keep':
896 run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
897 elif args.action == 'stop_keep':
898 stop_keep(num_servers=args.num_keep_servers)
899 elif args.action == 'start_keep_proxy':
901 elif args.action == 'stop_keep_proxy':
903 elif args.action == 'start_arv-git-httpd':
905 elif args.action == 'stop_arv-git-httpd':
907 elif args.action == 'start_keep-web':
909 elif args.action == 'stop_keep-web':
911 elif args.action == 'start_nginx':
913 print("export ARVADOS_API_HOST=0.0.0.0:{}".format(_getport('controller-ssl')))
914 elif args.action == 'stop_nginx':
917 raise Exception("action recognized but not implemented!?")