1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import print_function
6 from __future__ import division
7 from builtins import str
8 from builtins import range
29 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
30 if __name__ == '__main__' and os.path.exists(
31 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
32 # We're being launched to support another test suite.
33 # Add the Python SDK source to the library path.
34 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
39 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
40 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
41 if 'GOPATH' in os.environ:
42 # Add all GOPATH bin dirs to PATH -- but insert them after the
43 # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
44 # command, not the golang.org/x/tools/cmd/bundle command.
45 gopaths = os.environ['GOPATH'].split(':')
46 addbins = [os.path.join(path, 'bin') for path in gopaths]
48 for path in os.environ['PATH'].split(':'):
50 if os.path.exists(os.path.join(path, 'bundle')):
54 os.environ['PATH'] = ':'.join(newbins)
56 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
57 if not os.path.exists(TEST_TMPDIR):
62 _cached_db_config = {}
64 def find_server_pid(PID_PATH, wait=10):
68 while (not good_pid) and (now <= timeout):
71 with open(PID_PATH, 'r') as f:
72 server_pid = int(f.read())
73 good_pid = (os.kill(server_pid, 0) is None)
74 except EnvironmentError:
83 def kill_server_pid(pidfile, wait=10, passenger_root=False):
84 # Must re-import modules in order to work during atexit
95 # First try to shut down nicely
96 restore_cwd = os.getcwd()
97 os.chdir(passenger_root)
99 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
100 os.chdir(restore_cwd)
101 # Use up to half of the +wait+ period waiting for "passenger
102 # stop" to work. If the process hasn't exited by then, start
103 # sending TERM signals.
107 while now <= deadline and server_pid is None:
109 with open(pidfile, 'r') as f:
110 server_pid = int(f.read())
112 # No pidfile = nothing to kill.
114 except ValueError as error:
115 # Pidfile exists, but we can't parse it. Perhaps the
116 # server has created the file but hasn't written its PID
118 print("Parse error reading pidfile {}: {}".format(pidfile, error),
123 while now <= deadline:
125 exited, _ = os.waitpid(server_pid, os.WNOHANG)
127 _remove_pidfile(pidfile)
130 # already exited, or isn't our child process
134 os.kill(server_pid, signal.SIGTERM)
135 print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
137 except OSError as error:
138 if error.errno == errno.ESRCH:
139 # Thrown by os.getpgid() or os.kill() if the process
140 # does not exist, i.e., our work here is done.
141 _remove_pidfile(pidfile)
147 print("Server PID {} ({}) did not exit, giving up after {}s".
148 format(server_pid, pidfile, wait),
151 def _remove_pidfile(pidfile):
155 if os.path.lexists(pidfile):
158 def find_available_port():
159 """Return an IPv4 port number that is not in use right now.
161 We assume whoever needs to use the returned port is able to reuse
162 a recently used port without waiting for TIME_WAIT (see
163 SO_REUSEADDR / SO_REUSEPORT).
165 Some opportunity for races here, but it's better than choosing
166 something at random and not checking at all. If all of our servers
167 (hey Passenger) knew that listening on port 0 was a thing, the OS
168 would take care of the races, and this wouldn't be needed at all.
171 sock = socket.socket()
172 sock.bind(('0.0.0.0', 0))
173 port = sock.getsockname()[1]
177 def _wait_until_port_listens(port, timeout=10, warn=True):
178 """Wait for a process to start listening on the given port.
180 If nothing listens on the port within the specified timeout (given
181 in seconds), print a warning on stderr before returning.
184 subprocess.check_output(['which', 'lsof'])
185 except subprocess.CalledProcessError:
186 print("WARNING: No `lsof` -- cannot wait for port to listen. "+
187 "Sleeping 0.5 and hoping for the best.",
191 deadline = time.time() + timeout
192 while time.time() < deadline:
194 subprocess.check_output(
195 ['lsof', '-t', '-i', 'tcp:'+str(port)])
196 except subprocess.CalledProcessError:
202 "WARNING: Nothing is listening on port {} (waited {} seconds).".
203 format(port, timeout),
207 def _logfilename(label):
208 """Set up a labelled log file, and return a path to write logs to.
210 Normally, the returned path is {tmpdir}/{label}.log.
212 In debug mode, logs are also written to stderr, with [label]
213 prepended to each line. The returned path is a FIFO.
215 +label+ should contain only alphanumerics: it is also used as part
216 of the FIFO filename.
219 logfilename = os.path.join(TEST_TMPDIR, label+'.log')
220 if not os.environ.get('ARVADOS_DEBUG', ''):
222 fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
225 except OSError as error:
226 if error.errno != errno.ENOENT:
228 os.mkfifo(fifo, 0o700)
229 stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
230 # open(fifo, 'r') would block waiting for someone to open the fifo
231 # for writing, so we need a separate cat process to open it for
233 cat = subprocess.Popen(
234 stdbuf+['cat', fifo],
235 stdin=open('/dev/null'),
236 stdout=subprocess.PIPE)
237 tee = subprocess.Popen(
238 stdbuf+['tee', '-a', logfilename],
240 stdout=subprocess.PIPE)
242 stdbuf+['sed', '-e', 's/^/['+label+'] /'],
247 def run(leave_running_atexit=False):
248 """Ensure an API server is running, and ARVADOS_API_* env vars have
249 admin credentials for it.
251 If ARVADOS_TEST_API_HOST is set, a parent process has started a
252 test server for us to use: we just need to reset() it using the
255 If a previous call to run() started a new server process, and it
256 is still running, we just need to reset() it to fixture state and
259 If neither of those options work out, we'll really start a new
264 # Delete cached discovery documents.
266 # This will clear cached docs that belong to other processes (like
267 # concurrent test suites) even if they're still running. They should
268 # be able to tolerate that.
269 for fn in glob.glob(os.path.join(
270 str(arvados.http_cache('discovery')),
271 '*,arvados,v1,rest,*')):
274 pid_file = _pidfile('api')
275 pid_file_ok = find_server_pid(pid_file, 0)
277 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
278 if existing_api_host and pid_file_ok:
279 if existing_api_host == my_api_host:
283 # Fall through to shutdown-and-start case.
286 # Server was provided by parent. Can't recover if it's
290 # Before trying to start up our own server, call stop() to avoid
291 # "Phusion Passenger Standalone is already running on PID 12345".
292 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
293 # we know the server is ours to kill.)
296 restore_cwd = os.getcwd()
297 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
298 os.chdir(api_src_dir)
300 # Either we haven't started a server of our own yet, or it has
301 # died, or we have lost our credentials, or something else is
302 # preventing us from calling reset(). Start a new one.
304 if not os.path.exists('tmp'):
307 if not os.path.exists('tmp/api'):
308 os.makedirs('tmp/api')
310 if not os.path.exists('tmp/logs'):
311 os.makedirs('tmp/logs')
313 # Install the git repository fixtures.
314 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
315 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
316 if not os.path.isdir(gitdir):
318 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
320 # The nginx proxy isn't listening here yet, but we need to choose
321 # the wss:// port now so we can write the API server config file.
322 wss_port = find_available_port()
323 _setport('wss', wss_port)
325 port = find_available_port()
326 env = os.environ.copy()
327 env['RAILS_ENV'] = 'test'
328 env['ARVADOS_TEST_WSS_PORT'] = str(wss_port)
329 env.pop('ARVADOS_WEBSOCKETS', None)
330 env.pop('ARVADOS_TEST_API_HOST', None)
331 env.pop('ARVADOS_API_HOST', None)
332 env.pop('ARVADOS_API_HOST_INSECURE', None)
333 env.pop('ARVADOS_API_TOKEN', None)
334 start_msg = subprocess.check_output(
336 'passenger', 'start', '-d', '-p{}'.format(port),
337 '--pid-file', pid_file,
338 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
340 '--ssl-certificate', 'tmp/self-signed.pem',
341 '--ssl-certificate-key', 'tmp/self-signed.key'],
344 if not leave_running_atexit:
345 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
347 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
350 "Passenger did not report endpoint: {}".format(start_msg))
351 my_api_host = match.group(1)
352 os.environ['ARVADOS_API_HOST'] = my_api_host
354 # Make sure the server has written its pid file and started
355 # listening on its TCP port
356 find_server_pid(pid_file)
357 _wait_until_port_listens(port)
360 os.chdir(restore_cwd)
363 """Reset the test server to fixture state.
365 This resets the ARVADOS_TEST_API_HOST provided by a parent process
366 if any, otherwise the server started by run().
368 It also resets ARVADOS_* environment vars to point to the test
369 server with admin credentials.
371 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
372 token = auth_token('admin')
373 httpclient = httplib2.Http(ca_certs=os.path.join(
374 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
376 'https://{}/database/reset'.format(existing_api_host),
378 headers={'Authorization': 'OAuth2 {}'.format(token)})
379 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
380 os.environ['ARVADOS_API_TOKEN'] = token
381 if _wait_until_port_listens(_getport('controller-ssl'), timeout=0.5, warn=False):
382 os.environ['ARVADOS_API_HOST'] = '0.0.0.0:'+str(_getport('controller-ssl'))
384 os.environ['ARVADOS_API_HOST'] = existing_api_host
386 def stop(force=False):
387 """Stop the API server, if one is running.
389 If force==False, kill it only if we started it ourselves. (This
390 supports the use case where a Python test suite calls run(), but
391 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
392 process, and the test suite cleans up after itself by calling
393 stop(). In this case the test server provided by the parent
394 process should be left alone.)
396 If force==True, kill it even if we didn't start it
397 ourselves. (This supports the use case in __main__, where "run"
398 and "stop" happen in different processes.)
401 if force or my_api_host is not None:
402 kill_server_pid(_pidfile('api'))
405 def run_controller():
406 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
409 rails_api_port = int(string.split(os.environ.get('ARVADOS_TEST_API_HOST', my_api_host), ':')[-1])
410 port = find_available_port()
411 conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
412 with open(conf, 'w') as f:
416 HTTPRequestTimeout: 30s
426 "arvados-controller":
428 "arvados-api-server":
434 _dbconfig('database'),
435 _dbconfig('username'),
436 _dbconfig('password'),
440 logf = open(_logfilename('controller'), 'a')
441 controller = subprocess.Popen(
442 ["arvados-server", "controller", "-config", conf],
443 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
444 with open(_pidfile('controller'), 'w') as f:
445 f.write(str(controller.pid))
446 _wait_until_port_listens(port)
447 _setport('controller', port)
450 def stop_controller():
451 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
453 kill_server_pid(_pidfile('controller'))
456 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
459 port = find_available_port()
460 conf = os.path.join(TEST_TMPDIR, 'ws.yml')
461 with open(conf, 'w') as f:
474 """.format(os.environ['ARVADOS_API_HOST'],
476 ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
478 _dbconfig('database'),
479 _dbconfig('username'),
480 _dbconfig('password')))
481 logf = open(_logfilename('ws'), 'a')
482 ws = subprocess.Popen(
483 ["ws", "-config", conf],
484 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
485 with open(_pidfile('ws'), 'w') as f:
487 _wait_until_port_listens(port)
492 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
494 kill_server_pid(_pidfile('ws'))
496 def _start_keep(n, keep_args):
497 keep0 = tempfile.mkdtemp()
498 port = find_available_port()
499 keep_cmd = ["keepstore",
500 "-volume={}".format(keep0),
501 "-listen=:{}".format(port),
502 "-pid="+_pidfile('keep{}'.format(n))]
504 for arg, val in keep_args.items():
505 keep_cmd.append("{}={}".format(arg, val))
507 logf = open(_logfilename('keep{}'.format(n)), 'a')
508 kp0 = subprocess.Popen(
509 keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
511 with open(_pidfile('keep{}'.format(n)), 'w') as f:
512 f.write(str(kp0.pid))
514 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
517 _wait_until_port_listens(port)
521 def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
522 stop_keep(num_servers)
525 if not blob_signing_key:
526 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
527 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
528 keep_args['-blob-signing-key-file'] = f.name
529 f.write(blob_signing_key)
530 keep_args['-enforce-permissions'] = str(enforce_permissions).lower()
531 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
532 keep_args['-data-manager-token-file'] = f.name
533 f.write(auth_token('data_manager'))
534 keep_args['-never-delete'] = 'false'
538 host=os.environ['ARVADOS_API_HOST'],
539 token=os.environ['ARVADOS_API_TOKEN'],
542 for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']:
543 api.keep_services().delete(uuid=d['uuid']).execute()
544 for d in api.keep_disks().list().execute()['items']:
545 api.keep_disks().delete(uuid=d['uuid']).execute()
547 for d in range(0, num_servers):
548 port = _start_keep(d, keep_args)
549 svc = api.keep_services().create(body={'keep_service': {
550 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
551 'service_host': 'localhost',
552 'service_port': port,
553 'service_type': 'disk',
554 'service_ssl_flag': False,
556 api.keep_disks().create(body={
557 'keep_disk': {'keep_service_uuid': svc['uuid'] }
560 # If keepproxy and/or keep-web is running, send SIGHUP to make
561 # them discover the new keepstore services.
562 for svc in ('keepproxy', 'keep-web'):
563 pidfile = _pidfile('keepproxy')
564 if os.path.exists(pidfile):
566 os.kill(int(open(pidfile).read()), signal.SIGHUP)
571 kill_server_pid(_pidfile('keep{}'.format(n)))
572 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
573 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
574 shutil.rmtree(r.read(), True)
575 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
576 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
577 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
579 def stop_keep(num_servers=2):
580 for n in range(0, num_servers):
583 def run_keep_proxy():
584 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
585 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(_getport('keepproxy'))
589 port = find_available_port()
590 env = os.environ.copy()
591 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
592 logf = open(_logfilename('keepproxy'), 'a')
593 kp = subprocess.Popen(
595 '-pid='+_pidfile('keepproxy'),
596 '-listen=:{}'.format(port)],
597 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
601 host=os.environ['ARVADOS_API_HOST'],
602 token=auth_token('admin'),
604 for d in api.keep_services().list(
605 filters=[['service_type','=','proxy']]).execute()['items']:
606 api.keep_services().delete(uuid=d['uuid']).execute()
607 api.keep_services().create(body={'keep_service': {
608 'service_host': 'localhost',
609 'service_port': port,
610 'service_type': 'proxy',
611 'service_ssl_flag': False,
613 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
614 _setport('keepproxy', port)
615 _wait_until_port_listens(port)
617 def stop_keep_proxy():
618 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
620 kill_server_pid(_pidfile('keepproxy'))
622 def run_arv_git_httpd():
623 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
627 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
628 gitport = find_available_port()
629 env = os.environ.copy()
630 env.pop('ARVADOS_API_TOKEN', None)
631 logf = open(_logfilename('arv-git-httpd'), 'a')
632 agh = subprocess.Popen(
634 '-repo-root='+gitdir+'/test',
635 '-address=:'+str(gitport)],
636 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
637 with open(_pidfile('arv-git-httpd'), 'w') as f:
638 f.write(str(agh.pid))
639 _setport('arv-git-httpd', gitport)
640 _wait_until_port_listens(gitport)
642 def stop_arv_git_httpd():
643 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
645 kill_server_pid(_pidfile('arv-git-httpd'))
648 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
652 keepwebport = find_available_port()
653 env = os.environ.copy()
654 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
655 logf = open(_logfilename('keep-web'), 'a')
656 keepweb = subprocess.Popen(
659 '-attachment-only-host=download',
660 '-listen=:'+str(keepwebport)],
661 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
662 with open(_pidfile('keep-web'), 'w') as f:
663 f.write(str(keepweb.pid))
664 _setport('keep-web', keepwebport)
665 _wait_until_port_listens(keepwebport)
668 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
670 kill_server_pid(_pidfile('keep-web'))
673 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
677 nginxconf['CONTROLLERPORT'] = _getport('controller')
678 nginxconf['CONTROLLERSSLPORT'] = find_available_port()
679 nginxconf['KEEPWEBPORT'] = _getport('keep-web')
680 nginxconf['KEEPWEBDLSSLPORT'] = find_available_port()
681 nginxconf['KEEPWEBSSLPORT'] = find_available_port()
682 nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
683 nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
684 nginxconf['GITPORT'] = _getport('arv-git-httpd')
685 nginxconf['GITSSLPORT'] = find_available_port()
686 nginxconf['WSPORT'] = _getport('ws')
687 nginxconf['WSSPORT'] = _getport('wss')
688 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
689 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
690 nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
691 nginxconf['ERRORLOG'] = _logfilename('nginx_error')
692 nginxconf['TMPDIR'] = TEST_TMPDIR
694 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
695 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
696 with open(conffile, 'w') as f:
699 lambda match: str(nginxconf.get(match.group(1))),
700 open(conftemplatefile).read()))
702 env = os.environ.copy()
703 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
705 nginx = subprocess.Popen(
707 '-g', 'error_log stderr info;',
708 '-g', 'pid '+_pidfile('nginx')+';',
710 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
711 _setport('controller-ssl', nginxconf['CONTROLLERSSLPORT'])
712 _setport('keep-web-dl-ssl', nginxconf['KEEPWEBDLSSLPORT'])
713 _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
714 _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
715 _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
718 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
720 kill_server_pid(_pidfile('nginx'))
722 def _pidfile(program):
723 return os.path.join(TEST_TMPDIR, program + '.pid')
725 def _portfile(program):
726 return os.path.join(TEST_TMPDIR, program + '.port')
728 def _setport(program, port):
729 with open(_portfile(program), 'w') as f:
732 # Returns 9 if program is not up.
733 def _getport(program):
735 return int(open(_portfile(program)).read())
740 global _cached_db_config
741 if not _cached_db_config:
742 _cached_db_config = yaml.safe_load(open(os.path.join(
743 SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))
744 return _cached_db_config['test'][key]
747 global _cached_config
749 return _cached_config[key]
750 def _load(f, required=True):
751 fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
752 if not required and not os.path.exists(fullpath):
754 return yaml.safe_load(fullpath)
755 cdefault = _load('application.default.yml')
756 csite = _load('application.yml', required=False)
758 for section in [cdefault.get('common',{}), cdefault.get('test',{}),
759 csite.get('common',{}), csite.get('test',{})]:
760 _cached_config.update(section)
761 return _cached_config[key]
764 '''load a fixture yaml file'''
765 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
769 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
770 yaml_file = yaml_file[0:trim_index]
773 return yaml.safe_load(yaml_file)
775 def auth_token(token_name):
776 return fixture("api_client_authorizations")[token_name]["api_token"]
778 def authorize_with(token_name):
779 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
780 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
781 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
782 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
784 class TestCaseWithServers(unittest.TestCase):
785 """TestCase to start and stop supporting Arvados servers.
787 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
788 class variables as a dictionary of keyword arguments. If you do,
789 setUpClass will start the corresponding servers by passing these
790 keyword arguments to the run, run_keep, and/or run_keep_server
791 functions, respectively. It will also set Arvados environment
792 variables to point to these servers appropriately. If you don't
793 run a Keep or Keep proxy server, setUpClass will set up a
794 temporary directory for Keep local storage, and set it as
797 tearDownClass will stop any servers started, and restore the
798 original environment.
803 KEEP_PROXY_SERVER = None
804 KEEP_WEB_SERVER = None
807 def _restore_dict(src, dest):
808 for key in list(dest.keys()):
815 cls._orig_environ = os.environ.copy()
816 cls._orig_config = arvados.config.settings().copy()
817 cls._cleanup_funcs = []
818 os.environ.pop('ARVADOS_KEEP_SERVICES', None)
819 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
820 for server_kwargs, start_func, stop_func in (
821 (cls.MAIN_SERVER, run, reset),
822 (cls.WS_SERVER, run_ws, stop_ws),
823 (cls.KEEP_SERVER, run_keep, stop_keep),
824 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
825 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
826 if server_kwargs is not None:
827 start_func(**server_kwargs)
828 cls._cleanup_funcs.append(stop_func)
829 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
830 cls.local_store = tempfile.mkdtemp()
831 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
832 cls._cleanup_funcs.append(
833 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
835 os.environ.pop('KEEP_LOCAL_STORE', None)
836 arvados.config.initialize()
839 def tearDownClass(cls):
840 for clean_func in cls._cleanup_funcs:
842 cls._restore_dict(cls._orig_environ, os.environ)
843 cls._restore_dict(cls._orig_config, arvados.config.settings())
846 if __name__ == "__main__":
849 'start_ws', 'stop_ws',
850 'start_controller', 'stop_controller',
851 'start_keep', 'stop_keep',
852 'start_keep_proxy', 'stop_keep_proxy',
853 'start_keep-web', 'stop_keep-web',
854 'start_arv-git-httpd', 'stop_arv-git-httpd',
855 'start_nginx', 'stop_nginx',
857 parser = argparse.ArgumentParser()
858 parser.add_argument('action', type=str, help="one of {}".format(actions))
859 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
860 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
861 parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions")
863 args = parser.parse_args()
865 if args.action not in actions:
866 print("Unrecognized action '{}'. Actions are: {}.".
867 format(args.action, actions),
870 if args.action == 'start':
871 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
872 run(leave_running_atexit=True)
873 host = os.environ['ARVADOS_API_HOST']
874 if args.auth is not None:
875 token = auth_token(args.auth)
876 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
877 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
878 print("export ARVADOS_API_HOST_INSECURE=true")
881 elif args.action == 'stop':
882 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
883 elif args.action == 'start_ws':
885 elif args.action == 'stop_ws':
887 elif args.action == 'start_controller':
889 elif args.action == 'stop_controller':
891 elif args.action == 'start_keep':
892 run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
893 elif args.action == 'stop_keep':
894 stop_keep(num_servers=args.num_keep_servers)
895 elif args.action == 'start_keep_proxy':
897 elif args.action == 'stop_keep_proxy':
899 elif args.action == 'start_arv-git-httpd':
901 elif args.action == 'stop_arv-git-httpd':
903 elif args.action == 'start_keep-web':
905 elif args.action == 'stop_keep-web':
907 elif args.action == 'start_nginx':
909 print("export ARVADOS_API_HOST=0.0.0.0:{}".format(_getport('controller-ssl')))
910 elif args.action == 'stop_nginx':
913 raise Exception("action recognized but not implemented!?")