1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import print_function
6 from __future__ import division
7 from builtins import str
8 from builtins import range
29 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
30 if __name__ == '__main__' and os.path.exists(
31 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
32 # We're being launched to support another test suite.
33 # Add the Python SDK source to the library path.
34 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
39 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
40 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
41 if 'GOPATH' in os.environ:
42 # Add all GOPATH bin dirs to PATH -- but insert them after the
43 # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
44 # command, not the golang.org/x/tools/cmd/bundle command.
45 gopaths = os.environ['GOPATH'].split(':')
46 addbins = [os.path.join(path, 'bin') for path in gopaths]
48 for path in os.environ['PATH'].split(':'):
50 if os.path.exists(os.path.join(path, 'bundle')):
54 os.environ['PATH'] = ':'.join(newbins)
56 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
57 if not os.path.exists(TEST_TMPDIR):
62 _cached_db_config = {}
64 def find_server_pid(PID_PATH, wait=10):
68 while (not good_pid) and (now <= timeout):
71 with open(PID_PATH, 'r') as f:
72 server_pid = int(f.read())
73 good_pid = (os.kill(server_pid, 0) is None)
74 except EnvironmentError:
83 def kill_server_pid(pidfile, wait=10, passenger_root=False):
84 # Must re-import modules in order to work during atexit
95 # First try to shut down nicely
96 restore_cwd = os.getcwd()
97 os.chdir(passenger_root)
99 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
100 os.chdir(restore_cwd)
101 # Use up to half of the +wait+ period waiting for "passenger
102 # stop" to work. If the process hasn't exited by then, start
103 # sending TERM signals.
107 while now <= deadline and server_pid is None:
109 with open(pidfile, 'r') as f:
110 server_pid = int(f.read())
112 # No pidfile = nothing to kill.
114 except ValueError as error:
115 # Pidfile exists, but we can't parse it. Perhaps the
116 # server has created the file but hasn't written its PID
118 print("Parse error reading pidfile {}: {}".format(pidfile, error),
123 while now <= deadline:
125 exited, _ = os.waitpid(server_pid, os.WNOHANG)
127 _remove_pidfile(pidfile)
130 # already exited, or isn't our child process
134 os.kill(server_pid, signal.SIGTERM)
135 print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
137 except OSError as error:
138 if error.errno == errno.ESRCH:
139 # Thrown by os.getpgid() or os.kill() if the process
140 # does not exist, i.e., our work here is done.
141 _remove_pidfile(pidfile)
147 print("Server PID {} ({}) did not exit, giving up after {}s".
148 format(server_pid, pidfile, wait),
151 def _remove_pidfile(pidfile):
155 if os.path.lexists(pidfile):
158 def find_available_port():
159 """Return an IPv4 port number that is not in use right now.
161 We assume whoever needs to use the returned port is able to reuse
162 a recently used port without waiting for TIME_WAIT (see
163 SO_REUSEADDR / SO_REUSEPORT).
165 Some opportunity for races here, but it's better than choosing
166 something at random and not checking at all. If all of our servers
167 (hey Passenger) knew that listening on port 0 was a thing, the OS
168 would take care of the races, and this wouldn't be needed at all.
171 sock = socket.socket()
172 sock.bind(('0.0.0.0', 0))
173 port = sock.getsockname()[1]
177 def _wait_until_port_listens(port, timeout=10, warn=True):
178 """Wait for a process to start listening on the given port.
180 If nothing listens on the port within the specified timeout (given
181 in seconds), print a warning on stderr before returning.
184 subprocess.check_output(['which', 'netstat'])
185 except subprocess.CalledProcessError:
186 print("WARNING: No `netstat` -- cannot wait for port to listen. "+
187 "Sleeping 0.5 and hoping for the best.",
191 deadline = time.time() + timeout
192 while time.time() < deadline:
193 if re.search(r'\ntcp.*:'+str(port)+' .* LISTEN *\n', subprocess.check_output(['netstat', '-Wln']).decode()):
198 "WARNING: Nothing is listening on port {} (waited {} seconds).".
199 format(port, timeout),
203 def _logfilename(label):
204 """Set up a labelled log file, and return a path to write logs to.
206 Normally, the returned path is {tmpdir}/{label}.log.
208 In debug mode, logs are also written to stderr, with [label]
209 prepended to each line. The returned path is a FIFO.
211 +label+ should contain only alphanumerics: it is also used as part
212 of the FIFO filename.
215 logfilename = os.path.join(TEST_TMPDIR, label+'.log')
216 if not os.environ.get('ARVADOS_DEBUG', ''):
218 fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
221 except OSError as error:
222 if error.errno != errno.ENOENT:
224 os.mkfifo(fifo, 0o700)
225 stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
226 # open(fifo, 'r') would block waiting for someone to open the fifo
227 # for writing, so we need a separate cat process to open it for
229 cat = subprocess.Popen(
230 stdbuf+['cat', fifo],
231 stdin=open('/dev/null'),
232 stdout=subprocess.PIPE)
233 tee = subprocess.Popen(
234 stdbuf+['tee', '-a', logfilename],
236 stdout=subprocess.PIPE)
238 stdbuf+['sed', '-e', 's/^/['+label+'] /'],
243 def run(leave_running_atexit=False):
244 """Ensure an API server is running, and ARVADOS_API_* env vars have
245 admin credentials for it.
247 If ARVADOS_TEST_API_HOST is set, a parent process has started a
248 test server for us to use: we just need to reset() it using the
251 If a previous call to run() started a new server process, and it
252 is still running, we just need to reset() it to fixture state and
255 If neither of those options work out, we'll really start a new
260 # Delete cached discovery documents.
262 # This will clear cached docs that belong to other processes (like
263 # concurrent test suites) even if they're still running. They should
264 # be able to tolerate that.
265 for fn in glob.glob(os.path.join(
266 str(arvados.http_cache('discovery')),
267 '*,arvados,v1,rest,*')):
270 pid_file = _pidfile('api')
271 pid_file_ok = find_server_pid(pid_file, 0)
273 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
274 if existing_api_host and pid_file_ok:
275 if existing_api_host == my_api_host:
279 # Fall through to shutdown-and-start case.
282 # Server was provided by parent. Can't recover if it's
286 # Before trying to start up our own server, call stop() to avoid
287 # "Phusion Passenger Standalone is already running on PID 12345".
288 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
289 # we know the server is ours to kill.)
292 restore_cwd = os.getcwd()
293 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
294 os.chdir(api_src_dir)
296 # Either we haven't started a server of our own yet, or it has
297 # died, or we have lost our credentials, or something else is
298 # preventing us from calling reset(). Start a new one.
300 if not os.path.exists('tmp'):
303 if not os.path.exists('tmp/api'):
304 os.makedirs('tmp/api')
306 if not os.path.exists('tmp/logs'):
307 os.makedirs('tmp/logs')
309 # Install the git repository fixtures.
310 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
311 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
312 if not os.path.isdir(gitdir):
314 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
316 port = internal_port_from_config("RailsAPI")
317 env = os.environ.copy()
318 env['RAILS_ENV'] = 'test'
319 env.pop('ARVADOS_WEBSOCKETS', None)
320 env.pop('ARVADOS_TEST_API_HOST', None)
321 env.pop('ARVADOS_API_HOST', None)
322 env.pop('ARVADOS_API_HOST_INSECURE', None)
323 env.pop('ARVADOS_API_TOKEN', None)
324 start_msg = subprocess.check_output(
326 'passenger', 'start', '-d', '-p{}'.format(port),
327 '--pid-file', pid_file,
328 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
330 '--ssl-certificate', 'tmp/self-signed.pem',
331 '--ssl-certificate-key', 'tmp/self-signed.key'],
334 if not leave_running_atexit:
335 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
337 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
340 "Passenger did not report endpoint: {}".format(start_msg))
341 my_api_host = match.group(1)
342 os.environ['ARVADOS_API_HOST'] = my_api_host
344 # Make sure the server has written its pid file and started
345 # listening on its TCP port
346 find_server_pid(pid_file)
347 _wait_until_port_listens(port)
350 os.chdir(restore_cwd)
353 """Reset the test server to fixture state.
355 This resets the ARVADOS_TEST_API_HOST provided by a parent process
356 if any, otherwise the server started by run().
358 It also resets ARVADOS_* environment vars to point to the test
359 server with admin credentials.
361 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
362 token = auth_token('admin')
363 httpclient = httplib2.Http(ca_certs=os.path.join(
364 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
366 'https://{}/database/reset'.format(existing_api_host),
368 headers={'Authorization': 'OAuth2 {}'.format(token), 'Connection':'close'})
370 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
371 os.environ['ARVADOS_API_TOKEN'] = token
372 os.environ['ARVADOS_API_HOST'] = existing_api_host
374 def stop(force=False):
375 """Stop the API server, if one is running.
377 If force==False, kill it only if we started it ourselves. (This
378 supports the use case where a Python test suite calls run(), but
379 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
380 process, and the test suite cleans up after itself by calling
381 stop(). In this case the test server provided by the parent
382 process should be left alone.)
384 If force==True, kill it even if we didn't start it
385 ourselves. (This supports the use case in __main__, where "run"
386 and "stop" happen in different processes.)
389 if force or my_api_host is not None:
390 kill_server_pid(_pidfile('api'))
394 with open(os.environ["ARVADOS_CONFIG"]) as f:
395 return yaml.safe_load(f)
397 def internal_port_from_config(service):
398 return int(list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys())[0].split(":")[2])
400 def external_port_from_config(service):
401 return int(get_config()["Clusters"]["zzzzz"]["Services"][service]["ExternalURL"].split(":")[2])
403 def run_controller():
404 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
407 logf = open(_logfilename('controller'), 'a')
408 port = internal_port_from_config("Controller")
409 controller = subprocess.Popen(
410 ["arvados-server", "controller"],
411 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
412 with open(_pidfile('controller'), 'w') as f:
413 f.write(str(controller.pid))
414 _wait_until_port_listens(port)
417 def stop_controller():
418 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
420 kill_server_pid(_pidfile('controller'))
423 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
426 port = internal_port_from_config("Websocket")
427 logf = open(_logfilename('ws'), 'a')
428 ws = subprocess.Popen(["ws"],
429 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
430 with open(_pidfile('ws'), 'w') as f:
432 _wait_until_port_listens(port)
436 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
438 kill_server_pid(_pidfile('ws'))
440 def _start_keep(n, keep_args):
441 keep0 = tempfile.mkdtemp()
442 port = find_available_port()
443 keep_cmd = ["keepstore",
444 "-volume={}".format(keep0),
445 "-listen=:{}".format(port),
446 "-pid="+_pidfile('keep{}'.format(n))]
448 for arg, val in keep_args.items():
449 keep_cmd.append("{}={}".format(arg, val))
451 with open(_logfilename('keep{}'.format(n)), 'a') as logf:
452 with open('/dev/null') as _stdin:
453 kp0 = subprocess.Popen(
454 keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
456 with open(_pidfile('keep{}'.format(n)), 'w') as f:
457 f.write(str(kp0.pid))
459 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
462 _wait_until_port_listens(port)
466 def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
467 stop_keep(num_servers)
470 if not blob_signing_key:
471 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
472 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
473 keep_args['-blob-signing-key-file'] = f.name
474 f.write(blob_signing_key)
475 keep_args['-enforce-permissions'] = str(enforce_permissions).lower()
476 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
477 keep_args['-data-manager-token-file'] = f.name
478 f.write(auth_token('data_manager'))
479 keep_args['-never-delete'] = 'false'
483 host=os.environ['ARVADOS_API_HOST'],
484 token=os.environ['ARVADOS_API_TOKEN'],
487 for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']:
488 api.keep_services().delete(uuid=d['uuid']).execute()
489 for d in api.keep_disks().list().execute()['items']:
490 api.keep_disks().delete(uuid=d['uuid']).execute()
492 for d in range(0, num_servers):
493 port = _start_keep(d, keep_args)
494 svc = api.keep_services().create(body={'keep_service': {
495 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
496 'service_host': 'localhost',
497 'service_port': port,
498 'service_type': 'disk',
499 'service_ssl_flag': False,
501 api.keep_disks().create(body={
502 'keep_disk': {'keep_service_uuid': svc['uuid'] }
505 # If keepproxy and/or keep-web is running, send SIGHUP to make
506 # them discover the new keepstore services.
507 for svc in ('keepproxy', 'keep-web'):
508 pidfile = _pidfile('keepproxy')
509 if os.path.exists(pidfile):
511 with open(pidfile) as pid:
512 os.kill(int(pid.read()), signal.SIGHUP)
517 kill_server_pid(_pidfile('keep{}'.format(n)))
518 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
519 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
520 shutil.rmtree(r.read(), True)
521 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
522 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
523 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
525 def stop_keep(num_servers=2):
526 for n in range(0, num_servers):
529 def run_keep_proxy():
530 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
531 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(internal_port_from_config('Keepproxy'))
535 port = internal_port_from_config("Keepproxy")
536 env = os.environ.copy()
537 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
538 logf = open(_logfilename('keepproxy'), 'a')
539 kp = subprocess.Popen(
541 '-pid='+_pidfile('keepproxy'),
542 '-listen=:{}'.format(port)],
543 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
545 print("Using API %s token %s" % (os.environ['ARVADOS_API_HOST'], auth_token('admin')), file=sys.stdout)
548 host=os.environ['ARVADOS_API_HOST'],
549 token=auth_token('admin'),
551 for d in api.keep_services().list(
552 filters=[['service_type','=','proxy']]).execute()['items']:
553 api.keep_services().delete(uuid=d['uuid']).execute()
554 api.keep_services().create(body={'keep_service': {
555 'service_host': 'localhost',
556 'service_port': port,
557 'service_type': 'proxy',
558 'service_ssl_flag': False,
560 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
561 _wait_until_port_listens(port)
563 def stop_keep_proxy():
564 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
566 kill_server_pid(_pidfile('keepproxy'))
568 def run_arv_git_httpd():
569 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
573 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
574 gitport = internal_port_from_config("GitHTTP")
575 env = os.environ.copy()
576 env.pop('ARVADOS_API_TOKEN', None)
577 logf = open(_logfilename('arv-git-httpd'), 'a')
578 agh = subprocess.Popen(
580 '-repo-root='+gitdir+'/test',
581 '-management-token=e687950a23c3a9bceec28c6223a06c79',
582 '-address=:'+str(gitport)],
583 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
584 with open(_pidfile('arv-git-httpd'), 'w') as f:
585 f.write(str(agh.pid))
586 _wait_until_port_listens(gitport)
588 def stop_arv_git_httpd():
589 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
591 kill_server_pid(_pidfile('arv-git-httpd'))
594 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
598 keepwebport = internal_port_from_config("WebDAV")
599 env = os.environ.copy()
600 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
601 logf = open(_logfilename('keep-web'), 'a')
602 keepweb = subprocess.Popen(
605 '-attachment-only-host=download',
606 '-management-token=e687950a23c3a9bceec28c6223a06c79',
607 '-listen=:'+str(keepwebport)],
608 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
609 with open(_pidfile('keep-web'), 'w') as f:
610 f.write(str(keepweb.pid))
611 _wait_until_port_listens(keepwebport)
614 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
616 kill_server_pid(_pidfile('keep-web'))
619 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
623 nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
624 nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")
625 nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV")
626 nginxconf['KEEPWEBDLSSLPORT'] = external_port_from_config("WebDAVDownload")
627 nginxconf['KEEPWEBSSLPORT'] = external_port_from_config("WebDAV")
628 nginxconf['KEEPPROXYPORT'] = internal_port_from_config("Keepproxy")
629 nginxconf['KEEPPROXYSSLPORT'] = external_port_from_config("Keepproxy")
630 nginxconf['GITPORT'] = internal_port_from_config("GitHTTP")
631 nginxconf['GITSSLPORT'] = external_port_from_config("GitHTTP")
632 nginxconf['WSPORT'] = internal_port_from_config("Websocket")
633 nginxconf['WSSPORT'] = external_port_from_config("Websocket")
634 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
635 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
636 nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
637 nginxconf['ERRORLOG'] = _logfilename('nginx_error')
638 nginxconf['TMPDIR'] = TEST_TMPDIR
640 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
641 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
642 with open(conffile, 'w') as f:
645 lambda match: str(nginxconf.get(match.group(1))),
646 open(conftemplatefile).read()))
648 env = os.environ.copy()
649 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
651 nginx = subprocess.Popen(
653 '-g', 'error_log stderr info;',
654 '-g', 'pid '+_pidfile('nginx')+';',
656 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
659 rails_api_port = find_available_port()
660 controller_port = find_available_port()
661 controller_external_port = find_available_port()
662 websocket_port = find_available_port()
663 websocket_external_port = find_available_port()
664 git_httpd_port = find_available_port()
665 git_httpd_external_port = find_available_port()
666 keepproxy_port = find_available_port()
667 keepproxy_external_port = find_available_port()
668 keep_web_port = find_available_port()
669 keep_web_external_port = find_available_port()
670 keep_web_dl_port = find_available_port()
671 keep_web_dl_external_port = find_available_port()
673 if "CONFIGSRC" in os.environ:
674 dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
676 dbconf = "/etc/arvados/config.yml"
678 print("Getting config from %s" % dbconf, file=sys.stderr)
680 pgconnection = list(yaml.safe_load(open(dbconf))["Clusters"].values())[0]["PostgreSQL"]["Connection"]
682 if "test" not in pgconnection["dbname"]:
683 pgconnection["dbname"] = "arvados_test"
690 "ExternalURL": "https://localhost:%s" % controller_external_port,
694 "ExternalURL": "https://localhost:%s" % websocket_external_port,
698 "ExternalURL": "https://localhost:%s" % git_httpd_external_port,
702 "ExternalURL": "https://localhost:%s" % keepproxy_external_port,
706 "ExternalURL": "https://localhost:%s" % keep_web_external_port,
710 "ExternalURL": "https://localhost:%s" % keep_web_dl_external_port,
714 services["RailsAPI"]["InternalURLs"]["https://localhost:%s"%rails_api_port] = {}
715 services["Controller"]["InternalURLs"]["http://localhost:%s"%controller_port] = {}
716 services["Websocket"]["InternalURLs"]["http://localhost:%s"%websocket_port] = {}
717 services["GitHTTP"]["InternalURLs"]["http://localhost:%s"%git_httpd_port] = {}
718 services["Keepproxy"]["InternalURLs"]["http://localhost:%s"%keepproxy_port] = {}
719 services["WebDAV"]["InternalURLs"]["http://localhost:%s"%keep_web_port] = {}
720 services["WebDAVDownload"]["InternalURLs"]["http://localhost:%s"%keep_web_dl_port] = {}
725 "EnableBetaController14287": ('14287' in os.environ.get('ARVADOS_EXPERIMENTAL', '')),
726 "ManagementToken": "e687950a23c3a9bceec28c6223a06c79",
728 "RequestTimeout": "30s"
731 "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug')
734 "Connection": pgconnection,
744 conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
745 with open(conf, 'w') as f:
746 yaml.safe_dump(config, f)
748 ex = "export ARVADOS_CONFIG="+conf
749 print(ex, file=sys.stderr)
754 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
756 kill_server_pid(_pidfile('nginx'))
758 def _pidfile(program):
759 return os.path.join(TEST_TMPDIR, program + '.pid')
762 '''load a fixture yaml file'''
763 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
767 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
768 yaml_file = yaml_file[0:trim_index]
771 return yaml.safe_load(yaml_file)
773 def auth_token(token_name):
774 return fixture("api_client_authorizations")[token_name]["api_token"]
776 def authorize_with(token_name):
777 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
778 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
779 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
780 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
782 class TestCaseWithServers(unittest.TestCase):
783 """TestCase to start and stop supporting Arvados servers.
785 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
786 class variables as a dictionary of keyword arguments. If you do,
787 setUpClass will start the corresponding servers by passing these
788 keyword arguments to the run, run_keep, and/or run_keep_server
789 functions, respectively. It will also set Arvados environment
790 variables to point to these servers appropriately. If you don't
791 run a Keep or Keep proxy server, setUpClass will set up a
792 temporary directory for Keep local storage, and set it as
795 tearDownClass will stop any servers started, and restore the
796 original environment.
801 KEEP_PROXY_SERVER = None
802 KEEP_WEB_SERVER = None
805 def _restore_dict(src, dest):
806 for key in list(dest.keys()):
813 cls._orig_environ = os.environ.copy()
814 cls._orig_config = arvados.config.settings().copy()
815 cls._cleanup_funcs = []
816 os.environ.pop('ARVADOS_KEEP_SERVICES', None)
817 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
818 for server_kwargs, start_func, stop_func in (
819 (cls.MAIN_SERVER, run, reset),
820 (cls.WS_SERVER, run_ws, stop_ws),
821 (cls.KEEP_SERVER, run_keep, stop_keep),
822 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
823 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
824 if server_kwargs is not None:
825 start_func(**server_kwargs)
826 cls._cleanup_funcs.append(stop_func)
827 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
828 cls.local_store = tempfile.mkdtemp()
829 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
830 cls._cleanup_funcs.append(
831 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
833 os.environ.pop('KEEP_LOCAL_STORE', None)
834 arvados.config.initialize()
837 def tearDownClass(cls):
838 for clean_func in cls._cleanup_funcs:
840 cls._restore_dict(cls._orig_environ, os.environ)
841 cls._restore_dict(cls._orig_config, arvados.config.settings())
844 if __name__ == "__main__":
847 'start_ws', 'stop_ws',
848 'start_controller', 'stop_controller',
849 'start_keep', 'stop_keep',
850 'start_keep_proxy', 'stop_keep_proxy',
851 'start_keep-web', 'stop_keep-web',
852 'start_arv-git-httpd', 'stop_arv-git-httpd',
853 'start_nginx', 'stop_nginx', 'setup_config',
855 parser = argparse.ArgumentParser()
856 parser.add_argument('action', type=str, help="one of {}".format(actions))
857 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
858 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
859 parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions")
861 args = parser.parse_args()
863 if args.action not in actions:
864 print("Unrecognized action '{}'. Actions are: {}.".
865 format(args.action, actions),
868 if args.action == 'start':
869 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
870 run(leave_running_atexit=True)
871 host = os.environ['ARVADOS_API_HOST']
872 if args.auth is not None:
873 token = auth_token(args.auth)
874 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
875 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
876 print("export ARVADOS_API_HOST_INSECURE=true")
879 elif args.action == 'stop':
880 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
881 elif args.action == 'start_ws':
883 elif args.action == 'stop_ws':
885 elif args.action == 'start_controller':
887 elif args.action == 'stop_controller':
889 elif args.action == 'start_keep':
890 run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
891 elif args.action == 'stop_keep':
892 stop_keep(num_servers=args.num_keep_servers)
893 elif args.action == 'start_keep_proxy':
895 elif args.action == 'stop_keep_proxy':
897 elif args.action == 'start_arv-git-httpd':
899 elif args.action == 'stop_arv-git-httpd':
901 elif args.action == 'start_keep-web':
903 elif args.action == 'stop_keep-web':
905 elif args.action == 'start_nginx':
907 print("export ARVADOS_API_HOST=0.0.0.0:{}".format(external_port_from_config('Controller')))
908 elif args.action == 'stop_nginx':
910 elif args.action == 'setup_config':
913 raise Exception("action recognized but not implemented!?")