1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import print_function
6 from __future__ import division
7 from builtins import str
8 from builtins import range
30 from urllib.parse import urlparse
32 from urlparse import urlparse
34 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
35 if __name__ == '__main__' and os.path.exists(
36 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
37 # We're being launched to support another test suite.
38 # Add the Python SDK source to the library path.
39 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
44 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
45 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
46 if 'GOPATH' in os.environ:
47 # Add all GOPATH bin dirs to PATH -- but insert them after the
48 # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
49 # command, not the golang.org/x/tools/cmd/bundle command.
50 gopaths = os.environ['GOPATH'].split(':')
51 addbins = [os.path.join(path, 'bin') for path in gopaths]
53 for path in os.environ['PATH'].split(':'):
55 if os.path.exists(os.path.join(path, 'bundle')):
59 os.environ['PATH'] = ':'.join(newbins)
61 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
62 if not os.path.exists(TEST_TMPDIR):
67 _cached_db_config = {}
69 def find_server_pid(PID_PATH, wait=10):
73 while (not good_pid) and (now <= timeout):
76 with open(PID_PATH, 'r') as f:
77 server_pid = int(f.read())
78 good_pid = (os.kill(server_pid, 0) is None)
79 except EnvironmentError:
88 def kill_server_pid(pidfile, wait=10, passenger_root=False):
89 # Must re-import modules in order to work during atexit
100 # First try to shut down nicely
101 restore_cwd = os.getcwd()
102 os.chdir(passenger_root)
104 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
105 os.chdir(restore_cwd)
106 # Use up to half of the +wait+ period waiting for "passenger
107 # stop" to work. If the process hasn't exited by then, start
108 # sending TERM signals.
112 while now <= deadline and server_pid is None:
114 with open(pidfile, 'r') as f:
115 server_pid = int(f.read())
117 # No pidfile = nothing to kill.
119 except ValueError as error:
120 # Pidfile exists, but we can't parse it. Perhaps the
121 # server has created the file but hasn't written its PID
123 print("Parse error reading pidfile {}: {}".format(pidfile, error),
128 while now <= deadline:
130 exited, _ = os.waitpid(server_pid, os.WNOHANG)
132 _remove_pidfile(pidfile)
135 # already exited, or isn't our child process
139 os.kill(server_pid, signal.SIGTERM)
140 print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
142 except OSError as error:
143 if error.errno == errno.ESRCH:
144 # Thrown by os.getpgid() or os.kill() if the process
145 # does not exist, i.e., our work here is done.
146 _remove_pidfile(pidfile)
152 print("Server PID {} ({}) did not exit, giving up after {}s".
153 format(server_pid, pidfile, wait),
156 def _remove_pidfile(pidfile):
160 if os.path.lexists(pidfile):
163 def find_available_port():
164 """Return an IPv4 port number that is not in use right now.
166 We assume whoever needs to use the returned port is able to reuse
167 a recently used port without waiting for TIME_WAIT (see
168 SO_REUSEADDR / SO_REUSEPORT).
170 Some opportunity for races here, but it's better than choosing
171 something at random and not checking at all. If all of our servers
172 (hey Passenger) knew that listening on port 0 was a thing, the OS
173 would take care of the races, and this wouldn't be needed at all.
176 sock = socket.socket()
177 sock.bind(('0.0.0.0', 0))
178 port = sock.getsockname()[1]
182 def _wait_until_port_listens(port, timeout=10, warn=True):
183 """Wait for a process to start listening on the given port.
185 If nothing listens on the port within the specified timeout (given
186 in seconds), print a warning on stderr before returning.
189 subprocess.check_output(['which', 'netstat'])
190 except subprocess.CalledProcessError:
191 print("WARNING: No `netstat` -- cannot wait for port to listen. "+
192 "Sleeping 0.5 and hoping for the best.",
196 deadline = time.time() + timeout
197 while time.time() < deadline:
198 if re.search(r'\ntcp.*:'+str(port)+' .* LISTEN *\n', subprocess.check_output(['netstat', '-Wln']).decode()):
203 "WARNING: Nothing is listening on port {} (waited {} seconds).".
204 format(port, timeout),
208 def _logfilename(label):
209 """Set up a labelled log file, and return a path to write logs to.
211 Normally, the returned path is {tmpdir}/{label}.log.
213 In debug mode, logs are also written to stderr, with [label]
214 prepended to each line. The returned path is a FIFO.
216 +label+ should contain only alphanumerics: it is also used as part
217 of the FIFO filename.
220 logfilename = os.path.join(TEST_TMPDIR, label+'.log')
221 if not os.environ.get('ARVADOS_DEBUG', ''):
223 fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
226 except OSError as error:
227 if error.errno != errno.ENOENT:
229 os.mkfifo(fifo, 0o700)
230 stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
231 # open(fifo, 'r') would block waiting for someone to open the fifo
232 # for writing, so we need a separate cat process to open it for
234 cat = subprocess.Popen(
235 stdbuf+['cat', fifo],
236 stdin=open('/dev/null'),
237 stdout=subprocess.PIPE)
238 tee = subprocess.Popen(
239 stdbuf+['tee', '-a', logfilename],
241 stdout=subprocess.PIPE)
243 stdbuf+['sed', '-e', 's/^/['+label+'] /'],
248 def run(leave_running_atexit=False):
249 """Ensure an API server is running, and ARVADOS_API_* env vars have
250 admin credentials for it.
252 If ARVADOS_TEST_API_HOST is set, a parent process has started a
253 test server for us to use: we just need to reset() it using the
256 If a previous call to run() started a new server process, and it
257 is still running, we just need to reset() it to fixture state and
260 If neither of those options work out, we'll really start a new
265 # Delete cached discovery documents.
267 # This will clear cached docs that belong to other processes (like
268 # concurrent test suites) even if they're still running. They should
269 # be able to tolerate that.
270 for fn in glob.glob(os.path.join(
271 str(arvados.http_cache('discovery')),
272 '*,arvados,v1,rest,*')):
275 pid_file = _pidfile('api')
276 pid_file_ok = find_server_pid(pid_file, 0)
278 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
279 if existing_api_host and pid_file_ok:
280 if existing_api_host == my_api_host:
284 # Fall through to shutdown-and-start case.
287 # Server was provided by parent. Can't recover if it's
291 # Before trying to start up our own server, call stop() to avoid
292 # "Phusion Passenger Standalone is already running on PID 12345".
293 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
294 # we know the server is ours to kill.)
297 restore_cwd = os.getcwd()
298 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
299 os.chdir(api_src_dir)
301 # Either we haven't started a server of our own yet, or it has
302 # died, or we have lost our credentials, or something else is
303 # preventing us from calling reset(). Start a new one.
305 if not os.path.exists('tmp'):
308 if not os.path.exists('tmp/api'):
309 os.makedirs('tmp/api')
311 if not os.path.exists('tmp/logs'):
312 os.makedirs('tmp/logs')
314 # Install the git repository fixtures.
315 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
316 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
317 if not os.path.isdir(gitdir):
319 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
321 port = internal_port_from_config("RailsAPI")
322 env = os.environ.copy()
323 env['RAILS_ENV'] = 'test'
324 env.pop('ARVADOS_WEBSOCKETS', None)
325 env.pop('ARVADOS_TEST_API_HOST', None)
326 env.pop('ARVADOS_API_HOST', None)
327 env.pop('ARVADOS_API_HOST_INSECURE', None)
328 env.pop('ARVADOS_API_TOKEN', None)
329 start_msg = subprocess.check_output(
331 'passenger', 'start', '-d', '-p{}'.format(port),
332 '--pid-file', pid_file,
333 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
335 '--ssl-certificate', 'tmp/self-signed.pem',
336 '--ssl-certificate-key', 'tmp/self-signed.key'],
339 if not leave_running_atexit:
340 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
342 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
345 "Passenger did not report endpoint: {}".format(start_msg))
346 my_api_host = match.group(1)
347 os.environ['ARVADOS_API_HOST'] = my_api_host
349 # Make sure the server has written its pid file and started
350 # listening on its TCP port
351 find_server_pid(pid_file)
352 _wait_until_port_listens(port)
355 os.chdir(restore_cwd)
358 """Reset the test server to fixture state.
360 This resets the ARVADOS_TEST_API_HOST provided by a parent process
361 if any, otherwise the server started by run().
363 It also resets ARVADOS_* environment vars to point to the test
364 server with admin credentials.
366 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
367 token = auth_token('admin')
368 httpclient = httplib2.Http(ca_certs=os.path.join(
369 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
371 'https://{}/database/reset'.format(existing_api_host),
373 headers={'Authorization': 'OAuth2 {}'.format(token), 'Connection':'close'})
375 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
376 os.environ['ARVADOS_API_TOKEN'] = token
377 os.environ['ARVADOS_API_HOST'] = existing_api_host
379 def stop(force=False):
380 """Stop the API server, if one is running.
382 If force==False, kill it only if we started it ourselves. (This
383 supports the use case where a Python test suite calls run(), but
384 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
385 process, and the test suite cleans up after itself by calling
386 stop(). In this case the test server provided by the parent
387 process should be left alone.)
389 If force==True, kill it even if we didn't start it
390 ourselves. (This supports the use case in __main__, where "run"
391 and "stop" happen in different processes.)
394 if force or my_api_host is not None:
395 kill_server_pid(_pidfile('api'))
399 with open(os.environ["ARVADOS_CONFIG"]) as f:
400 return yaml.safe_load(f)
402 def internal_port_from_config(service):
404 list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys())[0]).
405 netloc.split(":")[1])
407 def external_port_from_config(service):
408 return int(urlparse(get_config()["Clusters"]["zzzzz"]["Services"][service]["ExternalURL"]).netloc.split(":")[1])
410 def run_controller():
411 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
414 logf = open(_logfilename('controller'), 'a')
415 port = internal_port_from_config("Controller")
416 controller = subprocess.Popen(
417 ["arvados-server", "controller"],
418 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
419 with open(_pidfile('controller'), 'w') as f:
420 f.write(str(controller.pid))
421 _wait_until_port_listens(port)
424 def stop_controller():
425 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
427 kill_server_pid(_pidfile('controller'))
430 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
433 port = internal_port_from_config("Websocket")
434 logf = open(_logfilename('ws'), 'a')
435 ws = subprocess.Popen(["ws"],
436 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
437 with open(_pidfile('ws'), 'w') as f:
439 _wait_until_port_listens(port)
443 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
445 kill_server_pid(_pidfile('ws'))
447 def _start_keep(n, keep_args):
448 keep0 = tempfile.mkdtemp()
449 port = find_available_port()
450 keep_cmd = ["keepstore",
451 "-volume={}".format(keep0),
452 "-listen=:{}".format(port),
453 "-pid="+_pidfile('keep{}'.format(n))]
455 for arg, val in keep_args.items():
456 keep_cmd.append("{}={}".format(arg, val))
458 with open(_logfilename('keep{}'.format(n)), 'a') as logf:
459 with open('/dev/null') as _stdin:
460 kp0 = subprocess.Popen(
461 keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
463 with open(_pidfile('keep{}'.format(n)), 'w') as f:
464 f.write(str(kp0.pid))
466 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
469 _wait_until_port_listens(port)
473 def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
474 stop_keep(num_servers)
477 if not blob_signing_key:
478 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
479 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
480 keep_args['-blob-signing-key-file'] = f.name
481 f.write(blob_signing_key)
482 keep_args['-enforce-permissions'] = str(enforce_permissions).lower()
483 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
484 keep_args['-data-manager-token-file'] = f.name
485 f.write(auth_token('data_manager'))
486 keep_args['-never-delete'] = 'false'
490 host=os.environ['ARVADOS_API_HOST'],
491 token=os.environ['ARVADOS_API_TOKEN'],
494 for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']:
495 api.keep_services().delete(uuid=d['uuid']).execute()
496 for d in api.keep_disks().list().execute()['items']:
497 api.keep_disks().delete(uuid=d['uuid']).execute()
499 for d in range(0, num_servers):
500 port = _start_keep(d, keep_args)
501 svc = api.keep_services().create(body={'keep_service': {
502 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
503 'service_host': 'localhost',
504 'service_port': port,
505 'service_type': 'disk',
506 'service_ssl_flag': False,
508 api.keep_disks().create(body={
509 'keep_disk': {'keep_service_uuid': svc['uuid'] }
512 # If keepproxy and/or keep-web is running, send SIGHUP to make
513 # them discover the new keepstore services.
514 for svc in ('keepproxy', 'keep-web'):
515 pidfile = _pidfile('keepproxy')
516 if os.path.exists(pidfile):
518 with open(pidfile) as pid:
519 os.kill(int(pid.read()), signal.SIGHUP)
524 kill_server_pid(_pidfile('keep{}'.format(n)))
525 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
526 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
527 shutil.rmtree(r.read(), True)
528 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
529 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
530 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
532 def stop_keep(num_servers=2):
533 for n in range(0, num_servers):
536 def run_keep_proxy():
537 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
538 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(internal_port_from_config('Keepproxy'))
542 port = internal_port_from_config("Keepproxy")
543 env = os.environ.copy()
544 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
545 logf = open(_logfilename('keepproxy'), 'a')
546 kp = subprocess.Popen(
547 ['keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
549 with open(_pidfile('keepproxy'), 'w') as f:
551 _wait_until_port_listens(port)
553 print("Using API %s token %s" % (os.environ['ARVADOS_API_HOST'], auth_token('admin')), file=sys.stdout)
556 host=os.environ['ARVADOS_API_HOST'],
557 token=auth_token('admin'),
559 for d in api.keep_services().list(
560 filters=[['service_type','=','proxy']]).execute()['items']:
561 api.keep_services().delete(uuid=d['uuid']).execute()
562 api.keep_services().create(body={'keep_service': {
563 'service_host': 'localhost',
564 'service_port': port,
565 'service_type': 'proxy',
566 'service_ssl_flag': False,
568 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
569 _wait_until_port_listens(port)
571 def stop_keep_proxy():
572 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
574 kill_server_pid(_pidfile('keepproxy'))
576 def run_arv_git_httpd():
577 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
581 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
582 gitport = internal_port_from_config("GitHTTP")
583 env = os.environ.copy()
584 env.pop('ARVADOS_API_TOKEN', None)
585 logf = open(_logfilename('arv-git-httpd'), 'a')
586 agh = subprocess.Popen(
588 '-repo-root='+gitdir+'/test',
589 '-management-token=e687950a23c3a9bceec28c6223a06c79',
590 '-address=:'+str(gitport)],
591 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
592 with open(_pidfile('arv-git-httpd'), 'w') as f:
593 f.write(str(agh.pid))
594 _wait_until_port_listens(gitport)
596 def stop_arv_git_httpd():
597 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
599 kill_server_pid(_pidfile('arv-git-httpd'))
602 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
606 keepwebport = internal_port_from_config("WebDAV")
607 env = os.environ.copy()
608 logf = open(_logfilename('keep-web'), 'a')
609 keepweb = subprocess.Popen(
611 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
612 with open(_pidfile('keep-web'), 'w') as f:
613 f.write(str(keepweb.pid))
614 _wait_until_port_listens(keepwebport)
617 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
619 kill_server_pid(_pidfile('keep-web'))
622 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
626 nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
627 nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")
628 nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV")
629 nginxconf['KEEPWEBDLSSLPORT'] = external_port_from_config("WebDAVDownload")
630 nginxconf['KEEPWEBSSLPORT'] = external_port_from_config("WebDAV")
631 nginxconf['KEEPPROXYPORT'] = internal_port_from_config("Keepproxy")
632 nginxconf['KEEPPROXYSSLPORT'] = external_port_from_config("Keepproxy")
633 nginxconf['GITPORT'] = internal_port_from_config("GitHTTP")
634 nginxconf['GITSSLPORT'] = external_port_from_config("GitHTTP")
635 nginxconf['WSPORT'] = internal_port_from_config("Websocket")
636 nginxconf['WSSPORT'] = external_port_from_config("Websocket")
637 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
638 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
639 nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
640 nginxconf['ERRORLOG'] = _logfilename('nginx_error')
641 nginxconf['TMPDIR'] = TEST_TMPDIR
643 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
644 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
645 with open(conffile, 'w') as f:
648 lambda match: str(nginxconf.get(match.group(1))),
649 open(conftemplatefile).read()))
651 env = os.environ.copy()
652 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
654 nginx = subprocess.Popen(
656 '-g', 'error_log stderr info;',
657 '-g', 'pid '+_pidfile('nginx')+';',
659 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
662 rails_api_port = find_available_port()
663 controller_port = find_available_port()
664 controller_external_port = find_available_port()
665 websocket_port = find_available_port()
666 websocket_external_port = find_available_port()
667 git_httpd_port = find_available_port()
668 git_httpd_external_port = find_available_port()
669 keepproxy_port = find_available_port()
670 keepproxy_external_port = find_available_port()
671 keep_web_port = find_available_port()
672 keep_web_external_port = find_available_port()
673 keep_web_dl_port = find_available_port()
674 keep_web_dl_external_port = find_available_port()
676 dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
678 print("Getting config from %s" % dbconf, file=sys.stderr)
680 pgconnection = yaml.safe_load(open(dbconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
682 localhost = "127.0.0.1"
686 "https://%s:%s"%(localhost, rails_api_port): {}
690 "ExternalURL": "https://%s:%s" % (localhost, controller_external_port),
692 "http://%s:%s"%(localhost, controller_port): {}
696 "ExternalURL": "wss://%s:%s/websocket" % (localhost, websocket_external_port),
698 "http://%s:%s"%(localhost, websocket_port): {}
702 "ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port),
704 "http://%s:%s"%(localhost, git_httpd_port): {}
708 "ExternalURL": "https://%s:%s" % (localhost, keepproxy_external_port),
710 "http://%s:%s"%(localhost, keepproxy_port): {}
714 "ExternalURL": "https://%s:%s" % (localhost, keep_web_external_port),
716 "http://%s:%s"%(localhost, keep_web_port): {}
720 "ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port),
722 "http://%s:%s"%(localhost, keep_web_dl_port): {}
730 "EnableBetaController14287": ('14287' in os.environ.get('ARVADOS_EXPERIMENTAL', '')),
731 "ManagementToken": "e687950a23c3a9bceec28c6223a06c79",
733 "RequestTimeout": "30s"
736 "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug')
739 "Connection": pgconnection,
744 "Services": services,
746 "AnonymousUserToken": auth_token('anonymous')
749 "TrustAllContent": True
755 conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
756 with open(conf, 'w') as f:
757 yaml.safe_dump(config, f)
759 ex = "export ARVADOS_CONFIG="+conf
764 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
766 kill_server_pid(_pidfile('nginx'))
768 def _pidfile(program):
769 return os.path.join(TEST_TMPDIR, program + '.pid')
772 '''load a fixture yaml file'''
773 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
777 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
778 yaml_file = yaml_file[0:trim_index]
781 return yaml.safe_load(yaml_file)
783 def auth_token(token_name):
784 return fixture("api_client_authorizations")[token_name]["api_token"]
786 def authorize_with(token_name):
787 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
788 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
789 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
790 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
792 class TestCaseWithServers(unittest.TestCase):
793 """TestCase to start and stop supporting Arvados servers.
795 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
796 class variables as a dictionary of keyword arguments. If you do,
797 setUpClass will start the corresponding servers by passing these
798 keyword arguments to the run, run_keep, and/or run_keep_server
799 functions, respectively. It will also set Arvados environment
800 variables to point to these servers appropriately. If you don't
801 run a Keep or Keep proxy server, setUpClass will set up a
802 temporary directory for Keep local storage, and set it as
805 tearDownClass will stop any servers started, and restore the
806 original environment.
811 KEEP_PROXY_SERVER = None
812 KEEP_WEB_SERVER = None
815 def _restore_dict(src, dest):
816 for key in list(dest.keys()):
823 cls._orig_environ = os.environ.copy()
824 cls._orig_config = arvados.config.settings().copy()
825 cls._cleanup_funcs = []
826 os.environ.pop('ARVADOS_KEEP_SERVICES', None)
827 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
828 for server_kwargs, start_func, stop_func in (
829 (cls.MAIN_SERVER, run, reset),
830 (cls.WS_SERVER, run_ws, stop_ws),
831 (cls.KEEP_SERVER, run_keep, stop_keep),
832 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
833 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
834 if server_kwargs is not None:
835 start_func(**server_kwargs)
836 cls._cleanup_funcs.append(stop_func)
837 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
838 cls.local_store = tempfile.mkdtemp()
839 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
840 cls._cleanup_funcs.append(
841 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
843 os.environ.pop('KEEP_LOCAL_STORE', None)
844 arvados.config.initialize()
847 def tearDownClass(cls):
848 for clean_func in cls._cleanup_funcs:
850 cls._restore_dict(cls._orig_environ, os.environ)
851 cls._restore_dict(cls._orig_config, arvados.config.settings())
854 if __name__ == "__main__":
857 'start_ws', 'stop_ws',
858 'start_controller', 'stop_controller',
859 'start_keep', 'stop_keep',
860 'start_keep_proxy', 'stop_keep_proxy',
861 'start_keep-web', 'stop_keep-web',
862 'start_arv-git-httpd', 'stop_arv-git-httpd',
863 'start_nginx', 'stop_nginx', 'setup_config',
865 parser = argparse.ArgumentParser()
866 parser.add_argument('action', type=str, help="one of {}".format(actions))
867 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
868 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
869 parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions")
871 args = parser.parse_args()
873 if args.action not in actions:
874 print("Unrecognized action '{}'. Actions are: {}.".
875 format(args.action, actions),
878 if args.action == 'start':
879 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
880 run(leave_running_atexit=True)
881 host = os.environ['ARVADOS_API_HOST']
882 if args.auth is not None:
883 token = auth_token(args.auth)
884 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
885 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
886 print("export ARVADOS_API_HOST_INSECURE=true")
889 elif args.action == 'stop':
890 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
891 elif args.action == 'start_ws':
893 elif args.action == 'stop_ws':
895 elif args.action == 'start_controller':
897 elif args.action == 'stop_controller':
899 elif args.action == 'start_keep':
900 run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
901 elif args.action == 'stop_keep':
902 stop_keep(num_servers=args.num_keep_servers)
903 elif args.action == 'start_keep_proxy':
905 elif args.action == 'stop_keep_proxy':
907 elif args.action == 'start_arv-git-httpd':
909 elif args.action == 'stop_arv-git-httpd':
911 elif args.action == 'start_keep-web':
913 elif args.action == 'stop_keep-web':
915 elif args.action == 'start_nginx':
917 print("export ARVADOS_API_HOST=0.0.0.0:{}".format(external_port_from_config('Controller')))
918 elif args.action == 'stop_nginx':
920 elif args.action == 'setup_config':
923 raise Exception("action recognized but not implemented!?")