1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import print_function
6 from __future__ import division
7 from builtins import str
8 from builtins import range
30 from urllib.parse import urlparse
32 from urlparse import urlparse
34 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
35 if __name__ == '__main__' and os.path.exists(
36 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
37 # We're being launched to support another test suite.
38 # Add the Python SDK source to the library path.
39 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
44 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
45 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
46 if 'GOPATH' in os.environ:
47 # Add all GOPATH bin dirs to PATH -- but insert them after the
48 # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
49 # command, not the golang.org/x/tools/cmd/bundle command.
50 gopaths = os.environ['GOPATH'].split(':')
51 addbins = [os.path.join(path, 'bin') for path in gopaths]
53 for path in os.environ['PATH'].split(':'):
55 if os.path.exists(os.path.join(path, 'bundle')):
59 os.environ['PATH'] = ':'.join(newbins)
61 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
62 if not os.path.exists(TEST_TMPDIR):
67 _cached_db_config = {}
69 def find_server_pid(PID_PATH, wait=10):
73 while (not good_pid) and (now <= timeout):
76 with open(PID_PATH, 'r') as f:
77 server_pid = int(f.read())
78 good_pid = (os.kill(server_pid, 0) is None)
79 except EnvironmentError:
88 def kill_server_pid(pidfile, wait=10, passenger_root=False):
89 # Must re-import modules in order to work during atexit
100 # First try to shut down nicely
101 restore_cwd = os.getcwd()
102 os.chdir(passenger_root)
104 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
105 os.chdir(restore_cwd)
106 # Use up to half of the +wait+ period waiting for "passenger
107 # stop" to work. If the process hasn't exited by then, start
108 # sending TERM signals.
112 while now <= deadline and server_pid is None:
114 with open(pidfile, 'r') as f:
115 server_pid = int(f.read())
117 # No pidfile = nothing to kill.
119 except ValueError as error:
120 # Pidfile exists, but we can't parse it. Perhaps the
121 # server has created the file but hasn't written its PID
123 print("Parse error reading pidfile {}: {}".format(pidfile, error),
128 while now <= deadline:
130 exited, _ = os.waitpid(server_pid, os.WNOHANG)
132 _remove_pidfile(pidfile)
135 # already exited, or isn't our child process
139 os.kill(server_pid, signal.SIGTERM)
140 print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
142 except OSError as error:
143 if error.errno == errno.ESRCH:
144 # Thrown by os.getpgid() or os.kill() if the process
145 # does not exist, i.e., our work here is done.
146 _remove_pidfile(pidfile)
152 print("Server PID {} ({}) did not exit, giving up after {}s".
153 format(server_pid, pidfile, wait),
156 def _remove_pidfile(pidfile):
160 if os.path.lexists(pidfile):
163 def find_available_port():
164 """Return an IPv4 port number that is not in use right now.
166 We assume whoever needs to use the returned port is able to reuse
167 a recently used port without waiting for TIME_WAIT (see
168 SO_REUSEADDR / SO_REUSEPORT).
170 Some opportunity for races here, but it's better than choosing
171 something at random and not checking at all. If all of our servers
172 (hey Passenger) knew that listening on port 0 was a thing, the OS
173 would take care of the races, and this wouldn't be needed at all.
176 sock = socket.socket()
177 sock.bind(('0.0.0.0', 0))
178 port = sock.getsockname()[1]
182 def _wait_until_port_listens(port, timeout=10, warn=True):
183 """Wait for a process to start listening on the given port.
185 If nothing listens on the port within the specified timeout (given
186 in seconds), print a warning on stderr before returning.
189 subprocess.check_output(['which', 'netstat'])
190 except subprocess.CalledProcessError:
191 print("WARNING: No `netstat` -- cannot wait for port to listen. "+
192 "Sleeping 0.5 and hoping for the best.",
196 deadline = time.time() + timeout
197 while time.time() < deadline:
198 if re.search(r'\ntcp.*:'+str(port)+' .* LISTEN *\n', subprocess.check_output(['netstat', '-Wln']).decode()):
203 "WARNING: Nothing is listening on port {} (waited {} seconds).".
204 format(port, timeout),
208 def _logfilename(label):
209 """Set up a labelled log file, and return a path to write logs to.
211 Normally, the returned path is {tmpdir}/{label}.log.
213 In debug mode, logs are also written to stderr, with [label]
214 prepended to each line. The returned path is a FIFO.
216 +label+ should contain only alphanumerics: it is also used as part
217 of the FIFO filename.
220 logfilename = os.path.join(TEST_TMPDIR, label+'.log')
221 if not os.environ.get('ARVADOS_DEBUG', ''):
223 fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
226 except OSError as error:
227 if error.errno != errno.ENOENT:
229 os.mkfifo(fifo, 0o700)
230 stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
231 # open(fifo, 'r') would block waiting for someone to open the fifo
232 # for writing, so we need a separate cat process to open it for
234 cat = subprocess.Popen(
235 stdbuf+['cat', fifo],
236 stdin=open('/dev/null'),
237 stdout=subprocess.PIPE)
238 tee = subprocess.Popen(
239 stdbuf+['tee', '-a', logfilename],
241 stdout=subprocess.PIPE)
243 stdbuf+['sed', '-e', 's/^/['+label+'] /'],
248 def run(leave_running_atexit=False):
249 """Ensure an API server is running, and ARVADOS_API_* env vars have
250 admin credentials for it.
252 If ARVADOS_TEST_API_HOST is set, a parent process has started a
253 test server for us to use: we just need to reset() it using the
256 If a previous call to run() started a new server process, and it
257 is still running, we just need to reset() it to fixture state and
260 If neither of those options work out, we'll really start a new
265 # Delete cached discovery documents.
267 # This will clear cached docs that belong to other processes (like
268 # concurrent test suites) even if they're still running. They should
269 # be able to tolerate that.
270 for fn in glob.glob(os.path.join(
271 str(arvados.http_cache('discovery')),
272 '*,arvados,v1,rest,*')):
275 pid_file = _pidfile('api')
276 pid_file_ok = find_server_pid(pid_file, 0)
278 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
279 if existing_api_host and pid_file_ok:
280 if existing_api_host == my_api_host:
284 # Fall through to shutdown-and-start case.
287 # Server was provided by parent. Can't recover if it's
291 # Before trying to start up our own server, call stop() to avoid
292 # "Phusion Passenger Standalone is already running on PID 12345".
293 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
294 # we know the server is ours to kill.)
297 restore_cwd = os.getcwd()
298 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
299 os.chdir(api_src_dir)
301 # Either we haven't started a server of our own yet, or it has
302 # died, or we have lost our credentials, or something else is
303 # preventing us from calling reset(). Start a new one.
305 if not os.path.exists('tmp'):
308 if not os.path.exists('tmp/api'):
309 os.makedirs('tmp/api')
311 if not os.path.exists('tmp/logs'):
312 os.makedirs('tmp/logs')
314 # Install the git repository fixtures.
315 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
316 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
317 if not os.path.isdir(gitdir):
319 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
321 port = internal_port_from_config("RailsAPI")
322 env = os.environ.copy()
323 env['RAILS_ENV'] = 'test'
324 env['ARVADOS_RAILS_LOG_TO_STDOUT'] = '1'
325 env.pop('ARVADOS_WEBSOCKETS', None)
326 env.pop('ARVADOS_TEST_API_HOST', None)
327 env.pop('ARVADOS_API_HOST', None)
328 env.pop('ARVADOS_API_HOST_INSECURE', None)
329 env.pop('ARVADOS_API_TOKEN', None)
330 logf = open(_logfilename('railsapi'), 'a')
331 railsapi = subprocess.Popen(
333 'passenger', 'start', '-p{}'.format(port),
334 '--pid-file', pid_file,
335 '--log-file', '/dev/stdout',
337 '--ssl-certificate', 'tmp/self-signed.pem',
338 '--ssl-certificate-key', 'tmp/self-signed.key'],
339 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
341 if not leave_running_atexit:
342 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
344 my_api_host = "127.0.0.1:"+str(port)
345 os.environ['ARVADOS_API_HOST'] = my_api_host
347 # Make sure the server has written its pid file and started
348 # listening on its TCP port
349 _wait_until_port_listens(port)
350 find_server_pid(pid_file)
353 os.chdir(restore_cwd)
356 """Reset the test server to fixture state.
358 This resets the ARVADOS_TEST_API_HOST provided by a parent process
359 if any, otherwise the server started by run().
361 It also resets ARVADOS_* environment vars to point to the test
362 server with admin credentials.
364 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
365 token = auth_token('admin')
366 httpclient = httplib2.Http(ca_certs=os.path.join(
367 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
369 'https://{}/database/reset'.format(existing_api_host),
371 headers={'Authorization': 'OAuth2 {}'.format(token), 'Connection':'close'})
373 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
374 os.environ['ARVADOS_API_TOKEN'] = token
375 os.environ['ARVADOS_API_HOST'] = existing_api_host
377 def stop(force=False):
378 """Stop the API server, if one is running.
380 If force==False, kill it only if we started it ourselves. (This
381 supports the use case where a Python test suite calls run(), but
382 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
383 process, and the test suite cleans up after itself by calling
384 stop(). In this case the test server provided by the parent
385 process should be left alone.)
387 If force==True, kill it even if we didn't start it
388 ourselves. (This supports the use case in __main__, where "run"
389 and "stop" happen in different processes.)
392 if force or my_api_host is not None:
393 kill_server_pid(_pidfile('api'))
397 with open(os.environ["ARVADOS_CONFIG"]) as f:
398 return yaml.safe_load(f)
400 def internal_port_from_config(service, idx=0):
402 sorted(list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys()))[idx]).
403 netloc.split(":")[1])
405 def external_port_from_config(service):
406 return int(urlparse(get_config()["Clusters"]["zzzzz"]["Services"][service]["ExternalURL"]).netloc.split(":")[1])
408 def run_controller():
409 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
412 logf = open(_logfilename('controller'), 'a')
413 port = internal_port_from_config("Controller")
414 controller = subprocess.Popen(
415 ["arvados-server", "controller"],
416 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
417 with open(_pidfile('controller'), 'w') as f:
418 f.write(str(controller.pid))
419 _wait_until_port_listens(port)
422 def stop_controller():
423 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
425 kill_server_pid(_pidfile('controller'))
428 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
431 port = internal_port_from_config("Websocket")
432 logf = open(_logfilename('ws'), 'a')
433 ws = subprocess.Popen(["ws"],
434 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
435 with open(_pidfile('ws'), 'w') as f:
437 _wait_until_port_listens(port)
441 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
443 kill_server_pid(_pidfile('ws'))
445 def _start_keep(n, blob_signing=False):
446 datadir = os.path.join(TEST_TMPDIR, "keep%d.data"%n)
447 if os.path.exists(datadir):
448 shutil.rmtree(datadir)
450 port = internal_port_from_config("Keepstore", idx=n)
452 # Currently, if there are multiple InternalURLs for a single host,
453 # the only way to tell a keepstore process which one it's supposed
454 # to listen on is to supply a redacted version of the config, with
455 # the other InternalURLs removed.
456 conf = os.path.join(TEST_TMPDIR, "keep%d.yaml"%n)
457 confdata = get_config()
458 confdata['Clusters']['zzzzz']['Services']['Keepstore']['InternalURLs'] = {"http://127.0.0.1:%d"%port: {}}
459 confdata['Clusters']['zzzzz']['Collections']['BlobSigning'] = blob_signing
460 with open(conf, 'w') as f:
461 yaml.safe_dump(confdata, f)
462 keep_cmd = ["keepstore", "-config", conf]
464 with open(_logfilename('keep{}'.format(n)), 'a') as logf:
465 with open('/dev/null') as _stdin:
466 child = subprocess.Popen(
467 keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
469 print('child.pid is %d'%child.pid, file=sys.stderr)
470 with open(_pidfile('keep{}'.format(n)), 'w') as f:
471 f.write(str(child.pid))
473 _wait_until_port_listens(port)
477 def run_keep(num_servers=2, **kwargs):
478 stop_keep(num_servers)
482 host=os.environ['ARVADOS_API_HOST'],
483 token=os.environ['ARVADOS_API_TOKEN'],
486 for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']:
487 api.keep_services().delete(uuid=d['uuid']).execute()
488 for d in api.keep_disks().list().execute()['items']:
489 api.keep_disks().delete(uuid=d['uuid']).execute()
491 for d in range(0, num_servers):
492 port = _start_keep(d, **kwargs)
493 svc = api.keep_services().create(body={'keep_service': {
494 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
495 'service_host': 'localhost',
496 'service_port': port,
497 'service_type': 'disk',
498 'service_ssl_flag': False,
500 api.keep_disks().create(body={
501 'keep_disk': {'keep_service_uuid': svc['uuid'] }
504 # If keepproxy and/or keep-web is running, send SIGHUP to make
505 # them discover the new keepstore services.
506 for svc in ('keepproxy', 'keep-web'):
507 pidfile = _pidfile('keepproxy')
508 if os.path.exists(pidfile):
510 with open(pidfile) as pid:
511 os.kill(int(pid.read()), signal.SIGHUP)
516 kill_server_pid(_pidfile('keep{}'.format(n)))
518 def stop_keep(num_servers=2):
519 for n in range(0, num_servers):
522 def run_keep_proxy():
523 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
524 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(internal_port_from_config('Keepproxy'))
528 port = internal_port_from_config("Keepproxy")
529 env = os.environ.copy()
530 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
531 logf = open(_logfilename('keepproxy'), 'a')
532 kp = subprocess.Popen(
533 ['keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
535 with open(_pidfile('keepproxy'), 'w') as f:
537 _wait_until_port_listens(port)
539 print("Using API %s token %s" % (os.environ['ARVADOS_API_HOST'], auth_token('admin')), file=sys.stdout)
542 host=os.environ['ARVADOS_API_HOST'],
543 token=auth_token('admin'),
545 for d in api.keep_services().list(
546 filters=[['service_type','=','proxy']]).execute()['items']:
547 api.keep_services().delete(uuid=d['uuid']).execute()
548 api.keep_services().create(body={'keep_service': {
549 'service_host': 'localhost',
550 'service_port': port,
551 'service_type': 'proxy',
552 'service_ssl_flag': False,
554 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
555 _wait_until_port_listens(port)
557 def stop_keep_proxy():
558 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
560 kill_server_pid(_pidfile('keepproxy'))
562 def run_arv_git_httpd():
563 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
567 gitport = internal_port_from_config("GitHTTP")
568 env = os.environ.copy()
569 env.pop('ARVADOS_API_TOKEN', None)
570 logf = open(_logfilename('arv-git-httpd'), 'a')
571 agh = subprocess.Popen(['arv-git-httpd'],
572 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
573 with open(_pidfile('arv-git-httpd'), 'w') as f:
574 f.write(str(agh.pid))
575 _wait_until_port_listens(gitport)
577 def stop_arv_git_httpd():
578 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
580 kill_server_pid(_pidfile('arv-git-httpd'))
583 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
587 keepwebport = internal_port_from_config("WebDAV")
588 env = os.environ.copy()
589 logf = open(_logfilename('keep-web'), 'a')
590 keepweb = subprocess.Popen(
592 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
593 with open(_pidfile('keep-web'), 'w') as f:
594 f.write(str(keepweb.pid))
595 _wait_until_port_listens(keepwebport)
598 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
600 kill_server_pid(_pidfile('keep-web'))
603 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
607 nginxconf['LISTENHOST'] = 'localhost'
608 nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
609 nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")
610 nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV")
611 nginxconf['KEEPWEBDLSSLPORT'] = external_port_from_config("WebDAVDownload")
612 nginxconf['KEEPWEBSSLPORT'] = external_port_from_config("WebDAV")
613 nginxconf['KEEPPROXYPORT'] = internal_port_from_config("Keepproxy")
614 nginxconf['KEEPPROXYSSLPORT'] = external_port_from_config("Keepproxy")
615 nginxconf['GITPORT'] = internal_port_from_config("GitHTTP")
616 nginxconf['GITSSLPORT'] = external_port_from_config("GitHTTP")
617 nginxconf['WSPORT'] = internal_port_from_config("Websocket")
618 nginxconf['WSSSLPORT'] = external_port_from_config("Websocket")
619 nginxconf['WORKBENCH1PORT'] = internal_port_from_config("Workbench1")
620 nginxconf['WORKBENCH1SSLPORT'] = external_port_from_config("Workbench1")
621 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
622 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
623 nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
624 nginxconf['ERRORLOG'] = _logfilename('nginx_error')
625 nginxconf['TMPDIR'] = TEST_TMPDIR
627 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
628 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
629 with open(conffile, 'w') as f:
631 r'{{([A-Z]+[A-Z0-9]+)}}',
632 lambda match: str(nginxconf.get(match.group(1))),
633 open(conftemplatefile).read()))
635 env = os.environ.copy()
636 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
638 nginx = subprocess.Popen(
640 '-g', 'error_log stderr info;',
641 '-g', 'pid '+_pidfile('nginx')+';',
643 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
644 _wait_until_port_listens(nginxconf['CONTROLLERSSLPORT'])
647 rails_api_port = find_available_port()
648 controller_port = find_available_port()
649 controller_external_port = find_available_port()
650 websocket_port = find_available_port()
651 websocket_external_port = find_available_port()
652 workbench1_port = find_available_port()
653 workbench1_external_port = find_available_port()
654 git_httpd_port = find_available_port()
655 git_httpd_external_port = find_available_port()
656 keepproxy_port = find_available_port()
657 keepproxy_external_port = find_available_port()
658 keepstore_ports = sorted([str(find_available_port()) for _ in xrange(0,4)])
659 keep_web_port = find_available_port()
660 keep_web_external_port = find_available_port()
661 keep_web_dl_port = find_available_port()
662 keep_web_dl_external_port = find_available_port()
664 dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
666 print("Getting config from %s" % dbconf, file=sys.stderr)
668 pgconnection = yaml.safe_load(open(dbconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
670 localhost = "127.0.0.1"
674 "https://%s:%s"%(localhost, rails_api_port): {},
678 "ExternalURL": "https://%s:%s" % (localhost, controller_external_port),
680 "http://%s:%s"%(localhost, controller_port): {},
684 "ExternalURL": "wss://%s:%s/websocket" % (localhost, websocket_external_port),
686 "http://%s:%s"%(localhost, websocket_port): {},
690 "ExternalURL": "https://%s:%s/" % (localhost, workbench1_external_port),
692 "http://%s:%s"%(localhost, workbench1_port): {},
696 "ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port),
698 "http://%s:%s"%(localhost, git_httpd_port): {}
703 "http://%s:%s"%(localhost, port): {} for port in keepstore_ports
707 "ExternalURL": "https://%s:%s" % (localhost, keepproxy_external_port),
709 "http://%s:%s"%(localhost, keepproxy_port): {},
713 "ExternalURL": "https://%s:%s" % (localhost, keep_web_external_port),
715 "http://%s:%s"%(localhost, keep_web_port): {},
719 "ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port),
721 "http://%s:%s"%(localhost, keep_web_dl_port): {},
725 "ExternalURL": "http://localhost:3002",
732 "ManagementToken": "e687950a23c3a9bceec28c6223a06c79",
733 "SystemRootToken": auth_token('system_user'),
735 "RequestTimeout": "30s",
736 "RailsSessionSecretToken": "e24205c490ac07e028fd5f8a692dcb398bcd654eff1aef5f9fe6891994b18483",
739 "ProviderAppID": "arvados-server",
740 "ProviderAppSecret": "608dbf356a327e2d0d4932b60161e212c2d8d8f5e25690d7b622f850a990cd33",
743 "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
746 "Connection": pgconnection,
751 "Services": services,
753 "AnonymousUserToken": auth_token('anonymous'),
754 "UserProfileNotificationAddress": "arvados@example.com",
757 "BlobSigningKey": "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc",
758 "TrustAllContent": False,
759 "ForwardSlashNameSubstitution": "/",
760 "TrashSweepInterval": "-1s",
763 "Repositories": os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git', 'test'),
767 "GitInternalDir": os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'internal.git'),
769 "SupportedDockerImageFormats": {"v1": {}},
772 "zzzzz-nyw5e-%015d"%n: {
774 "http://%s:%s" % (localhost, keepstore_ports[n]): {},
776 "Driver": "Directory",
777 "DriverParameters": {
778 "Root": os.path.join(TEST_TMPDIR, "keep%d.data"%n),
780 } for n in range(len(keepstore_ports))
786 conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
787 with open(conf, 'w') as f:
788 yaml.safe_dump(config, f)
790 ex = "export ARVADOS_CONFIG="+conf
795 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
797 kill_server_pid(_pidfile('nginx'))
799 def _pidfile(program):
800 return os.path.join(TEST_TMPDIR, program + '.pid')
803 '''load a fixture yaml file'''
804 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
808 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
809 yaml_file = yaml_file[0:trim_index]
812 return yaml.safe_load(yaml_file)
814 def auth_token(token_name):
815 return fixture("api_client_authorizations")[token_name]["api_token"]
817 def authorize_with(token_name):
818 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
819 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
820 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
821 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
823 class TestCaseWithServers(unittest.TestCase):
824 """TestCase to start and stop supporting Arvados servers.
826 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
827 class variables as a dictionary of keyword arguments. If you do,
828 setUpClass will start the corresponding servers by passing these
829 keyword arguments to the run, run_keep, and/or run_keep_server
830 functions, respectively. It will also set Arvados environment
831 variables to point to these servers appropriately. If you don't
832 run a Keep or Keep proxy server, setUpClass will set up a
833 temporary directory for Keep local storage, and set it as
836 tearDownClass will stop any servers started, and restore the
837 original environment.
842 KEEP_PROXY_SERVER = None
843 KEEP_WEB_SERVER = None
846 def _restore_dict(src, dest):
847 for key in list(dest.keys()):
854 cls._orig_environ = os.environ.copy()
855 cls._orig_config = arvados.config.settings().copy()
856 cls._cleanup_funcs = []
857 os.environ.pop('ARVADOS_KEEP_SERVICES', None)
858 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
859 for server_kwargs, start_func, stop_func in (
860 (cls.MAIN_SERVER, run, reset),
861 (cls.WS_SERVER, run_ws, stop_ws),
862 (cls.KEEP_SERVER, run_keep, stop_keep),
863 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
864 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
865 if server_kwargs is not None:
866 start_func(**server_kwargs)
867 cls._cleanup_funcs.append(stop_func)
868 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
869 cls.local_store = tempfile.mkdtemp()
870 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
871 cls._cleanup_funcs.append(
872 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
874 os.environ.pop('KEEP_LOCAL_STORE', None)
875 arvados.config.initialize()
878 def tearDownClass(cls):
879 for clean_func in cls._cleanup_funcs:
881 cls._restore_dict(cls._orig_environ, os.environ)
882 cls._restore_dict(cls._orig_config, arvados.config.settings())
885 if __name__ == "__main__":
888 'start_ws', 'stop_ws',
889 'start_controller', 'stop_controller',
890 'start_keep', 'stop_keep',
891 'start_keep_proxy', 'stop_keep_proxy',
892 'start_keep-web', 'stop_keep-web',
893 'start_arv-git-httpd', 'stop_arv-git-httpd',
894 'start_nginx', 'stop_nginx', 'setup_config',
896 parser = argparse.ArgumentParser()
897 parser.add_argument('action', type=str, help="one of {}".format(actions))
898 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
899 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
900 parser.add_argument('--keep-blob-signing', action="store_true", help="Enable blob signing for keepstore servers")
902 args = parser.parse_args()
904 if args.action not in actions:
905 print("Unrecognized action '{}'. Actions are: {}.".
906 format(args.action, actions),
909 if args.action == 'start':
910 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
911 run(leave_running_atexit=True)
912 host = os.environ['ARVADOS_API_HOST']
913 if args.auth is not None:
914 token = auth_token(args.auth)
915 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
916 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
917 print("export ARVADOS_API_HOST_INSECURE=true")
920 elif args.action == 'stop':
921 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
922 elif args.action == 'start_ws':
924 elif args.action == 'stop_ws':
926 elif args.action == 'start_controller':
928 elif args.action == 'stop_controller':
930 elif args.action == 'start_keep':
931 run_keep(blob_signing=args.keep_blob_signing, num_servers=args.num_keep_servers)
932 elif args.action == 'stop_keep':
933 stop_keep(num_servers=args.num_keep_servers)
934 elif args.action == 'start_keep_proxy':
936 elif args.action == 'stop_keep_proxy':
938 elif args.action == 'start_arv-git-httpd':
940 elif args.action == 'stop_arv-git-httpd':
942 elif args.action == 'start_keep-web':
944 elif args.action == 'stop_keep-web':
946 elif args.action == 'start_nginx':
948 print("export ARVADOS_API_HOST=0.0.0.0:{}".format(external_port_from_config('Controller')))
949 elif args.action == 'stop_nginx':
951 elif args.action == 'setup_config':
954 raise Exception("action recognized but not implemented!?")