1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import print_function
6 from __future__ import division
7 from builtins import str
8 from builtins import range
30 from urllib.parse import urlparse
32 from urlparse import urlparse
34 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
35 if __name__ == '__main__' and os.path.exists(
36 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
37 # We're being launched to support another test suite.
38 # Add the Python SDK source to the library path.
39 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
44 # This module starts subprocesses and records them in pidfiles so they
45 # can be managed by other processes (incl. after this process
46 # exits). But if we don't keep a reference to each subprocess object
47 # somewhere, the subprocess destructor runs, and we get a lot of
48 # ResourceWarning noise in test logs. This is our bucket of subprocess
49 # objects whose destructors we don't want to run but are otherwise
51 _detachedSubprocesses = []
53 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
54 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
56 # Work around https://bugs.python.org/issue27805, should be no longer
57 # necessary from sometime in Python 3.8.x
58 if not os.environ.get('ARVADOS_DEBUG', ''):
63 if 'GOPATH' in os.environ:
64 # Add all GOPATH bin dirs to PATH -- but insert them after the
65 # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
66 # command, not the golang.org/x/tools/cmd/bundle command.
67 gopaths = os.environ['GOPATH'].split(':')
68 addbins = [os.path.join(path, 'bin') for path in gopaths]
70 for path in os.environ['PATH'].split(':'):
72 if os.path.exists(os.path.join(path, 'bundle')):
76 os.environ['PATH'] = ':'.join(newbins)
78 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
79 if not os.path.exists(TEST_TMPDIR):
84 _cached_db_config = {}
85 _already_used_port = {}
87 def find_server_pid(PID_PATH, wait=10):
91 while (not good_pid) and (now <= timeout):
94 with open(PID_PATH, 'r') as f:
95 server_pid = int(f.read())
96 good_pid = (os.kill(server_pid, 0) is None)
97 except EnvironmentError:
106 def kill_server_pid(pidfile, wait=10, passenger_root=False):
107 # Must re-import modules in order to work during atexit
115 deadline = now + wait
118 # First try to shut down nicely
119 restore_cwd = os.getcwd()
120 os.chdir(passenger_root)
122 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
123 os.chdir(restore_cwd)
124 # Use up to half of the +wait+ period waiting for "passenger
125 # stop" to work. If the process hasn't exited by then, start
126 # sending TERM signals.
130 while now <= deadline and server_pid is None:
132 with open(pidfile, 'r') as f:
133 server_pid = int(f.read())
135 # No pidfile = nothing to kill.
137 except ValueError as error:
138 # Pidfile exists, but we can't parse it. Perhaps the
139 # server has created the file but hasn't written its PID
141 print("Parse error reading pidfile {}: {}".format(pidfile, error),
146 while now <= deadline:
148 exited, _ = os.waitpid(server_pid, os.WNOHANG)
150 _remove_pidfile(pidfile)
153 # already exited, or isn't our child process
157 os.kill(server_pid, signal.SIGTERM)
158 print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
160 except OSError as error:
161 if error.errno == errno.ESRCH:
162 # Thrown by os.getpgid() or os.kill() if the process
163 # does not exist, i.e., our work here is done.
164 _remove_pidfile(pidfile)
170 print("Server PID {} ({}) did not exit, giving up after {}s".
171 format(server_pid, pidfile, wait),
174 def _remove_pidfile(pidfile):
178 if os.path.lexists(pidfile):
181 def find_available_port():
182 """Return an IPv4 port number that is not in use right now.
184 We assume whoever needs to use the returned port is able to reuse
185 a recently used port without waiting for TIME_WAIT (see
186 SO_REUSEADDR / SO_REUSEPORT).
188 Some opportunity for races here, but it's better than choosing
189 something at random and not checking at all. If all of our servers
190 (hey Passenger) knew that listening on port 0 was a thing, the OS
191 would take care of the races, and this wouldn't be needed at all.
194 global _already_used_port
196 sock = socket.socket()
197 sock.bind(('0.0.0.0', 0))
198 port = sock.getsockname()[1]
200 if port not in _already_used_port:
201 _already_used_port[port] = True
204 def _wait_until_port_listens(port, timeout=10, warn=True):
205 """Wait for a process to start listening on the given port.
207 If nothing listens on the port within the specified timeout (given
208 in seconds), print a warning on stderr before returning.
211 subprocess.check_output(['which', 'netstat'])
212 except subprocess.CalledProcessError:
213 print("WARNING: No `netstat` -- cannot wait for port to listen. "+
214 "Sleeping 0.5 and hoping for the best.",
218 deadline = time.time() + timeout
219 while time.time() < deadline:
220 if re.search(r'\ntcp.*:'+str(port)+' .* LISTEN *\n', subprocess.check_output(['netstat', '-Wln']).decode()):
225 "WARNING: Nothing is listening on port {} (waited {} seconds).".
226 format(port, timeout),
230 def _logfilename(label):
231 """Set up a labelled log file, and return a path to write logs to.
233 Normally, the returned path is {tmpdir}/{label}.log.
235 In debug mode, logs are also written to stderr, with [label]
236 prepended to each line. The returned path is a FIFO.
238 +label+ should contain only alphanumerics: it is also used as part
239 of the FIFO filename.
242 logfilename = os.path.join(TEST_TMPDIR, label+'.log')
243 if not os.environ.get('ARVADOS_DEBUG', ''):
245 fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
248 except OSError as error:
249 if error.errno != errno.ENOENT:
251 os.mkfifo(fifo, 0o700)
252 stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
253 # open(fifo, 'r') would block waiting for someone to open the fifo
254 # for writing, so we need a separate cat process to open it for
256 cat = subprocess.Popen(
257 stdbuf+['cat', fifo],
258 stdin=open('/dev/null'),
259 stdout=subprocess.PIPE)
260 _detachedSubprocesses.append(cat)
261 tee = subprocess.Popen(
262 stdbuf+['tee', '-a', logfilename],
264 stdout=subprocess.PIPE)
265 _detachedSubprocesses.append(tee)
266 sed = subprocess.Popen(
267 stdbuf+['sed', '-e', 's/^/['+label+'] /'],
270 _detachedSubprocesses.append(sed)
273 def run(leave_running_atexit=False):
274 """Ensure an API server is running, and ARVADOS_API_* env vars have
275 admin credentials for it.
277 If ARVADOS_TEST_API_HOST is set, a parent process has started a
278 test server for us to use: we just need to reset() it using the
281 If a previous call to run() started a new server process, and it
282 is still running, we just need to reset() it to fixture state and
285 If neither of those options work out, we'll really start a new
290 # Delete cached discovery documents.
292 # This will clear cached docs that belong to other processes (like
293 # concurrent test suites) even if they're still running. They should
294 # be able to tolerate that.
295 for fn in glob.glob(os.path.join(
296 str(arvados.http_cache('discovery')),
297 '*,arvados,v1,rest,*')):
300 pid_file = _pidfile('api')
301 pid_file_ok = find_server_pid(pid_file, 0)
303 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
304 if existing_api_host and pid_file_ok:
305 if existing_api_host == my_api_host:
309 # Fall through to shutdown-and-start case.
312 # Server was provided by parent. Can't recover if it's
316 # Before trying to start up our own server, call stop() to avoid
317 # "Phusion Passenger Standalone is already running on PID 12345".
318 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
319 # we know the server is ours to kill.)
322 restore_cwd = os.getcwd()
323 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
324 os.chdir(api_src_dir)
326 # Either we haven't started a server of our own yet, or it has
327 # died, or we have lost our credentials, or something else is
328 # preventing us from calling reset(). Start a new one.
330 if not os.path.exists('tmp'):
333 if not os.path.exists('tmp/api'):
334 os.makedirs('tmp/api')
336 if not os.path.exists('tmp/logs'):
337 os.makedirs('tmp/logs')
339 # Install the git repository fixtures.
340 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
341 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
342 if not os.path.isdir(gitdir):
344 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
346 # Customizing the passenger config template is the only documented
347 # way to override the default passenger_stat_throttle_rate (10 s).
348 # In the testing environment, we want restart.txt to take effect
350 resdir = subprocess.check_output(['bundle', 'exec', 'passenger-config', 'about', 'resourcesdir']).decode().rstrip()
351 with open(resdir + '/templates/standalone/config.erb') as f:
353 newtemplate = re.sub(r'http \{', 'http {\n passenger_stat_throttle_rate 0;', template)
354 if newtemplate == template:
355 raise "template edit failed"
356 with open('tmp/passenger-nginx.conf.erb', 'w') as f:
359 port = internal_port_from_config("RailsAPI")
360 env = os.environ.copy()
361 env['RAILS_ENV'] = 'test'
362 env['ARVADOS_RAILS_LOG_TO_STDOUT'] = '1'
363 env.pop('ARVADOS_WEBSOCKETS', None)
364 env.pop('ARVADOS_TEST_API_HOST', None)
365 env.pop('ARVADOS_API_HOST', None)
366 env.pop('ARVADOS_API_HOST_INSECURE', None)
367 env.pop('ARVADOS_API_TOKEN', None)
368 logf = open(_logfilename('railsapi'), WRITE_MODE)
369 railsapi = subprocess.Popen(
371 'passenger', 'start', '-p{}'.format(port),
372 '--nginx-config-template', 'tmp/passenger-nginx.conf.erb',
373 '--no-friendly-error-pages',
374 '--disable-anonymous-telemetry',
375 '--disable-security-update-check',
376 '--pid-file', pid_file,
377 '--log-file', '/dev/stdout',
379 '--ssl-certificate', 'tmp/self-signed.pem',
380 '--ssl-certificate-key', 'tmp/self-signed.key'],
381 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
382 _detachedSubprocesses.append(railsapi)
384 if not leave_running_atexit:
385 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
387 my_api_host = "127.0.0.1:"+str(port)
388 os.environ['ARVADOS_API_HOST'] = my_api_host
390 # Make sure the server has written its pid file and started
391 # listening on its TCP port
392 _wait_until_port_listens(port)
393 find_server_pid(pid_file)
396 os.chdir(restore_cwd)
399 """Reset the test server to fixture state.
401 This resets the ARVADOS_TEST_API_HOST provided by a parent process
402 if any, otherwise the server started by run().
404 It also resets ARVADOS_* environment vars to point to the test
405 server with admin credentials.
407 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
408 token = auth_token('admin')
409 httpclient = httplib2.Http(ca_certs=os.path.join(
410 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
412 'https://{}/database/reset'.format(existing_api_host),
414 headers={'Authorization': 'OAuth2 {}'.format(token), 'Connection':'close'})
416 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
417 os.environ['ARVADOS_API_TOKEN'] = token
418 os.environ['ARVADOS_API_HOST'] = existing_api_host
420 def stop(force=False):
421 """Stop the API server, if one is running.
423 If force==False, kill it only if we started it ourselves. (This
424 supports the use case where a Python test suite calls run(), but
425 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
426 process, and the test suite cleans up after itself by calling
427 stop(). In this case the test server provided by the parent
428 process should be left alone.)
430 If force==True, kill it even if we didn't start it
431 ourselves. (This supports the use case in __main__, where "run"
432 and "stop" happen in different processes.)
435 if force or my_api_host is not None:
436 kill_server_pid(_pidfile('api'))
440 with open(os.environ["ARVADOS_CONFIG"]) as f:
441 return yaml.safe_load(f)
443 def internal_port_from_config(service, idx=0):
445 sorted(list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys()))[idx]).
446 netloc.split(":")[1])
448 def external_port_from_config(service):
449 return int(urlparse(get_config()["Clusters"]["zzzzz"]["Services"][service]["ExternalURL"]).netloc.split(":")[1])
451 def run_controller():
452 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
455 logf = open(_logfilename('controller'), WRITE_MODE)
456 port = internal_port_from_config("Controller")
457 controller = subprocess.Popen(
458 ["arvados-server", "controller"],
459 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
460 _detachedSubprocesses.append(controller)
461 with open(_pidfile('controller'), 'w') as f:
462 f.write(str(controller.pid))
463 _wait_until_port_listens(port)
466 def stop_controller():
467 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
469 kill_server_pid(_pidfile('controller'))
472 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
475 port = internal_port_from_config("Websocket")
476 logf = open(_logfilename('ws'), WRITE_MODE)
477 ws = subprocess.Popen(
478 ["arvados-server", "ws"],
479 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
480 _detachedSubprocesses.append(ws)
481 with open(_pidfile('ws'), 'w') as f:
483 _wait_until_port_listens(port)
487 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
489 kill_server_pid(_pidfile('ws'))
491 def _start_keep(n, blob_signing=False):
492 datadir = os.path.join(TEST_TMPDIR, "keep%d.data"%n)
493 if os.path.exists(datadir):
494 shutil.rmtree(datadir)
496 port = internal_port_from_config("Keepstore", idx=n)
498 # Currently, if there are multiple InternalURLs for a single host,
499 # the only way to tell a keepstore process which one it's supposed
500 # to listen on is to supply a redacted version of the config, with
501 # the other InternalURLs removed.
502 conf = os.path.join(TEST_TMPDIR, "keep%d.yaml"%n)
503 confdata = get_config()
504 confdata['Clusters']['zzzzz']['Services']['Keepstore']['InternalURLs'] = {"http://127.0.0.1:%d"%port: {}}
505 confdata['Clusters']['zzzzz']['Collections']['BlobSigning'] = blob_signing
506 with open(conf, 'w') as f:
507 yaml.safe_dump(confdata, f)
508 keep_cmd = ["arvados-server", "keepstore", "-config", conf]
510 with open(_logfilename('keep{}'.format(n)), WRITE_MODE) as logf:
511 with open('/dev/null') as _stdin:
512 child = subprocess.Popen(
513 keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
514 _detachedSubprocesses.append(child)
516 print('child.pid is %d'%child.pid, file=sys.stderr)
517 with open(_pidfile('keep{}'.format(n)), 'w') as f:
518 f.write(str(child.pid))
520 _wait_until_port_listens(port)
524 def run_keep(num_servers=2, **kwargs):
525 stop_keep(num_servers)
529 host=os.environ['ARVADOS_API_HOST'],
530 token=os.environ['ARVADOS_API_TOKEN'],
533 for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']:
534 api.keep_services().delete(uuid=d['uuid']).execute()
535 for d in api.keep_disks().list().execute()['items']:
536 api.keep_disks().delete(uuid=d['uuid']).execute()
538 for d in range(0, num_servers):
539 port = _start_keep(d, **kwargs)
540 svc = api.keep_services().create(body={'keep_service': {
541 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
542 'service_host': 'localhost',
543 'service_port': port,
544 'service_type': 'disk',
545 'service_ssl_flag': False,
547 api.keep_disks().create(body={
548 'keep_disk': {'keep_service_uuid': svc['uuid'] }
551 # If keepproxy and/or keep-web is running, send SIGHUP to make
552 # them discover the new keepstore services.
553 for svc in ('keepproxy', 'keep-web'):
554 pidfile = _pidfile(svc)
555 if os.path.exists(pidfile):
557 with open(pidfile) as pid:
558 os.kill(int(pid.read()), signal.SIGHUP)
563 kill_server_pid(_pidfile('keep{}'.format(n)))
565 def stop_keep(num_servers=2):
566 for n in range(0, num_servers):
569 def run_keep_proxy():
570 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
571 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(internal_port_from_config('Keepproxy'))
575 port = internal_port_from_config("Keepproxy")
576 env = os.environ.copy()
577 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
578 logf = open(_logfilename('keepproxy'), WRITE_MODE)
579 kp = subprocess.Popen(
580 ['arvados-server', 'keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
581 _detachedSubprocesses.append(kp)
583 with open(_pidfile('keepproxy'), 'w') as f:
585 _wait_until_port_listens(port)
587 print("Using API %s token %s" % (os.environ['ARVADOS_API_HOST'], auth_token('admin')), file=sys.stdout)
590 host=os.environ['ARVADOS_API_HOST'],
591 token=auth_token('admin'),
593 for d in api.keep_services().list(
594 filters=[['service_type','=','proxy']]).execute()['items']:
595 api.keep_services().delete(uuid=d['uuid']).execute()
596 api.keep_services().create(body={'keep_service': {
597 'service_host': 'localhost',
598 'service_port': port,
599 'service_type': 'proxy',
600 'service_ssl_flag': False,
602 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
603 _wait_until_port_listens(port)
605 def stop_keep_proxy():
606 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
608 kill_server_pid(_pidfile('keepproxy'))
610 def run_arv_git_httpd():
611 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
615 gitport = internal_port_from_config("GitHTTP")
616 env = os.environ.copy()
617 env.pop('ARVADOS_API_TOKEN', None)
618 logf = open(_logfilename('githttpd'), WRITE_MODE)
619 agh = subprocess.Popen(['arvados-server', 'git-httpd'],
620 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
621 _detachedSubprocesses.append(agh)
622 with open(_pidfile('githttpd'), 'w') as f:
623 f.write(str(agh.pid))
624 _wait_until_port_listens(gitport)
626 def stop_arv_git_httpd():
627 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
629 kill_server_pid(_pidfile('githttpd'))
632 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
636 keepwebport = internal_port_from_config("WebDAV")
637 env = os.environ.copy()
638 logf = open(_logfilename('keep-web'), WRITE_MODE)
639 keepweb = subprocess.Popen(
640 ['arvados-server', 'keep-web'],
641 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
642 _detachedSubprocesses.append(keepweb)
643 with open(_pidfile('keep-web'), 'w') as f:
644 f.write(str(keepweb.pid))
645 _wait_until_port_listens(keepwebport)
648 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
650 kill_server_pid(_pidfile('keep-web'))
653 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
657 nginxconf['UPSTREAMHOST'] = '127.0.0.1'
658 nginxconf['LISTENHOST'] = '127.0.0.1'
659 nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
660 nginxconf['ARVADOS_API_HOST'] = "0.0.0.0:" + str(external_port_from_config("Controller"))
661 nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")
662 nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV")
663 nginxconf['KEEPWEBDLSSLPORT'] = external_port_from_config("WebDAVDownload")
664 nginxconf['KEEPWEBSSLPORT'] = external_port_from_config("WebDAV")
665 nginxconf['KEEPPROXYPORT'] = internal_port_from_config("Keepproxy")
666 nginxconf['KEEPPROXYSSLPORT'] = external_port_from_config("Keepproxy")
667 nginxconf['GITPORT'] = internal_port_from_config("GitHTTP")
668 nginxconf['GITSSLPORT'] = external_port_from_config("GitHTTP")
669 nginxconf['HEALTHPORT'] = internal_port_from_config("Health")
670 nginxconf['HEALTHSSLPORT'] = external_port_from_config("Health")
671 nginxconf['WSPORT'] = internal_port_from_config("Websocket")
672 nginxconf['WSSSLPORT'] = external_port_from_config("Websocket")
673 nginxconf['WORKBENCH1PORT'] = internal_port_from_config("Workbench1")
674 nginxconf['WORKBENCH1SSLPORT'] = external_port_from_config("Workbench1")
675 nginxconf['WORKBENCH2PORT'] = internal_port_from_config("Workbench2")
676 nginxconf['WORKBENCH2SSLPORT'] = external_port_from_config("Workbench2")
677 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
678 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
679 nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
680 nginxconf['ERRORLOG'] = _logfilename('nginx_error')
681 nginxconf['TMPDIR'] = TEST_TMPDIR + '/nginx'
682 nginxconf['INTERNALSUBNETS'] = '169.254.0.0/16 0;'
684 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
685 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
686 with open(conffile, 'w') as f:
688 r'{{([A-Z]+[A-Z0-9]+)}}',
689 lambda match: str(nginxconf.get(match.group(1))),
690 open(conftemplatefile).read()))
692 env = os.environ.copy()
693 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
695 nginx = subprocess.Popen(
697 '-g', 'error_log stderr info; pid '+_pidfile('nginx')+';',
699 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
700 _detachedSubprocesses.append(nginx)
701 _wait_until_port_listens(nginxconf['CONTROLLERSSLPORT'])
704 rails_api_port = find_available_port()
705 controller_port = find_available_port()
706 controller_external_port = find_available_port()
707 websocket_port = find_available_port()
708 websocket_external_port = find_available_port()
709 workbench1_port = find_available_port()
710 workbench1_external_port = find_available_port()
711 workbench2_port = find_available_port()
712 workbench2_external_port = find_available_port()
713 git_httpd_port = find_available_port()
714 git_httpd_external_port = find_available_port()
715 health_httpd_port = find_available_port()
716 health_httpd_external_port = find_available_port()
717 keepproxy_port = find_available_port()
718 keepproxy_external_port = find_available_port()
719 keepstore_ports = sorted([str(find_available_port()) for _ in range(0,4)])
720 keep_web_port = find_available_port()
721 keep_web_external_port = find_available_port()
722 keep_web_dl_external_port = find_available_port()
724 configsrc = os.environ.get("CONFIGSRC", None)
726 clusterconf = os.path.join(configsrc, "config.yml")
727 print("Getting config from %s" % clusterconf, file=sys.stderr)
728 pgconnection = yaml.safe_load(open(clusterconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
730 # assume "arvados-server install -type test" has set up the
731 # conventional db credentials
733 "client_encoding": "utf8",
735 "dbname": "arvados_test",
737 "password": "insecure_arvados_test",
740 localhost = "127.0.0.1"
744 "https://%s:%s"%(localhost, rails_api_port): {},
748 "ExternalURL": "https://%s:%s" % (localhost, controller_external_port),
750 "http://%s:%s"%(localhost, controller_port): {},
754 "ExternalURL": "wss://%s:%s/websocket" % (localhost, websocket_external_port),
756 "http://%s:%s"%(localhost, websocket_port): {},
760 "ExternalURL": "https://%s:%s/" % (localhost, workbench1_external_port),
762 "http://%s:%s"%(localhost, workbench1_port): {},
766 "ExternalURL": "https://%s:%s/" % (localhost, workbench2_external_port),
768 "http://%s:%s"%(localhost, workbench2_port): {},
772 "ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port),
774 "http://%s:%s"%(localhost, git_httpd_port): {}
778 "ExternalURL": "https://%s:%s" % (localhost, health_httpd_external_port),
780 "http://%s:%s"%(localhost, health_httpd_port): {}
785 "http://%s:%s"%(localhost, port): {} for port in keepstore_ports
789 "ExternalURL": "https://%s:%s" % (localhost, keepproxy_external_port),
791 "http://%s:%s"%(localhost, keepproxy_port): {},
795 "ExternalURL": "https://%s:%s" % (localhost, keep_web_external_port),
797 "http://%s:%s"%(localhost, keep_web_port): {},
801 "ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port),
803 "http://%s:%s"%(localhost, keep_web_port): {},
811 "ManagementToken": "e687950a23c3a9bceec28c6223a06c79",
812 "SystemRootToken": auth_token('system_user'),
814 "RequestTimeout": "30s",
815 "LockBeforeUpdate": True,
822 "Email": "alice@example.com",
829 "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
832 "Connection": pgconnection,
837 "Services": services,
839 "AnonymousUserToken": auth_token('anonymous'),
840 "UserProfileNotificationAddress": "arvados@example.com",
843 "CollectionVersioning": True,
844 "BlobSigningKey": "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc",
845 "TrustAllContent": False,
846 "ForwardSlashNameSubstitution": "/",
847 "TrashSweepInterval": "-1s",
850 "Repositories": os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git', 'test'),
854 "GitInternalDir": os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'internal.git'),
856 "LocalKeepBlobBuffersPerVCPU": 0,
858 "SweepInterval": 0, # disable, otherwise test cases can't acquire dblock
860 "SupportedDockerImageFormats": {"v1": {}},
867 "zzzzz-nyw5e-%015d"%n: {
869 "http://%s:%s" % (localhost, keepstore_ports[n]): {},
871 "Driver": "Directory",
872 "DriverParameters": {
873 "Root": os.path.join(TEST_TMPDIR, "keep%d.data"%n),
875 } for n in range(len(keepstore_ports))
881 conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
882 with open(conf, 'w') as f:
883 yaml.safe_dump(config, f)
885 ex = "export ARVADOS_CONFIG="+conf
890 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
892 kill_server_pid(_pidfile('nginx'))
894 def _pidfile(program):
895 return os.path.join(TEST_TMPDIR, program + '.pid')
898 '''load a fixture yaml file'''
899 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
903 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
904 yaml_file = yaml_file[0:trim_index]
907 return yaml.safe_load(yaml_file)
909 def auth_token(token_name):
910 return fixture("api_client_authorizations")[token_name]["api_token"]
912 def authorize_with(token_name):
913 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
914 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
915 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
916 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
918 class TestCaseWithServers(unittest.TestCase):
919 """TestCase to start and stop supporting Arvados servers.
921 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
922 class variables as a dictionary of keyword arguments. If you do,
923 setUpClass will start the corresponding servers by passing these
924 keyword arguments to the run, run_keep, and/or run_keep_server
925 functions, respectively. It will also set Arvados environment
926 variables to point to these servers appropriately. If you don't
927 run a Keep or Keep proxy server, setUpClass will set up a
928 temporary directory for Keep local storage, and set it as
931 tearDownClass will stop any servers started, and restore the
932 original environment.
937 KEEP_PROXY_SERVER = None
938 KEEP_WEB_SERVER = None
941 def _restore_dict(src, dest):
942 for key in list(dest.keys()):
949 cls._orig_environ = os.environ.copy()
950 cls._orig_config = arvados.config.settings().copy()
951 cls._cleanup_funcs = []
952 os.environ.pop('ARVADOS_KEEP_SERVICES', None)
953 for server_kwargs, start_func, stop_func in (
954 (cls.MAIN_SERVER, run, reset),
955 (cls.WS_SERVER, run_ws, stop_ws),
956 (cls.KEEP_SERVER, run_keep, stop_keep),
957 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
958 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
959 if server_kwargs is not None:
960 start_func(**server_kwargs)
961 cls._cleanup_funcs.append(stop_func)
962 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
963 cls.local_store = tempfile.mkdtemp()
964 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
965 cls._cleanup_funcs.append(
966 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
968 os.environ.pop('KEEP_LOCAL_STORE', None)
969 arvados.config.initialize()
972 def tearDownClass(cls):
973 for clean_func in cls._cleanup_funcs:
975 cls._restore_dict(cls._orig_environ, os.environ)
976 cls._restore_dict(cls._orig_config, arvados.config.settings())
979 if __name__ == "__main__":
982 'start_ws', 'stop_ws',
983 'start_controller', 'stop_controller',
984 'start_keep', 'stop_keep',
985 'start_keep_proxy', 'stop_keep_proxy',
986 'start_keep-web', 'stop_keep-web',
987 'start_githttpd', 'stop_githttpd',
988 'start_nginx', 'stop_nginx', 'setup_config',
990 parser = argparse.ArgumentParser()
991 parser.add_argument('action', type=str, help="one of {}".format(actions))
992 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
993 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
994 parser.add_argument('--keep-blob-signing', action="store_true", help="Enable blob signing for keepstore servers")
996 args = parser.parse_args()
998 if args.action not in actions:
999 print("Unrecognized action '{}'. Actions are: {}.".
1000 format(args.action, actions),
1003 # Create a new process group so our child processes don't exit on
1004 # ^C in run-tests.sh interactive mode.
1006 if args.action == 'start':
1007 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
1008 run(leave_running_atexit=True)
1009 host = os.environ['ARVADOS_API_HOST']
1010 if args.auth is not None:
1011 token = auth_token(args.auth)
1012 print("export ARVADOS_API_TOKEN={}".format(shlex.quote(token)))
1013 print("export ARVADOS_API_HOST={}".format(shlex.quote(host)))
1014 print("export ARVADOS_API_HOST_INSECURE=true")
1017 elif args.action == 'stop':
1018 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
1019 elif args.action == 'start_ws':
1021 elif args.action == 'stop_ws':
1023 elif args.action == 'start_controller':
1025 elif args.action == 'stop_controller':
1027 elif args.action == 'start_keep':
1028 run_keep(blob_signing=args.keep_blob_signing, num_servers=args.num_keep_servers)
1029 elif args.action == 'stop_keep':
1030 stop_keep(num_servers=args.num_keep_servers)
1031 elif args.action == 'start_keep_proxy':
1033 elif args.action == 'stop_keep_proxy':
1035 elif args.action == 'start_githttpd':
1037 elif args.action == 'stop_githttpd':
1038 stop_arv_git_httpd()
1039 elif args.action == 'start_keep-web':
1041 elif args.action == 'stop_keep-web':
1043 elif args.action == 'start_nginx':
1045 print("export ARVADOS_API_HOST=0.0.0.0:{}".format(external_port_from_config('Controller')))
1046 elif args.action == 'stop_nginx':
1048 elif args.action == 'setup_config':
1051 raise Exception("action recognized but not implemented!?")