3 from __future__ import print_function
24 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
25 if __name__ == '__main__' and os.path.exists(
26 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
27 # We're being launched to support another test suite.
28 # Add the Python SDK source to the library path.
29 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
34 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
35 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
36 if 'GOPATH' in os.environ:
37 gopaths = os.environ['GOPATH'].split(':')
38 gobins = [os.path.join(path, 'bin') for path in gopaths]
39 os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
41 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
42 if not os.path.exists(TEST_TMPDIR):
48 def find_server_pid(PID_PATH, wait=10):
52 while (not good_pid) and (now <= timeout):
55 with open(PID_PATH, 'r') as f:
56 server_pid = int(f.read())
57 good_pid = (os.kill(server_pid, 0) is None)
58 except EnvironmentError:
67 def kill_server_pid(pidfile, wait=10, passenger_root=False):
68 # Must re-import modules in order to work during atexit
79 # First try to shut down nicely
80 restore_cwd = os.getcwd()
81 os.chdir(passenger_root)
83 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
85 # Use up to half of the +wait+ period waiting for "passenger
86 # stop" to work. If the process hasn't exited by then, start
87 # sending TERM signals.
91 while now <= deadline and server_pid is None:
93 with open(pidfile, 'r') as f:
94 server_pid = int(f.read())
96 # No pidfile = nothing to kill.
98 except ValueError as error:
99 # Pidfile exists, but we can't parse it. Perhaps the
100 # server has created the file but hasn't written its PID
102 print("Parse error reading pidfile {}: {}".format(pidfile, error),
107 while now <= deadline:
109 exited, _ = os.waitpid(server_pid, os.WNOHANG)
113 # already exited, or isn't our child process
117 os.kill(server_pid, signal.SIGTERM)
118 print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
120 except OSError as error:
121 if error.errno == errno.ESRCH:
122 # Thrown by os.getpgid() or os.kill() if the process
123 # does not exist, i.e., our work here is done.
129 print("Server PID {} ({}) did not exit, giving up after {}s".
130 format(server_pid, pidfile, wait),
133 def find_available_port():
134 """Return an IPv4 port number that is not in use right now.
136 We assume whoever needs to use the returned port is able to reuse
137 a recently used port without waiting for TIME_WAIT (see
138 SO_REUSEADDR / SO_REUSEPORT).
140 Some opportunity for races here, but it's better than choosing
141 something at random and not checking at all. If all of our servers
142 (hey Passenger) knew that listening on port 0 was a thing, the OS
143 would take care of the races, and this wouldn't be needed at all.
146 sock = socket.socket()
147 sock.bind(('0.0.0.0', 0))
148 port = sock.getsockname()[1]
152 def _wait_until_port_listens(port, timeout=10):
153 """Wait for a process to start listening on the given port.
155 If nothing listens on the port within the specified timeout (given
156 in seconds), print a warning on stderr before returning.
159 subprocess.check_output(['which', 'lsof'])
160 except subprocess.CalledProcessError:
161 print("WARNING: No `lsof` -- cannot wait for port to listen. "+
162 "Sleeping 0.5 and hoping for the best.",
166 deadline = time.time() + timeout
167 while time.time() < deadline:
169 subprocess.check_output(
170 ['lsof', '-t', '-i', 'tcp:'+str(port)])
171 except subprocess.CalledProcessError:
176 "WARNING: Nothing is listening on port {} (waited {} seconds).".
177 format(port, timeout),
180 def _fifo2stderr(label):
181 """Create a fifo, and copy it to stderr, prepending label to each line.
183 Return value is the path to the new FIFO.
185 +label+ should contain only alphanumerics: it is also used as part
186 of the FIFO filename.
188 fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
191 except OSError as error:
192 if error.errno != errno.ENOENT:
194 os.mkfifo(fifo, 0700)
196 ['stdbuf', '-i0', '-oL', '-eL', 'sed', '-e', 's/^/['+label+'] /', fifo],
200 def run(leave_running_atexit=False):
201 """Ensure an API server is running, and ARVADOS_API_* env vars have
202 admin credentials for it.
204 If ARVADOS_TEST_API_HOST is set, a parent process has started a
205 test server for us to use: we just need to reset() it using the
208 If a previous call to run() started a new server process, and it
209 is still running, we just need to reset() it to fixture state and
212 If neither of those options work out, we'll really start a new
217 # Delete cached discovery documents.
219 # This will clear cached docs that belong to other processes (like
220 # concurrent test suites) even if they're still running. They should
221 # be able to tolerate that.
222 for fn in glob.glob(os.path.join(arvados.http_cache('discovery'),
223 '*,arvados,v1,rest,*')):
226 pid_file = _pidfile('api')
227 pid_file_ok = find_server_pid(pid_file, 0)
229 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
230 if existing_api_host and pid_file_ok:
231 if existing_api_host == my_api_host:
235 # Fall through to shutdown-and-start case.
238 # Server was provided by parent. Can't recover if it's
242 # Before trying to start up our own server, call stop() to avoid
243 # "Phusion Passenger Standalone is already running on PID 12345".
244 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
245 # we know the server is ours to kill.)
248 restore_cwd = os.getcwd()
249 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
250 os.chdir(api_src_dir)
252 # Either we haven't started a server of our own yet, or it has
253 # died, or we have lost our credentials, or something else is
254 # preventing us from calling reset(). Start a new one.
256 if not os.path.exists('tmp'):
259 if not os.path.exists('tmp/api'):
260 os.makedirs('tmp/api')
262 if not os.path.exists('tmp/logs'):
263 os.makedirs('tmp/logs')
265 if not os.path.exists('tmp/self-signed.pem'):
266 # We assume here that either passenger reports its listening
267 # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
268 # then the certificate won't match the host and reset() will
269 # fail certificate verification. If it reports "localhost",
270 # clients (notably Python SDK's websocket client) might
271 # resolve localhost as ::1 and then fail to connect.
272 subprocess.check_call([
273 'openssl', 'req', '-new', '-x509', '-nodes',
274 '-out', 'tmp/self-signed.pem',
275 '-keyout', 'tmp/self-signed.key',
277 '-subj', '/CN=0.0.0.0'],
280 # Install the git repository fixtures.
281 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
282 gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
283 if not os.path.isdir(gitdir):
285 subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
287 port = find_available_port()
288 env = os.environ.copy()
289 env['RAILS_ENV'] = 'test'
290 env['ARVADOS_WEBSOCKETS'] = 'yes'
291 env.pop('ARVADOS_TEST_API_HOST', None)
292 env.pop('ARVADOS_API_HOST', None)
293 env.pop('ARVADOS_API_HOST_INSECURE', None)
294 env.pop('ARVADOS_API_TOKEN', None)
295 start_msg = subprocess.check_output(
297 'passenger', 'start', '-d', '-p{}'.format(port),
298 '--pid-file', pid_file,
299 '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
301 '--ssl-certificate', 'tmp/self-signed.pem',
302 '--ssl-certificate-key', 'tmp/self-signed.key'],
305 if not leave_running_atexit:
306 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
308 match = re.search(r'Accessible via: https://(.*?)/', start_msg)
311 "Passenger did not report endpoint: {}".format(start_msg))
312 my_api_host = match.group(1)
313 os.environ['ARVADOS_API_HOST'] = my_api_host
315 # Make sure the server has written its pid file and started
316 # listening on its TCP port
317 find_server_pid(pid_file)
318 _wait_until_port_listens(port)
321 os.chdir(restore_cwd)
324 """Reset the test server to fixture state.
326 This resets the ARVADOS_TEST_API_HOST provided by a parent process
327 if any, otherwise the server started by run().
329 It also resets ARVADOS_* environment vars to point to the test
330 server with admin credentials.
332 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
333 token = auth_token('admin')
334 httpclient = httplib2.Http(ca_certs=os.path.join(
335 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
337 'https://{}/database/reset'.format(existing_api_host),
339 headers={'Authorization': 'OAuth2 {}'.format(token)})
340 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
341 os.environ['ARVADOS_API_HOST'] = existing_api_host
342 os.environ['ARVADOS_API_TOKEN'] = token
344 def stop(force=False):
345 """Stop the API server, if one is running.
347 If force==False, kill it only if we started it ourselves. (This
348 supports the use case where a Python test suite calls run(), but
349 run() just uses the ARVADOS_TEST_API_HOST provided by the parent
350 process, and the test suite cleans up after itself by calling
351 stop(). In this case the test server provided by the parent
352 process should be left alone.)
354 If force==True, kill it even if we didn't start it
355 ourselves. (This supports the use case in __main__, where "run"
356 and "stop" happen in different processes.)
359 if force or my_api_host is not None:
360 kill_server_pid(_pidfile('api'))
363 def _start_keep(n, keep_args):
364 keep0 = tempfile.mkdtemp()
365 port = find_available_port()
366 keep_cmd = ["keepstore",
367 "-volume={}".format(keep0),
368 "-listen=:{}".format(port),
369 "-pid="+_pidfile('keep{}'.format(n))]
371 for arg, val in keep_args.iteritems():
372 keep_cmd.append("{}={}".format(arg, val))
374 logf = open(_fifo2stderr('keep{}'.format(n)), 'w')
375 kp0 = subprocess.Popen(
376 keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
378 with open(_pidfile('keep{}'.format(n)), 'w') as f:
379 f.write(str(kp0.pid))
381 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
384 _wait_until_port_listens(port)
388 def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
389 stop_keep(num_servers)
392 if not blob_signing_key:
393 blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
394 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
395 keep_args['-blob-signing-key-file'] = f.name
396 f.write(blob_signing_key)
397 keep_args['-enforce-permissions'] = str(enforce_permissions).lower()
398 with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
399 keep_args['-data-manager-token-file'] = f.name
400 f.write(auth_token('data_manager'))
401 keep_args['-never-delete'] = 'false'
405 host=os.environ['ARVADOS_API_HOST'],
406 token=os.environ['ARVADOS_API_TOKEN'],
409 for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']:
410 api.keep_services().delete(uuid=d['uuid']).execute()
411 for d in api.keep_disks().list().execute()['items']:
412 api.keep_disks().delete(uuid=d['uuid']).execute()
414 for d in range(0, num_servers):
415 port = _start_keep(d, keep_args)
416 svc = api.keep_services().create(body={'keep_service': {
417 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
418 'service_host': 'localhost',
419 'service_port': port,
420 'service_type': 'disk',
421 'service_ssl_flag': False,
423 api.keep_disks().create(body={
424 'keep_disk': {'keep_service_uuid': svc['uuid'] }
427 # If keepproxy is running, send SIGHUP to make it discover the new
428 # keepstore services.
429 proxypidfile = _pidfile('keepproxy')
430 if os.path.exists(proxypidfile):
432 os.kill(int(open(proxypidfile).read()), signal.SIGHUP)
434 os.remove(proxypidfile)
437 kill_server_pid(_pidfile('keep{}'.format(n)))
438 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
439 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
440 shutil.rmtree(r.read(), True)
441 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
442 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
443 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
445 def stop_keep(num_servers=2):
446 for n in range(0, num_servers):
449 def run_keep_proxy():
450 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
454 port = find_available_port()
455 env = os.environ.copy()
456 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
457 logf = open(_fifo2stderr('keepproxy'), 'w')
458 kp = subprocess.Popen(
460 '-pid='+_pidfile('keepproxy'),
461 '-listen=:{}'.format(port)],
462 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
466 host=os.environ['ARVADOS_API_HOST'],
467 token=auth_token('admin'),
469 for d in api.keep_services().list(
470 filters=[['service_type','=','proxy']]).execute()['items']:
471 api.keep_services().delete(uuid=d['uuid']).execute()
472 api.keep_services().create(body={'keep_service': {
473 'service_host': 'localhost',
474 'service_port': port,
475 'service_type': 'proxy',
476 'service_ssl_flag': False,
478 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
479 _setport('keepproxy', port)
480 _wait_until_port_listens(port)
482 def stop_keep_proxy():
483 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
485 kill_server_pid(_pidfile('keepproxy'))
487 def run_arv_git_httpd():
488 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
492 gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
493 gitport = find_available_port()
494 env = os.environ.copy()
495 env.pop('ARVADOS_API_TOKEN', None)
496 logf = open(_fifo2stderr('arv-git-httpd'), 'w')
497 agh = subprocess.Popen(
499 '-repo-root='+gitdir+'/test',
500 '-address=:'+str(gitport)],
501 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
502 with open(_pidfile('arv-git-httpd'), 'w') as f:
503 f.write(str(agh.pid))
504 _setport('arv-git-httpd', gitport)
505 _wait_until_port_listens(gitport)
507 def stop_arv_git_httpd():
508 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
510 kill_server_pid(_pidfile('arv-git-httpd'))
513 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
517 keepwebport = find_available_port()
518 env = os.environ.copy()
519 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
520 logf = open(_fifo2stderr('keep-web'), 'w')
521 keepweb = subprocess.Popen(
524 '-attachment-only-host=download:'+str(keepwebport),
525 '-listen=:'+str(keepwebport)],
526 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
527 with open(_pidfile('keep-web'), 'w') as f:
528 f.write(str(keepweb.pid))
529 _setport('keep-web', keepwebport)
530 _wait_until_port_listens(keepwebport)
533 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
535 kill_server_pid(_pidfile('keep-web'))
538 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
541 nginxconf['KEEPWEBPORT'] = _getport('keep-web')
542 nginxconf['KEEPWEBDLSSLPORT'] = find_available_port()
543 nginxconf['KEEPWEBSSLPORT'] = find_available_port()
544 nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
545 nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
546 nginxconf['GITPORT'] = _getport('arv-git-httpd')
547 nginxconf['GITSSLPORT'] = find_available_port()
548 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
549 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
550 nginxconf['ACCESSLOG'] = _fifo2stderr('nginx_access_log')
552 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
553 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
554 with open(conffile, 'w') as f:
557 lambda match: str(nginxconf.get(match.group(1))),
558 open(conftemplatefile).read()))
560 env = os.environ.copy()
561 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
563 nginx = subprocess.Popen(
565 '-g', 'error_log stderr info;',
566 '-g', 'pid '+_pidfile('nginx')+';',
568 env=env, stdin=open('/dev/null'), stdout=sys.stderr)
569 _setport('keep-web-dl-ssl', nginxconf['KEEPWEBDLSSLPORT'])
570 _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
571 _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
572 _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
575 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
577 kill_server_pid(_pidfile('nginx'))
579 def _pidfile(program):
580 return os.path.join(TEST_TMPDIR, program + '.pid')
582 def _portfile(program):
583 return os.path.join(TEST_TMPDIR, program + '.port')
585 def _setport(program, port):
586 with open(_portfile(program), 'w') as f:
589 # Returns 9 if program is not up.
590 def _getport(program):
592 return int(open(_portfile(program)).read())
598 return _cached_config[key]
599 def _load(f, required=True):
600 fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
601 if not required and not os.path.exists(fullpath):
603 return yaml.load(fullpath)
604 cdefault = _load('application.default.yml')
605 csite = _load('application.yml', required=False)
607 for section in [cdefault.get('common',{}), cdefault.get('test',{}),
608 csite.get('common',{}), csite.get('test',{})]:
609 _cached_config.update(section)
610 return _cached_config[key]
613 '''load a fixture yaml file'''
614 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
618 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
619 yaml_file = yaml_file[0:trim_index]
622 return yaml.load(yaml_file)
624 def auth_token(token_name):
625 return fixture("api_client_authorizations")[token_name]["api_token"]
627 def authorize_with(token_name):
628 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
629 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
630 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
631 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
633 class TestCaseWithServers(unittest.TestCase):
634 """TestCase to start and stop supporting Arvados servers.
636 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
637 class variables as a dictionary of keyword arguments. If you do,
638 setUpClass will start the corresponding servers by passing these
639 keyword arguments to the run, run_keep, and/or run_keep_server
640 functions, respectively. It will also set Arvados environment
641 variables to point to these servers appropriately. If you don't
642 run a Keep or Keep proxy server, setUpClass will set up a
643 temporary directory for Keep local storage, and set it as
646 tearDownClass will stop any servers started, and restore the
647 original environment.
651 KEEP_PROXY_SERVER = None
652 KEEP_WEB_SERVER = None
655 def _restore_dict(src, dest):
656 for key in dest.keys():
663 cls._orig_environ = os.environ.copy()
664 cls._orig_config = arvados.config.settings().copy()
665 cls._cleanup_funcs = []
666 os.environ.pop('ARVADOS_KEEP_SERVICES', None)
667 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
668 for server_kwargs, start_func, stop_func in (
669 (cls.MAIN_SERVER, run, reset),
670 (cls.KEEP_SERVER, run_keep, stop_keep),
671 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
672 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
673 if server_kwargs is not None:
674 start_func(**server_kwargs)
675 cls._cleanup_funcs.append(stop_func)
676 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
677 cls.local_store = tempfile.mkdtemp()
678 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
679 cls._cleanup_funcs.append(
680 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
682 os.environ.pop('KEEP_LOCAL_STORE', None)
683 arvados.config.initialize()
686 def tearDownClass(cls):
687 for clean_func in cls._cleanup_funcs:
689 cls._restore_dict(cls._orig_environ, os.environ)
690 cls._restore_dict(cls._orig_config, arvados.config.settings())
693 if __name__ == "__main__":
696 'start_keep', 'stop_keep',
697 'start_keep_proxy', 'stop_keep_proxy',
698 'start_keep-web', 'stop_keep-web',
699 'start_arv-git-httpd', 'stop_arv-git-httpd',
700 'start_nginx', 'stop_nginx',
702 parser = argparse.ArgumentParser()
703 parser.add_argument('action', type=str, help="one of {}".format(actions))
704 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
705 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
706 parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions")
708 args = parser.parse_args()
710 if args.action not in actions:
711 print("Unrecognized action '{}'. Actions are: {}.".
712 format(args.action, actions),
715 if args.action == 'start':
716 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
717 run(leave_running_atexit=True)
718 host = os.environ['ARVADOS_API_HOST']
719 if args.auth is not None:
720 token = auth_token(args.auth)
721 print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
722 print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
723 print("export ARVADOS_API_HOST_INSECURE=true")
726 elif args.action == 'stop':
727 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
728 elif args.action == 'start_keep':
729 run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
730 elif args.action == 'stop_keep':
731 stop_keep(num_servers=args.num_keep_servers)
732 elif args.action == 'start_keep_proxy':
734 elif args.action == 'stop_keep_proxy':
736 elif args.action == 'start_arv-git-httpd':
738 elif args.action == 'stop_arv-git-httpd':
740 elif args.action == 'start_keep-web':
742 elif args.action == 'stop_keep-web':
744 elif args.action == 'start_nginx':
746 elif args.action == 'stop_nginx':
749 raise Exception("action recognized but not implemented!?")