X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/df070cb903ec46ce51eb610d44530369a824b12b..8e1e7e6bf355eb0f1defc7278f1434c393a75a75:/sdk/python/tests/run_test_server.py diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index 31493cc10a..972b7f9d51 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -3,6 +3,7 @@ from __future__ import print_function import argparse import atexit +import errno import httplib2 import os import pipes @@ -54,9 +55,7 @@ def find_server_pid(PID_PATH, wait=10): with open(PID_PATH, 'r') as f: server_pid = int(f.read()) good_pid = (os.kill(server_pid, 0) is None) - except IOError: - good_pid = False - except OSError: + except EnvironmentError: good_pid = False now = time.time() @@ -91,9 +90,7 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False): os.getpgid(server_pid) time.sleep(0.1) now = time.time() - except IOError: - pass - except OSError: + except EnvironmentError: pass def find_available_port(): @@ -122,16 +119,17 @@ def _wait_until_port_listens(port, timeout=10): in seconds), print a warning on stderr before returning. """ try: - subprocess.check_output(['fuser', '-l']) + subprocess.check_output(['which', 'lsof']) except subprocess.CalledProcessError: - print("WARNING: No `fuser` -- cannot wait for port to listen. "+ + print("WARNING: No `lsof` -- cannot wait for port to listen. "+ "Sleeping 0.5 and hoping for the best.") time.sleep(0.5) return deadline = time.time() + timeout while time.time() < deadline: try: - fuser_says = subprocess.check_output(['fuser', str(port)+'/tcp']) + subprocess.check_output( + ['lsof', '-t', '-i', 'tcp:'+str(port)]) except subprocess.CalledProcessError: time.sleep(0.1) continue @@ -309,8 +307,9 @@ def _start_keep(n, keep_args): for arg, val in keep_args.iteritems(): keep_cmd.append("{}={}".format(arg, val)) + logf = open(os.path.join(TEST_TMPDIR, 'keep{}.log'.format(n)), 'a+') kp0 = subprocess.Popen( - keep_cmd, stdin=open('/dev/null'), stdout=sys.stderr) + keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) with open(_pidfile('keep{}'.format(n)), 'w') as f: f.write(str(kp0.pid)) @@ -321,28 +320,34 @@ def _start_keep(n, keep_args): return port -def run_keep(blob_signing_key=None, enforce_permissions=False): - stop_keep() +def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2): + stop_keep(num_servers) keep_args = {} - if blob_signing_key: - with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f: - keep_args['--permission-key-file'] = f.name - f.write(blob_signing_key) + if not blob_signing_key: + blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc' + with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f: + keep_args['-blob-signing-key-file'] = f.name + f.write(blob_signing_key) if enforce_permissions: - keep_args['--enforce-permissions'] = 'true' + keep_args['-enforce-permissions'] = 'true' + with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f: + keep_args['-data-manager-token-file'] = f.name + f.write(auth_token('data_manager')) + keep_args['-never-delete'] = 'false' api = arvados.api( version='v1', host=os.environ['ARVADOS_API_HOST'], token=os.environ['ARVADOS_API_TOKEN'], insecure=True) + for d in api.keep_services().list().execute()['items']: api.keep_services().delete(uuid=d['uuid']).execute() for d in api.keep_disks().list().execute()['items']: api.keep_disks().delete(uuid=d['uuid']).execute() - for d in range(0, 2): + for d in range(0, num_servers): port = _start_keep(d, keep_args) svc = api.keep_services().create(body={'keep_service': { 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d), @@ -364,9 +369,9 @@ def _stop_keep(n): if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")): os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")) -def stop_keep(): - _stop_keep(0) - _stop_keep(1) +def stop_keep(num_servers=2): + for n in range(0, num_servers): + _stop_keep(n) def run_keep_proxy(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: @@ -440,6 +445,7 @@ def run_nginx(): nginxconf['GITSSLPORT'] = find_available_port() nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem') nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key') + nginxconf['ACCESSLOG'] = os.path.join(TEST_TMPDIR, 'nginx_access_log.fifo') conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf') conffile = os.path.join(TEST_TMPDIR, 'nginx.conf') @@ -451,12 +457,23 @@ def run_nginx(): env = os.environ.copy() env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin' + + try: + os.remove(nginxconf['ACCESSLOG']) + except OSError as error: + if error.errno != errno.ENOENT: + raise + + os.mkfifo(nginxconf['ACCESSLOG'], 0700) nginx = subprocess.Popen( ['nginx', '-g', 'error_log stderr info;', '-g', 'pid '+_pidfile('nginx')+';', '-c', conffile], env=env, stdin=open('/dev/null'), stdout=sys.stderr) + cat_access = subprocess.Popen( + ['cat', nginxconf['ACCESSLOG']], + stdout=sys.stderr) _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT']) _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT']) @@ -588,6 +605,9 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('action', type=str, help="one of {}".format(actions)) parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture') + parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired") + parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions") + args = parser.parse_args() if args.action not in actions: @@ -607,7 +627,7 @@ if __name__ == "__main__": elif args.action == 'stop': stop(force=('ARVADOS_TEST_API_HOST' not in os.environ)) elif args.action == 'start_keep': - run_keep() + run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers) elif args.action == 'stop_keep': stop_keep() elif args.action == 'start_keep_proxy':