import sys
import argparse
import arvados.config
+import arvados.api
import shutil
import tempfile
ARV_API_SERVER_DIR = '../../services/api'
KEEP_SERVER_DIR = '../../services/keep'
-SERVER_PID_PATH = 'tmp/pids/server.pid'
-WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger.3001.pid'
+SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'
+WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'
def find_server_pid(PID_PATH, wait=10):
now = time.time()
try:
now = time.time()
timeout = now + wait
+ with open(PID_PATH, 'r') as f:
+ server_pid = int(f.read())
while now <= timeout:
- with open(PID_PATH, 'r') as f:
- server_pid = int(f.read())
os.kill(server_pid, signal.SIGTERM) == None
+ os.getpgid(server_pid) # throw OSError if no such pid
now = time.time()
+ time.sleep(0.1)
except IOError:
good_pid = False
except OSError:
test_pid = find_server_pid(pid_file, 0)
if test_pid == None or not reuse_server:
- if test_pid != None:
- stop()
+ # do not try to run both server variants at once
+ stop()
# delete cached discovery document
- shutil.rmtree(os.path.join("~", ".cache", "arvados", "discovery"), True)
+ shutil.rmtree(arvados.http_cache('discovery'))
# Setup database
os.environ["RAILS_ENV"] = "test"
+ subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])
subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
'-keyout', './self-signed.key',
'-days', '3650',
'-subj', '/CN=localhost'])
- subprocess.call(['passenger', 'start', '-d', '-p3001', '--ssl',
+ subprocess.call(['bundle', 'exec',
+ 'passenger', 'start', '-d', '-p3333',
+ '--pid-file',
+ os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH),
+ '--ssl',
'--ssl-certificate', 'self-signed.pem',
'--ssl-certificate-key', 'self-signed.key'])
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333"
else:
- subprocess.call(['bundle', 'exec', 'rails', 'server', '-d', '-p3001'])
+ subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
+ '--pid',
+ os.path.join(os.getcwd(), SERVER_PID_PATH),
+ '-p3001'])
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
pid = find_server_pid(SERVER_PID_PATH)
- #os.environ["ARVADOS_API_HOST"] = "localhost:3001"
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
os.environ["ARVADOS_API_TOKEN"] = ""
os.chdir(cwd)
os.chdir(cwd)
-def _start_keep(n):
+def _start_keep(n, keep_args):
keep0 = tempfile.mkdtemp()
- kp0 = subprocess.Popen(["bin/keep", "-volumes={}".format(keep0), "-listen=:{}".format(25107+n)])
+ keep_cmd = ["bin/keep",
+ "-volumes={}".format(keep0),
+ "-listen=:{}".format(25107+n),
+ "-pid={}".format("tmp/keep{}.pid".format(n))]
+
+ for arg, val in keep_args.iteritems():
+ keep_cmd.append("{}={}".format(arg, val))
+
+ kp0 = subprocess.Popen(keep_cmd)
with open("tmp/keep{}.pid".format(n), 'w') as f:
f.write(str(kp0.pid))
+
with open("tmp/keep{}.volume".format(n), 'w') as f:
f.write(keep0)
-def run_keep():
+def run_keep(blob_signing_key=None, enforce_permissions=False):
stop_keep()
cwd = os.getcwd()
os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
- os.environ["GOPATH"] = os.getcwd()
- subprocess.call(["go", "install", "keep"])
+ if os.environ.get('GOPATH') == None:
+ os.environ["GOPATH"] = os.getcwd()
+ else:
+ os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]
+
+ subprocess.call(["./go.sh", "install", "keep"])
if not os.path.exists("tmp"):
os.mkdir("tmp")
- _start_keep(0)
- _start_keep(1)
+ keep_args = {}
+ if blob_signing_key:
+ with open("tmp/keep.blob_signing_key", "w") as f:
+ f.write(blob_signing_key)
+ keep_args['--permission-key-file'] = 'tmp/keep.blob_signing_key'
+ if enforce_permissions:
+ keep_args['--enforce-permissions'] = 'true'
+
+ _start_keep(0, keep_args)
+ _start_keep(1, keep_args)
+
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
+ os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
authorize_with("admin")
- api = arvados.api('v1')
- a = api.keep_disks().list().execute()
+ api = arvados.api('v1', cache=False)
+ for d in api.keep_services().list().execute()['items']:
+ api.keep_services().delete(uuid=d['uuid']).execute()
for d in api.keep_disks().list().execute()['items']:
api.keep_disks().delete(uuid=d['uuid']).execute()
- api.keep_disks().create(body={"keep_disk": {"service_host": "localhost", "service_port": 25107} }).execute()
- api.keep_disks().create(body={"keep_disk": {"service_host": "localhost", "service_port": 25108} }).execute()
+ s1 = api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25107, "service_type": "disk"} }).execute()
+ s2 = api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25108, "service_type": "disk"} }).execute()
+ api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute()
+ api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
os.chdir(cwd)
if os.path.exists("tmp/keep{}.volume".format(n)):
with open("tmp/keep{}.volume".format(n), 'r') as r:
shutil.rmtree(r.read(), True)
+ os.unlink("tmp/keep{}.volume".format(n))
+ if os.path.exists("tmp/keep.blob_signing_key"):
+ os.remove("tmp/keep.blob_signing_key")
def stop_keep():
cwd = os.getcwd()
_stop_keep(0)
_stop_keep(1)
- shutil.rmtree("tmp", True)
+ os.chdir(cwd)
+def run_keep_proxy(auth):
+ stop_keep_proxy()
+
+ cwd = os.getcwd()
+ os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+ if os.environ.get('GOPATH') == None:
+ os.environ["GOPATH"] = os.getcwd()
+ else:
+ os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]
+
+ subprocess.call(["./go.sh", "install", "arvados.org/keepproxy"])
+
+ if not os.path.exists("tmp"):
+ os.mkdir("tmp")
+
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
+ os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
+ os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
+
+ kp0 = subprocess.Popen(["bin/keepproxy", "-pid=tmp/keepproxy.pid", "-listen=:{}".format(25101)])
+
+ authorize_with("admin")
+ api = arvados.api('v1', cache=False)
+ api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25101, "service_type": "proxy"} }).execute()
+
+ arvados.config.settings()["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
+
+ os.chdir(cwd)
+
+def stop_keep_proxy():
+ cwd = os.getcwd()
+ os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+ kill_server_pid("tmp/keepproxy.pid", 0)
os.chdir(cwd)
def fixture(fix):
def authorize_with(token):
'''token is the symbolic name of the token from the api_client_authorizations fixture'''
arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
+ arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
+ arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
if __name__ == "__main__":
parser = argparse.ArgumentParser()
args = parser.parse_args()
if args.action == 'start':
- run(args.websockets, args.reuse)
+ run(websockets=args.websockets, reuse_server=args.reuse)
if args.auth != None:
authorize_with(args.auth)
print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
run_keep()
elif args.action == 'stop_keep':
stop_keep()
+ elif args.action == 'start_keep_proxy':
+ run_keep_proxy("admin")
+ elif args.action == 'stop_keep_proxy':
+ stop_keep_proxy()
+ else:
+ print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))
projects / arvados.git / blobdiff