14 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
15 if __name__ == '__main__' and os.path.exists(
16 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
17 # We're being launched to support another test suite.
18 # Add the Python SDK source to the library path.
19 sys.path.insert(1, os.path.dirname(MY_DIRNAME))
24 SERVICES_SRC_DIR = os.path.join(MY_DIRNAME, '../../../services')
25 SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'
26 WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'
27 if 'GOPATH' in os.environ:
28 gopaths = os.environ['GOPATH'].split(':')
29 gobins = [os.path.join(path, 'bin') for path in gopaths]
30 os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
32 if os.path.isdir('tests'):
33 TEST_TMPDIR = 'tests/tmp'
37 def find_server_pid(PID_PATH, wait=10):
41 while (not good_pid) and (now <= timeout):
44 with open(PID_PATH, 'r') as f:
45 server_pid = int(f.read())
46 good_pid = (os.kill(server_pid, 0) == None)
58 def kill_server_pid(PID_PATH, wait=10):
62 with open(PID_PATH, 'r') as f:
63 server_pid = int(f.read())
65 os.kill(server_pid, signal.SIGTERM) == None
66 os.getpgid(server_pid) # throw OSError if no such pid
74 def run(websockets=False, reuse_server=False):
76 os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
79 pid_file = WEBSOCKETS_SERVER_PID_PATH
81 pid_file = SERVER_PID_PATH
83 test_pid = find_server_pid(pid_file, 0)
85 if test_pid == None or not reuse_server:
86 # do not try to run both server variants at once
89 # delete cached discovery document
90 shutil.rmtree(arvados.http_cache('discovery'))
93 os.environ["RAILS_ENV"] = "test"
94 subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])
95 subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
96 subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
99 os.environ["ARVADOS_WEBSOCKETS"] = "true"
100 subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
101 '-out', './self-signed.pem',
102 '-keyout', './self-signed.key',
104 '-subj', '/CN=localhost'])
105 subprocess.call(['bundle', 'exec',
106 'passenger', 'start', '-d', '-p3333',
108 os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH),
110 '--ssl-certificate', 'self-signed.pem',
111 '--ssl-certificate-key', 'self-signed.key'])
112 os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333"
114 subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
116 os.path.join(os.getcwd(), SERVER_PID_PATH),
118 os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
120 pid = find_server_pid(SERVER_PID_PATH)
122 os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
123 os.environ["ARVADOS_API_TOKEN"] = ""
128 os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
130 kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
131 kill_server_pid(SERVER_PID_PATH, 0)
134 os.unlink('self-signed.pem')
139 os.unlink('self-signed.key')
145 def _start_keep(n, keep_args):
146 keep0 = tempfile.mkdtemp()
147 keep_cmd = ["keepstore",
148 "-volumes={}".format(keep0),
149 "-listen=:{}".format(25107+n),
150 "-pid={}".format("{}/keep{}.pid".format(TEST_TMPDIR, n))]
152 for arg, val in keep_args.iteritems():
153 keep_cmd.append("{}={}".format(arg, val))
155 kp0 = subprocess.Popen(keep_cmd)
156 with open("{}/keep{}.pid".format(TEST_TMPDIR, n), 'w') as f:
157 f.write(str(kp0.pid))
159 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
162 def run_keep(blob_signing_key=None, enforce_permissions=False):
165 if not os.path.exists(TEST_TMPDIR):
166 os.mkdir(TEST_TMPDIR)
170 with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
171 keep_args['--permission-key-file'] = f.name
172 f.write(blob_signing_key)
173 if enforce_permissions:
174 keep_args['--enforce-permissions'] = 'true'
176 _start_keep(0, keep_args)
177 _start_keep(1, keep_args)
179 os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
180 os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
182 authorize_with("admin")
183 api = arvados.api('v1', cache=False)
184 for d in api.keep_services().list().execute()['items']:
185 api.keep_services().delete(uuid=d['uuid']).execute()
186 for d in api.keep_disks().list().execute()['items']:
187 api.keep_disks().delete(uuid=d['uuid']).execute()
189 s1 = api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25107, "service_type": "disk"} }).execute()
190 s2 = api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25108, "service_type": "disk"} }).execute()
191 api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute()
192 api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
195 kill_server_pid("{}/keep{}.pid".format(TEST_TMPDIR, n), 0)
196 if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
197 with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
198 shutil.rmtree(r.read(), True)
199 os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
200 if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
201 os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
207 def run_keep_proxy(auth):
210 if not os.path.exists(TEST_TMPDIR):
211 os.mkdir(TEST_TMPDIR)
213 os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
214 os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
215 os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
217 kp0 = subprocess.Popen(["keepproxy",
218 "-pid={}/keepproxy.pid".format(TEST_TMPDIR),
219 "-listen=:{}".format(25101)])
221 authorize_with("admin")
222 api = arvados.api('v1', cache=False)
223 api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25101, "service_type": "proxy"} }).execute()
225 os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
227 def stop_keep_proxy():
228 kill_server_pid(os.path.join(TEST_TMPDIR, "keepproxy.pid"), 0)
231 '''load a fixture yaml file'''
232 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",
236 trim_index = yaml_file.index("# Test Helper trims the rest of the file")
237 yaml_file = yaml_file[0:trim_index]
240 return yaml.load(yaml_file)
242 def authorize_with(token):
243 '''token is the symbolic name of the token from the api_client_authorizations fixture'''
244 arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
245 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
246 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
248 class TestCaseWithServers(unittest.TestCase):
249 """TestCase to start and stop supporting Arvados servers.
251 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
252 class variables as a dictionary of keyword arguments. If you do,
253 setUpClass will start the corresponding servers by passing these
254 keyword arguments to the run, run_keep, and/or run_keep_server
255 functions, respectively. It will also set Arvados environment
256 variables to point to these servers appropriately. If you don't
257 run a Keep or Keep proxy server, setUpClass will set up a
258 temporary directory for Keep local storage, and set it as
261 tearDownClass will stop any servers started, and restore the
262 original environment.
266 KEEP_PROXY_SERVER = None
269 def _restore_dict(src, dest):
270 for key in dest.keys():
277 cls._orig_environ = os.environ.copy()
278 cls._orig_config = arvados.config.settings().copy()
279 cls._cleanup_funcs = []
280 for server_kwargs, start_func, stop_func in (
281 (cls.MAIN_SERVER, run, stop),
282 (cls.KEEP_SERVER, run_keep, stop_keep),
283 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
284 if server_kwargs is not None:
285 start_func(**server_kwargs)
286 cls._cleanup_funcs.append(stop_func)
287 os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
288 if cls.KEEP_PROXY_SERVER is None:
289 os.environ.pop('ARVADOS_KEEP_PROXY', None)
290 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
291 cls.local_store = tempfile.mkdtemp()
292 os.environ['KEEP_LOCAL_STORE'] = cls.local_store
293 cls._cleanup_funcs.append(
294 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
296 os.environ.pop('KEEP_LOCAL_STORE', None)
297 arvados.config.initialize()
300 def tearDownClass(cls):
301 for clean_func in cls._cleanup_funcs:
303 cls._restore_dict(cls._orig_environ, os.environ)
304 cls._restore_dict(cls._orig_config, arvados.config.settings())
307 if __name__ == "__main__":
308 parser = argparse.ArgumentParser()
309 parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
310 parser.add_argument('--websockets', action='store_true', default=False)
311 parser.add_argument('--reuse', action='store_true', default=False)
312 parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')
313 args = parser.parse_args()
315 if args.action == 'start':
316 run(websockets=args.websockets, reuse_server=args.reuse)
317 if args.auth != None:
318 authorize_with(args.auth)
319 print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
320 print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
321 print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))
322 elif args.action == 'stop':
324 elif args.action == 'start_keep':
326 elif args.action == 'stop_keep':
328 elif args.action == 'start_keep_proxy':
329 run_keep_proxy("admin")
330 elif args.action == 'stop_keep_proxy':
333 print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))