git.arvados.org - arvados.git/blob - sdk/python/run_test

1 import subprocess

2 import time

3 import os

4 import signal

5 import yaml

6 import sys

7 import argparse

8 import arvados.config

9 import arvados.api

10 import shutil

11 import tempfile

13 ARV_API_SERVER_DIR = '../../services/api'

14 KEEP_SERVER_DIR = '../../services/keep'

15 SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'

16 WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'

18 def find_server_pid(PID_PATH, wait=10):

19 now = time.time()

20 timeout = now + wait

21 good_pid = False

22 while (not good_pid) and (now <= timeout):

23 time.sleep(0.2)

24 try:

25 with open(PID_PATH, 'r') as f:

26 server_pid = int(f.read())

27 good_pid = (os.kill(server_pid, 0) == None)

28 except IOError:

29 good_pid = False

30 except OSError:

31 good_pid = False

32 now = time.time()

34 if not good_pid:

35 return None

37 return server_pid

39 def kill_server_pid(PID_PATH, wait=10):

40 try:

41 now = time.time()

42 timeout = now + wait

43 with open(PID_PATH, 'r') as f:

44 server_pid = int(f.read())

45 while now <= timeout:

46 os.kill(server_pid, signal.SIGTERM) == None

47 os.getpgid(server_pid) # throw OSError if no such pid

48 now = time.time()

49 time.sleep(0.1)

50 except IOError:

51 good_pid = False

52 except OSError:

53 good_pid = False

55 def run(websockets=False, reuse_server=False):

56 cwd = os.getcwd()

57 os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))

59 if websockets:

60 pid_file = WEBSOCKETS_SERVER_PID_PATH

61 else:

62 pid_file = SERVER_PID_PATH

64 test_pid = find_server_pid(pid_file, 0)

66 if test_pid == None or not reuse_server:

67 # do not try to run both server variants at once

68 stop()

70 # delete cached discovery document

71 shutil.rmtree(arvados.http_cache('discovery'))

73 # Setup database

74 os.environ["RAILS_ENV"] = "test"

75 subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])

76 subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])

77 subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])

79 if websockets:

80 os.environ["ARVADOS_WEBSOCKETS"] = "true"

81 subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',

82 '-out', './self-signed.pem',

83 '-keyout', './self-signed.key',

84 '-days', '3650',

85 '-subj', '/CN=localhost'])

86 subprocess.call(['bundle', 'exec',

87 'passenger', 'start', '-d', '-p3333',

88 '--pid-file',

89 os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH),

90 '--ssl',

91 '--ssl-certificate', 'self-signed.pem',

92 '--ssl-certificate-key', 'self-signed.key'])

93 os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333"

94 else:

95 subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',

96 '--pid',

97 os.path.join(os.getcwd(), SERVER_PID_PATH),

98 '-p3001'])

99 os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"

100

101 pid = find_server_pid(SERVER_PID_PATH)

102

103 os.environ["ARVADOS_API_HOST_INSECURE"] = "true"

104 os.environ["ARVADOS_API_TOKEN"] = ""

105 os.chdir(cwd)

106

107 def stop():

108 cwd = os.getcwd()

109 os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))

110

111 kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)

112 kill_server_pid(SERVER_PID_PATH, 0)

113

114 try:

115 os.unlink('self-signed.pem')

116 except:

117 pass

118

119 try:

120 os.unlink('self-signed.key')

121 except:

122 pass

123

124 os.chdir(cwd)

125

126 def _start_keep(n):

127 keep0 = tempfile.mkdtemp()

128 kp0 = subprocess.Popen(["bin/keep", "-volumes={}".format(keep0), "-listen=:{}".format(25107+n)])

129 with open("tmp/keep{}.pid".format(n), 'w') as f:

130 f.write(str(kp0.pid))

131 with open("tmp/keep{}.volume".format(n), 'w') as f:

132 f.write(keep0)

133

134 def run_keep():

135 stop_keep()

136

137 cwd = os.getcwd()

138 os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))

139 if os.environ.get('GOPATH') == None:

140 os.environ["GOPATH"] = os.getcwd()

141 else:

142 os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]

143

144 subprocess.call(["go", "install", "keep"])

145

146 if not os.path.exists("tmp"):

147 os.mkdir("tmp")

148

149 _start_keep(0)

150 _start_keep(1)

151

152 authorize_with("admin")

153 api = arvados.api('v1', cache=False)

154 a = api.keep_disks().list().execute()

155 for d in api.keep_disks().list().execute()['items']:

156 api.keep_disks().delete(uuid=d['uuid']).execute()

157

158 api.keep_disks().create(body={"keep_disk": {"service_host": "localhost", "service_port": 25107} }).execute()

159 api.keep_disks().create(body={"keep_disk": {"service_host": "localhost", "service_port": 25108} }).execute()

160

161 os.chdir(cwd)

162

163 def _stop_keep(n):

164 kill_server_pid("tmp/keep{}.pid".format(n), 0)

165 if os.path.exists("tmp/keep{}.volume".format(n)):

166 with open("tmp/keep{}.volume".format(n), 'r') as r:

167 shutil.rmtree(r.read(), True)

168

169 def stop_keep():

170 cwd = os.getcwd()

171 os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))

172

173 _stop_keep(0)

174 _stop_keep(1)

175

176 shutil.rmtree("tmp", True)

177

178 os.chdir(cwd)

179

180 def fixture(fix):

181 '''load a fixture yaml file'''

182 with open(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR, "test", "fixtures",

183 fix + ".yml")) as f:

184 return yaml.load(f.read())

185

186 def authorize_with(token):

187 '''token is the symbolic name of the token from the api_client_authorizations fixture'''

188 arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]

189 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")

190 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"

191

192 if __name__ == "__main__":

193 parser = argparse.ArgumentParser()

194 parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')

195 parser.add_argument('--websockets', action='store_true', default=False)

196 parser.add_argument('--reuse', action='store_true', default=False)

197 parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')

198 args = parser.parse_args()

199

200 if args.action == 'start':

201 run(websockets=args.websockets, reuse_server=args.reuse)

202 if args.auth != None:

203 authorize_with(args.auth)

204 print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))

205 print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))

206 print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))

207 elif args.action == 'stop':

208 stop()

209 elif args.action == 'start_keep':

210 run_keep()

211 elif args.action == 'stop_keep':

212 stop_keep()