git.arvados.org - arvados.git/blob - sdk/python/tests/run_test

2 #

3 # SPDX-License-Identifier: Apache-2.0

5 import argparse

6 import atexit

7 import errno

8 import glob

9 import httplib2

10 import os

11 import random

12 import re

13 import shlex

14 import shutil

15 import signal

16 import socket

17 import subprocess

18 import sys

19 import tempfile

20 import time

21 import unittest

22 import yaml

24 from urllib.parse import urlparse

26 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))

27 if __name__ == '__main__' and os.path.exists(

28 os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):

29 # We're being launched to support another test suite.

30 # Add the Python SDK source to the library path.

31 sys.path.insert(1, os.path.dirname(MY_DIRNAME))

33 import arvados

34 import arvados.config

36 # This module starts subprocesses and records them in pidfiles so they

37 # can be managed by other processes (incl. after this process

38 # exits). But if we don't keep a reference to each subprocess object

39 # somewhere, the subprocess destructor runs, and we get a lot of

40 # ResourceWarning noise in test logs. This is our bucket of subprocess

41 # objects whose destructors we don't want to run but are otherwise

42 # unneeded.

43 _detachedSubprocesses = []

45 ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))

46 SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')

48 # Work around https://bugs.python.org/issue27805, should be no longer

49 # necessary from sometime in Python 3.8.x

50 if not os.environ.get('ARVADOS_DEBUG', ''):

51 WRITE_MODE = 'a'

52 else:

53 WRITE_MODE = 'w'

55 if 'GOPATH' in os.environ:

56 # Add all GOPATH bin dirs to PATH -- but insert them after the

57 # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler

58 # command, not the golang.org/x/tools/cmd/bundle command.

59 gopaths = os.environ['GOPATH'].split(':')

60 addbins = [os.path.join(path, 'bin') for path in gopaths]

61 newbins = []

62 for path in os.environ['PATH'].split(':'):

63 newbins.append(path)

64 if os.path.exists(os.path.join(path, 'bundle')):

65 newbins += addbins

66 addbins = []

67 newbins += addbins

68 os.environ['PATH'] = ':'.join(newbins)

70 TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')

71 if not os.path.exists(TEST_TMPDIR):

72 os.mkdir(TEST_TMPDIR)

74 my_api_host = None

75 _cached_config = {}

76 _cached_db_config = {}

77 _already_used_port = {}

79 def find_server_pid(PID_PATH, wait=10):

80 now = time.time()

81 timeout = now + wait

82 good_pid = False

83 while (not good_pid) and (now <= timeout):

84 time.sleep(0.2)

85 try:

86 with open(PID_PATH, 'r') as f:

87 server_pid = int(f.read())

88 good_pid = (os.kill(server_pid, 0) is None)

89 except EnvironmentError:

90 good_pid = False

91 now = time.time()

93 if not good_pid:

94 return None

96 return server_pid

98 def kill_server_pid(pidfile, wait=10, passenger_root=False):

99 # Must re-import modules in order to work during atexit

100 import os

101 import signal

102 import subprocess

103 import time

104

105 now = time.time()

106 startTERM = now

107 deadline = now + wait

108

109 if passenger_root:

110 # First try to shut down nicely

111 restore_cwd = os.getcwd()

112 os.chdir(passenger_root)

113 subprocess.call([

114 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])

115 os.chdir(restore_cwd)

116 # Use up to half of the +wait+ period waiting for "passenger

117 # stop" to work. If the process hasn't exited by then, start

118 # sending TERM signals.

119 startTERM += wait//2

120

121 server_pid = None

122 while now <= deadline and server_pid is None:

123 try:

124 with open(pidfile, 'r') as f:

125 server_pid = int(f.read())

126 except IOError:

127 # No pidfile = nothing to kill.

128 return

129 except ValueError as error:

130 # Pidfile exists, but we can't parse it. Perhaps the

131 # server has created the file but hasn't written its PID

132 # yet?

133 print("Parse error reading pidfile {}: {}".format(pidfile, error),

134 file=sys.stderr)

135 time.sleep(0.1)

136 now = time.time()

137

138 while now <= deadline:

139 try:

140 exited, _ = os.waitpid(server_pid, os.WNOHANG)

141 if exited > 0:

142 _remove_pidfile(pidfile)

143 return

144 except OSError:

145 # already exited, or isn't our child process

146 pass

147 try:

148 if now >= startTERM:

149 os.kill(server_pid, signal.SIGTERM)

150 print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),

151 file=sys.stderr)

152 except OSError as error:

153 if error.errno == errno.ESRCH:

154 # Thrown by os.getpgid() or os.kill() if the process

155 # does not exist, i.e., our work here is done.

156 _remove_pidfile(pidfile)

157 return

158 raise

159 time.sleep(0.1)

160 now = time.time()

161

162 print("Server PID {} ({}) did not exit, giving up after {}s".

163 format(server_pid, pidfile, wait),

164 file=sys.stderr)

165

166 def _remove_pidfile(pidfile):

167 try:

168 os.unlink(pidfile)

169 except:

170 if os.path.lexists(pidfile):

171 raise

172

173 def find_available_port():

174 """Return an IPv4 port number that is not in use right now.

175

176 We assume whoever needs to use the returned port is able to reuse

177 a recently used port without waiting for TIME_WAIT (see

178 SO_REUSEADDR / SO_REUSEPORT).

179

180 Some opportunity for races here, but it's better than choosing

181 something at random and not checking at all. If all of our servers

182 (hey Passenger) knew that listening on port 0 was a thing, the OS

183 would take care of the races, and this wouldn't be needed at all.

184 """

185

186 global _already_used_port

187 while True:

188 sock = socket.socket()

189 sock.bind(('0.0.0.0', 0))

190 port = sock.getsockname()[1]

191 sock.close()

192 if port not in _already_used_port:

193 _already_used_port[port] = True

194 return port

195

196 def _wait_until_port_listens(port, timeout=10, warn=True):

197 """Wait for a process to start listening on the given port.

198

199 If nothing listens on the port within the specified timeout (given

200 in seconds), print a warning on stderr before returning.

201 """

202 try:

203 subprocess.check_output(['which', 'netstat'])

204 except subprocess.CalledProcessError:

205 print("WARNING: No `netstat` -- cannot wait for port to listen. "+

206 "Sleeping 0.5 and hoping for the best.",

207 file=sys.stderr)

208 time.sleep(0.5)

209 return

210 deadline = time.time() + timeout

211 while time.time() < deadline:

212 if re.search(r'\ntcp.*:'+str(port)+' .* LISTEN *\n', subprocess.check_output(['netstat', '-Wln']).decode()):

213 return True

214 time.sleep(0.1)

215 if warn:

216 print(

217 "WARNING: Nothing is listening on port {} (waited {} seconds).".

218 format(port, timeout),

219 file=sys.stderr)

220 return False

221

222 def _logfilename(label):

223 """Set up a labelled log file, and return a path to write logs to.

224

225 Normally, the returned path is {tmpdir}/{label}.log.

226

227 In debug mode, logs are also written to stderr, with [label]

228 prepended to each line. The returned path is a FIFO.

229

230 +label+ should contain only alphanumerics: it is also used as part

231 of the FIFO filename.

232

233 """

234 logfilename = os.path.join(TEST_TMPDIR, label+'.log')

235 if not os.environ.get('ARVADOS_DEBUG', ''):

236 return logfilename

237 fifo = os.path.join(TEST_TMPDIR, label+'.fifo')

238 try:

239 os.remove(fifo)

240 except OSError as error:

241 if error.errno != errno.ENOENT:

242 raise

243 os.mkfifo(fifo, 0o700)

244 stdbuf = ['stdbuf', '-i0', '-oL', '-eL']

245 # open(fifo, 'r') would block waiting for someone to open the fifo

246 # for writing, so we need a separate cat process to open it for

247 # us.

248 cat = subprocess.Popen(

249 stdbuf+['cat', fifo],

250 stdin=open('/dev/null'),

251 stdout=subprocess.PIPE)

252 _detachedSubprocesses.append(cat)

253 tee = subprocess.Popen(

254 stdbuf+['tee', '-a', logfilename],

255 stdin=cat.stdout,

256 stdout=subprocess.PIPE)

257 _detachedSubprocesses.append(tee)

258 sed = subprocess.Popen(

259 stdbuf+['sed', '-e', 's/^/['+label+'] /'],

260 stdin=tee.stdout,

261 stdout=sys.stderr)

262 _detachedSubprocesses.append(sed)

263 return fifo

264

265 def run(leave_running_atexit=False):

266 """Ensure an API server is running, and ARVADOS_API_* env vars have

267 admin credentials for it.

268

269 If ARVADOS_TEST_API_HOST is set, a parent process has started a

270 test server for us to use: we just need to reset() it using the

271 admin token fixture.

272

273 If a previous call to run() started a new server process, and it

274 is still running, we just need to reset() it to fixture state and

275 return.

276

277 If neither of those options work out, we'll really start a new

278 server.

279 """

280 global my_api_host

281

282 # Delete cached discovery documents.

283 #

284 # This will clear cached docs that belong to other processes (like

285 # concurrent test suites) even if they're still running. They should

286 # be able to tolerate that.

287 for fn in glob.glob(os.path.join(

288 str(arvados.http_cache('discovery')),

289 '*,arvados,v1,rest,*')):

290 os.unlink(fn)

291

292 pid_file = _pidfile('api')

293 pid_file_ok = find_server_pid(pid_file, 0)

294

295 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)

296 if existing_api_host and pid_file_ok:

297 if existing_api_host == my_api_host:

298 try:

299 return reset()

300 except:

301 # Fall through to shutdown-and-start case.

302 pass

303 else:

304 # Server was provided by parent. Can't recover if it's

305 # unresettable.

306 return reset()

307

308 # Before trying to start up our own server, call stop() to avoid

309 # "Phusion Passenger Standalone is already running on PID 12345".

310 # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so

311 # we know the server is ours to kill.)

312 stop(force=True)

313

314 restore_cwd = os.getcwd()

315 api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')

316 os.chdir(api_src_dir)

317

318 # Either we haven't started a server of our own yet, or it has

319 # died, or we have lost our credentials, or something else is

320 # preventing us from calling reset(). Start a new one.

321

322 if not os.path.exists('tmp'):

323 os.makedirs('tmp')

324

325 if not os.path.exists('tmp/api'):

326 os.makedirs('tmp/api')

327

328 if not os.path.exists('tmp/logs'):

329 os.makedirs('tmp/logs')

330

331 # Customizing the passenger config template is the only documented

332 # way to override the default passenger_stat_throttle_rate (10 s).

333 # In the testing environment, we want restart.txt to take effect

334 # immediately.

335 resdir = subprocess.check_output(['bundle', 'exec', 'passenger-config', 'about', 'resourcesdir']).decode().rstrip()

336 with open(resdir + '/templates/standalone/config.erb') as f:

337 template = f.read()

338 newtemplate = re.sub(r'http \{', 'http {\n passenger_stat_throttle_rate 0;', template)

339 if newtemplate == template:

340 raise "template edit failed"

341 with open('tmp/passenger-nginx.conf.erb', 'w') as f:

342 f.write(newtemplate)

343

344 port = internal_port_from_config("RailsAPI")

345 env = os.environ.copy()

346 env['RAILS_ENV'] = 'test'

347 env['ARVADOS_RAILS_LOG_TO_STDOUT'] = '1'

348 env.pop('ARVADOS_WEBSOCKETS', None)

349 env.pop('ARVADOS_TEST_API_HOST', None)

350 env.pop('ARVADOS_API_HOST', None)

351 env.pop('ARVADOS_API_HOST_INSECURE', None)

352 env.pop('ARVADOS_API_TOKEN', None)

353 logf = open(_logfilename('railsapi'), WRITE_MODE)

354 railsapi = subprocess.Popen(

355 ['bundle', 'exec',

356 'passenger', 'start', '-p{}'.format(port),

357 '--nginx-config-template', 'tmp/passenger-nginx.conf.erb',

358 '--no-friendly-error-pages',

359 '--disable-anonymous-telemetry',

360 '--disable-security-update-check',

361 '--pid-file', pid_file,

362 '--log-file', '/dev/stdout',

363 '--ssl',

364 '--ssl-certificate', 'tmp/self-signed.pem',

365 '--ssl-certificate-key', 'tmp/self-signed.key'],

366 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)

367 _detachedSubprocesses.append(railsapi)

368

369 if not leave_running_atexit:

370 atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)

371

372 my_api_host = "127.0.0.1:"+str(port)

373 os.environ['ARVADOS_API_HOST'] = my_api_host

374

375 # Make sure the server has written its pid file and started

376 # listening on its TCP port

377 _wait_until_port_listens(port)

378 find_server_pid(pid_file)

379

380 reset()

381 os.chdir(restore_cwd)

382

383 def reset():

384 """Reset the test server to fixture state.

385

386 This resets the ARVADOS_TEST_API_HOST provided by a parent process

387 if any, otherwise the server started by run().

388

389 It also resets ARVADOS_* environment vars to point to the test

390 server with admin credentials.

391 """

392 existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)

393 token = auth_token('admin')

394 httpclient = httplib2.Http(ca_certs=os.path.join(

395 SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))

396 httpclient.request(

397 'https://{}/database/reset'.format(existing_api_host),

398 'POST',

399 headers={'Authorization': 'OAuth2 {}'.format(token), 'Connection':'close'})

400

401 os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'

402 os.environ['ARVADOS_API_TOKEN'] = token

403 os.environ['ARVADOS_API_HOST'] = existing_api_host

404

405 def stop(force=False):

406 """Stop the API server, if one is running.

407

408 If force==False, kill it only if we started it ourselves. (This

409 supports the use case where a Python test suite calls run(), but

410 run() just uses the ARVADOS_TEST_API_HOST provided by the parent

411 process, and the test suite cleans up after itself by calling

412 stop(). In this case the test server provided by the parent

413 process should be left alone.)

414

415 If force==True, kill it even if we didn't start it

416 ourselves. (This supports the use case in __main__, where "run"

417 and "stop" happen in different processes.)

418 """

419 global my_api_host

420 if force or my_api_host is not None:

421 kill_server_pid(_pidfile('api'))

422 my_api_host = None

423

424 def get_config():

425 with open(os.environ["ARVADOS_CONFIG"]) as f:

426 return yaml.safe_load(f)

427

428 def internal_port_from_config(service, idx=0):

429 return int(urlparse(

430 sorted(list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys()))[idx]).

431 netloc.split(":")[1])

432

433 def external_port_from_config(service):

434 return int(urlparse(get_config()["Clusters"]["zzzzz"]["Services"][service]["ExternalURL"]).netloc.split(":")[1])

435

436 def run_controller():

437 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:

438 return

439 stop_controller()

440 logf = open(_logfilename('controller'), WRITE_MODE)

441 port = internal_port_from_config("Controller")

442 controller = subprocess.Popen(

443 ["arvados-server", "controller"],

444 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)

445 _detachedSubprocesses.append(controller)

446 with open(_pidfile('controller'), 'w') as f:

447 f.write(str(controller.pid))

448 _wait_until_port_listens(port)

449 return port

450

451 def stop_controller():

452 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:

453 return

454 kill_server_pid(_pidfile('controller'))

455

456 def run_ws():

457 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:

458 return

459 stop_ws()

460 port = internal_port_from_config("Websocket")

461 logf = open(_logfilename('ws'), WRITE_MODE)

462 ws = subprocess.Popen(

463 ["arvados-server", "ws"],

464 stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)

465 _detachedSubprocesses.append(ws)

466 with open(_pidfile('ws'), 'w') as f:

467 f.write(str(ws.pid))

468 _wait_until_port_listens(port)

469 return port

470

471 def stop_ws():

472 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:

473 return

474 kill_server_pid(_pidfile('ws'))

475

476 def _start_keep(n, blob_signing=False):

477 datadir = os.path.join(TEST_TMPDIR, "keep%d.data"%n)

478 if os.path.exists(datadir):

479 shutil.rmtree(datadir)

480 os.mkdir(datadir)

481 port = internal_port_from_config("Keepstore", idx=n)

482

483 # Currently, if there are multiple InternalURLs for a single host,

484 # the only way to tell a keepstore process which one it's supposed

485 # to listen on is to supply a redacted version of the config, with

486 # the other InternalURLs removed.

487 conf = os.path.join(TEST_TMPDIR, "keep%d.yaml"%n)

488 confdata = get_config()

489 confdata['Clusters']['zzzzz']['Services']['Keepstore']['InternalURLs'] = {"http://127.0.0.1:%d"%port: {}}

490 confdata['Clusters']['zzzzz']['Collections']['BlobSigning'] = blob_signing

491 with open(conf, 'w') as f:

492 yaml.safe_dump(confdata, f)

493 keep_cmd = ["arvados-server", "keepstore", "-config", conf]

494

495 with open(_logfilename('keep{}'.format(n)), WRITE_MODE) as logf:

496 with open('/dev/null') as _stdin:

497 child = subprocess.Popen(

498 keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)

499 _detachedSubprocesses.append(child)

500

501 print('child.pid is %d'%child.pid, file=sys.stderr)

502 with open(_pidfile('keep{}'.format(n)), 'w') as f:

503 f.write(str(child.pid))

504

505 _wait_until_port_listens(port)

506

507 return port

508

509 def run_keep(num_servers=2, **kwargs):

510 stop_keep(num_servers)

511

512 api = arvados.api(

513 version='v1',

514 host=os.environ['ARVADOS_API_HOST'],

515 token=os.environ['ARVADOS_API_TOKEN'],

516 insecure=True)

517

518 for d in api.keep_services().list(filters=[['service_type','=','disk']]).execute()['items']:

519 api.keep_services().delete(uuid=d['uuid']).execute()

520

521 for d in range(0, num_servers):

522 port = _start_keep(d, **kwargs)

523 svc = api.keep_services().create(body={'keep_service': {

524 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),

525 'service_host': 'localhost',

526 'service_port': port,

527 'service_type': 'disk',

528 'service_ssl_flag': False,

529 }}).execute()

530

531 # If keepproxy and/or keep-web is running, send SIGHUP to make

532 # them discover the new keepstore services.

533 for svc in ('keepproxy', 'keep-web'):

534 pidfile = _pidfile(svc)

535 if os.path.exists(pidfile):

536 try:

537 with open(pidfile) as pid:

538 os.kill(int(pid.read()), signal.SIGHUP)

539 except OSError:

540 os.remove(pidfile)

541

542 def _stop_keep(n):

543 kill_server_pid(_pidfile('keep{}'.format(n)))

544

545 def stop_keep(num_servers=2):

546 for n in range(0, num_servers):

547 _stop_keep(n)

548

549 def run_keep_proxy():

550 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:

551 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(internal_port_from_config('Keepproxy'))

552 return

553 stop_keep_proxy()

554

555 port = internal_port_from_config("Keepproxy")

556 env = os.environ.copy()

557 env['ARVADOS_API_TOKEN'] = auth_token('anonymous')

558 logf = open(_logfilename('keepproxy'), WRITE_MODE)

559 kp = subprocess.Popen(

560 ['arvados-server', 'keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)

561 _detachedSubprocesses.append(kp)

562

563 with open(_pidfile('keepproxy'), 'w') as f:

564 f.write(str(kp.pid))

565 _wait_until_port_listens(port)

566

567 print("Using API %s token %s" % (os.environ['ARVADOS_API_HOST'], auth_token('admin')), file=sys.stdout)

568 api = arvados.api(

569 version='v1',

570 host=os.environ['ARVADOS_API_HOST'],

571 token=auth_token('admin'),

572 insecure=True)

573 for d in api.keep_services().list(

574 filters=[['service_type','=','proxy']]).execute()['items']:

575 api.keep_services().delete(uuid=d['uuid']).execute()

576 api.keep_services().create(body={'keep_service': {

577 'service_host': 'localhost',

578 'service_port': port,

579 'service_type': 'proxy',

580 'service_ssl_flag': False,

581 }}).execute()

582 os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)

583 _wait_until_port_listens(port)

584

585 def stop_keep_proxy():

586 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:

587 return

588 kill_server_pid(_pidfile('keepproxy'))

589

590 def run_keep_web():

591 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:

592 return

593 stop_keep_web()

594

595 keepwebport = internal_port_from_config("WebDAV")

596 env = os.environ.copy()

597 logf = open(_logfilename('keep-web'), WRITE_MODE)

598 keepweb = subprocess.Popen(

599 ['arvados-server', 'keep-web'],

600 env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)

601 _detachedSubprocesses.append(keepweb)

602 with open(_pidfile('keep-web'), 'w') as f:

603 f.write(str(keepweb.pid))

604 _wait_until_port_listens(keepwebport)

605

606 def stop_keep_web():

607 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:

608 return

609 kill_server_pid(_pidfile('keep-web'))

610

611 def run_nginx():

612 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:

613 return

614 stop_nginx()

615 nginxconf = {}

616 nginxconf['UPSTREAMHOST'] = '127.0.0.1'

617 nginxconf['LISTENHOST'] = '127.0.0.1'

618 nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")

619 nginxconf['ARVADOS_API_HOST'] = "0.0.0.0:" + str(external_port_from_config("Controller"))

620 nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")

621 nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV")

622 nginxconf['KEEPWEBDLSSLPORT'] = external_port_from_config("WebDAVDownload")

623 nginxconf['KEEPWEBSSLPORT'] = external_port_from_config("WebDAV")

624 nginxconf['KEEPPROXYPORT'] = internal_port_from_config("Keepproxy")

625 nginxconf['KEEPPROXYSSLPORT'] = external_port_from_config("Keepproxy")

626 nginxconf['HEALTHPORT'] = internal_port_from_config("Health")

627 nginxconf['HEALTHSSLPORT'] = external_port_from_config("Health")

628 nginxconf['WSPORT'] = internal_port_from_config("Websocket")

629 nginxconf['WSSSLPORT'] = external_port_from_config("Websocket")

630 nginxconf['WORKBENCH1SSLPORT'] = external_port_from_config("Workbench1")

631 nginxconf['WORKBENCH2PORT'] = internal_port_from_config("Workbench2")

632 nginxconf['WORKBENCH2SSLPORT'] = external_port_from_config("Workbench2")

633 nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')

634 nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')

635 nginxconf['ACCESSLOG'] = _logfilename('nginx_access')

636 nginxconf['ERRORLOG'] = _logfilename('nginx_error')

637 nginxconf['TMPDIR'] = TEST_TMPDIR + '/nginx'

638 nginxconf['INTERNALSUBNETS'] = '169.254.0.0/16 0;'

639

640 conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')

641 conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')

642 with open(conffile, 'w') as f:

643 f.write(re.sub(

644 r'{{([A-Z]+[A-Z0-9]+)}}',

645 lambda match: str(nginxconf.get(match.group(1))),

646 open(conftemplatefile).read()))

647

648 env = os.environ.copy()

649 env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'

650

651 nginx = subprocess.Popen(

652 ['nginx',

653 '-g', 'error_log stderr notice; pid '+_pidfile('nginx')+';',

654 '-c', conffile],

655 env=env, stdin=open('/dev/null'), stdout=sys.stderr)

656 _detachedSubprocesses.append(nginx)

657 _wait_until_port_listens(nginxconf['CONTROLLERSSLPORT'])

658

659 def setup_config():

660 rails_api_port = find_available_port()

661 controller_port = find_available_port()

662 controller_external_port = find_available_port()

663 websocket_port = find_available_port()

664 websocket_external_port = find_available_port()

665 workbench1_external_port = find_available_port()

666 workbench2_port = find_available_port()

667 workbench2_external_port = find_available_port()

668 health_httpd_port = find_available_port()

669 health_httpd_external_port = find_available_port()

670 keepproxy_port = find_available_port()

671 keepproxy_external_port = find_available_port()

672 keepstore_ports = sorted([str(find_available_port()) for _ in range(0,4)])

673 keep_web_port = find_available_port()

674 keep_web_external_port = find_available_port()

675 keep_web_dl_external_port = find_available_port()

676

677 configsrc = os.environ.get("CONFIGSRC", None)

678 if configsrc:

679 clusterconf = os.path.join(configsrc, "config.yml")

680 print("Getting config from %s" % clusterconf, file=sys.stderr)

681 pgconnection = yaml.safe_load(open(clusterconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]

682 else:

683 # assume "arvados-server install -type test" has set up the

684 # conventional db credentials

685 pgconnection = {

686 "client_encoding": "utf8",

687 "host": "localhost",

688 "dbname": "arvados_test",

689 "user": "arvados",

690 "password": "insecure_arvados_test",

691 }

692

693 localhost = "127.0.0.1"

694 services = {

695 "RailsAPI": {

696 "InternalURLs": {

697 "https://%s:%s"%(localhost, rails_api_port): {},

698 },

699 },

700 "Controller": {

701 "ExternalURL": "https://%s:%s" % (localhost, controller_external_port),

702 "InternalURLs": {

703 "http://%s:%s"%(localhost, controller_port): {},

704 },

705 },

706 "Websocket": {

707 "ExternalURL": "wss://%s:%s/websocket" % (localhost, websocket_external_port),

708 "InternalURLs": {

709 "http://%s:%s"%(localhost, websocket_port): {},

710 },

711 },

712 "Workbench1": {

713 "ExternalURL": "https://%s:%s/" % (localhost, workbench1_external_port),

714 },

715 "Workbench2": {

716 "ExternalURL": "https://%s:%s/" % (localhost, workbench2_external_port),

717 "InternalURLs": {

718 "http://%s:%s"%(localhost, workbench2_port): {},

719 },

720 },

721 "Health": {

722 "ExternalURL": "https://%s:%s" % (localhost, health_httpd_external_port),

723 "InternalURLs": {

724 "http://%s:%s"%(localhost, health_httpd_port): {}

725 },

726 },

727 "Keepstore": {

728 "InternalURLs": {

729 "http://%s:%s"%(localhost, port): {} for port in keepstore_ports

730 },

731 },

732 "Keepproxy": {

733 "ExternalURL": "https://%s:%s" % (localhost, keepproxy_external_port),

734 "InternalURLs": {

735 "http://%s:%s"%(localhost, keepproxy_port): {},

736 },

737 },

738 "WebDAV": {

739 "ExternalURL": "https://%s:%s" % (localhost, keep_web_external_port),

740 "InternalURLs": {

741 "http://%s:%s"%(localhost, keep_web_port): {},

742 },

743 },

744 "WebDAVDownload": {

745 "ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port),

746 "InternalURLs": {

747 "http://%s:%s"%(localhost, keep_web_port): {},

748 },

749 },

750 }

751

752 config = {

753 "Clusters": {

754 "zzzzz": {

755 "ManagementToken": "e687950a23c3a9bceec28c6223a06c79",

756 "SystemRootToken": auth_token('system_user'),

757 "API": {

758 "RequestTimeout": "30s",

759 "LockBeforeUpdate": True,

760 },

761 "Login": {

762 "Test": {

763 "Enable": True,

764 "Users": {

765 "alice": {

766 "Email": "alice@example.com",

767 "Password": "xyzzy"

768 }

769 }

770 },

771 },

772 "SystemLogs": {

773 "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),

774 },

775 "PostgreSQL": {

776 "Connection": pgconnection,

777 },

778 "TLS": {

779 "Insecure": True,

780 },

781 "Services": services,

782 "Users": {

783 "AnonymousUserToken": auth_token('anonymous'),

784 "UserProfileNotificationAddress": "arvados@example.com",

785 },

786 "Collections": {

787 "CollectionVersioning": True,

788 "BlobSigningKey": "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc",

789 "TrustAllContent": False,

790 "ForwardSlashNameSubstitution": "/",

791 "TrashSweepInterval": "-1s", # disable, otherwise test cases can't acquire dblock

792 },

793 "Containers": {

794 "LocalKeepBlobBuffersPerVCPU": 0,

795 "SupportedDockerImageFormats": {"v1": {}},

796 "ShellAccess": {

797 "Admin": True,

798 "User": True,

799 },

800 },

801 "Volumes": {

802 "zzzzz-nyw5e-%015d"%n: {

803 "AccessViaHosts": {

804 "http://%s:%s" % (localhost, keepstore_ports[n]): {},

805 },

806 "Driver": "Directory",

807 "DriverParameters": {

808 "Root": os.path.join(TEST_TMPDIR, "keep%d.data"%n),

809 },

810 } for n in range(len(keepstore_ports))

811 },

812 },

813 },

814 }

815

816 conf = os.path.join(TEST_TMPDIR, 'arvados.yml')

817 with open(conf, 'w') as f:

818 yaml.safe_dump(config, f)

819

820 ex = "export ARVADOS_CONFIG="+conf

821 print(ex)

822

823

824 def stop_nginx():

825 if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:

826 return

827 kill_server_pid(_pidfile('nginx'))

828

829 def _pidfile(program):

830 return os.path.join(TEST_TMPDIR, program + '.pid')

831

832 def fixture(fix):

833 '''load a fixture yaml file'''

834 with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",

835 fix + ".yml")) as f:

836 yaml_file = f.read()

837 try:

838 trim_index = yaml_file.index("# Test Helper trims the rest of the file")

839 yaml_file = yaml_file[0:trim_index]

840 except ValueError:

841 pass

842 return yaml.safe_load(yaml_file)

843

844 def auth_token(token_name):

845 return fixture("api_client_authorizations")[token_name]["api_token"]

846

847 def authorize_with(token_name):

848 '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''

849 arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)

850 arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")

851 arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"

852

853 class TestCaseWithServers(unittest.TestCase):

854 """TestCase to start and stop supporting Arvados servers.

855

856 Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER

857 class variables as a dictionary of keyword arguments. If you do,

858 setUpClass will start the corresponding servers by passing these

859 keyword arguments to the run, run_keep, and/or run_keep_server

860 functions, respectively. It will also set Arvados environment

861 variables to point to these servers appropriately. If you don't

862 run a Keep or Keep proxy server, setUpClass will set up a

863 temporary directory for Keep local storage, and set it as

864 KEEP_LOCAL_STORE.

865

866 tearDownClass will stop any servers started, and restore the

867 original environment.

868 """

869 MAIN_SERVER = None

870 WS_SERVER = None

871 KEEP_SERVER = None

872 KEEP_PROXY_SERVER = None

873 KEEP_WEB_SERVER = None

874

875 @staticmethod

876 def _restore_dict(src, dest):

877 for key in list(dest.keys()):

878 if key not in src:

879 del dest[key]

880 dest.update(src)

881

882 @classmethod

883 def setUpClass(cls):

884 cls._orig_environ = os.environ.copy()

885 cls._orig_config = arvados.config.settings().copy()

886 cls._cleanup_funcs = []

887 os.environ.pop('ARVADOS_KEEP_SERVICES', None)

888 for server_kwargs, start_func, stop_func in (

889 (cls.MAIN_SERVER, run, reset),

890 (cls.WS_SERVER, run_ws, stop_ws),

891 (cls.KEEP_SERVER, run_keep, stop_keep),

892 (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),

893 (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):

894 if server_kwargs is not None:

895 start_func(**server_kwargs)

896 cls._cleanup_funcs.append(stop_func)

897 if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):

898 cls.local_store = tempfile.mkdtemp()

899 os.environ['KEEP_LOCAL_STORE'] = cls.local_store

900 cls._cleanup_funcs.append(

901 lambda: shutil.rmtree(cls.local_store, ignore_errors=True))

902 else:

903 os.environ.pop('KEEP_LOCAL_STORE', None)

904 arvados.config.initialize()

905

906 @classmethod

907 def tearDownClass(cls):

908 for clean_func in cls._cleanup_funcs:

909 clean_func()

910 cls._restore_dict(cls._orig_environ, os.environ)

911 cls._restore_dict(cls._orig_config, arvados.config.settings())

912

913

914 if __name__ == "__main__":

915 actions = [

916 'start', 'stop',

917 'start_ws', 'stop_ws',

918 'start_controller', 'stop_controller',

919 'start_keep', 'stop_keep',

920 'start_keep_proxy', 'stop_keep_proxy',

921 'start_keep-web', 'stop_keep-web',

922 'start_nginx', 'stop_nginx', 'setup_config',

923 ]

924 parser = argparse.ArgumentParser()

925 parser.add_argument('action', type=str, help="one of {}".format(actions))

926 parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')

927 parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")

928 parser.add_argument('--keep-blob-signing', action="store_true", help="Enable blob signing for keepstore servers")

929

930 args = parser.parse_args()

931

932 if args.action not in actions:

933 print("Unrecognized action '{}'. Actions are: {}.".

934 format(args.action, actions),

935 file=sys.stderr)

936 sys.exit(1)

937 # Create a new process group so our child processes don't exit on

938 # ^C in run-tests.sh interactive mode.

939 os.setpgid(0, 0)

940 if args.action == 'start':

941 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))

942 run(leave_running_atexit=True)

943 host = os.environ['ARVADOS_API_HOST']

944 if args.auth is not None:

945 token = auth_token(args.auth)

946 print("export ARVADOS_API_TOKEN={}".format(shlex.quote(token)))

947 print("export ARVADOS_API_HOST={}".format(shlex.quote(host)))

948 print("export ARVADOS_API_HOST_INSECURE=true")

949 else:

950 print(host)

951 elif args.action == 'stop':

952 stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))

953 elif args.action == 'start_ws':

954 run_ws()

955 elif args.action == 'stop_ws':

956 stop_ws()

957 elif args.action == 'start_controller':

958 run_controller()

959 elif args.action == 'stop_controller':

960 stop_controller()

961 elif args.action == 'start_keep':

962 run_keep(blob_signing=args.keep_blob_signing, num_servers=args.num_keep_servers)

963 elif args.action == 'stop_keep':

964 stop_keep(num_servers=args.num_keep_servers)

965 elif args.action == 'start_keep_proxy':

966 run_keep_proxy()

967 elif args.action == 'stop_keep_proxy':

968 stop_keep_proxy()

969 elif args.action == 'start_keep-web':

970 run_keep_web()

971 elif args.action == 'stop_keep-web':

972 stop_keep_web()

973 elif args.action == 'start_nginx':

974 run_nginx()

975 print("export ARVADOS_API_HOST=0.0.0.0:{}".format(external_port_from_config('Controller')))

976 elif args.action == 'stop_nginx':

977 stop_nginx()

978 elif args.action == 'setup_config':

979 setup_config()

980 else:

981 raise Exception("action recognized but not implemented!?")