From: Tom Clegg <tclegg@veritasgenetics.com>
Date: Tue, 23 Apr 2019 17:28:55 +0000 (-0400)
Subject: 15003: Check DB conn in controller health check. Check test services.
X-Git-Tag: 1.4.0~26^2~8
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/86074c13f4441fa0804e30a1d68781175ba32e0d

15003: Check DB conn in controller health check. Check test services.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>
---

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 9690cbbb93..a33dab7c32 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -379,6 +379,20 @@ checkpidfile() {
     echo "${svc} pid ${pid} ok"
 }
 
+checkhealth() {
+    svc="$1"
+    port="$(cat "$WORKSPACE/tmp/${svc}.port")"
+    scheme=http
+    if [[ ${svc} =~ -ssl$ || ${svc} = wss ]]; then
+        scheme=https
+    fi
+    url="$scheme://localhost:${port}/_health/ping"
+    if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then
+        echo "${url} failed"
+        return 1
+    fi
+}
+
 checkdiscoverydoc() {
     dd="https://${1}/discovery/v1/apis/arvados/v1/rest"
     if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then
@@ -412,12 +426,15 @@ start_services() {
         && checkdiscoverydoc $ARVADOS_API_HOST \
         && python sdk/python/tests/run_test_server.py start_controller \
         && checkpidfile controller \
+        && checkhealth controller \
         && python sdk/python/tests/run_test_server.py start_keep_proxy \
         && checkpidfile keepproxy \
         && python sdk/python/tests/run_test_server.py start_keep-web \
         && checkpidfile keep-web \
+        && checkhealth keep-web \
         && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
         && checkpidfile arv-git-httpd \
+        && checkhealth arv-git-httpd \
         && python sdk/python/tests/run_test_server.py start_ws \
         && checkpidfile ws \
         && eval $(python sdk/python/tests/run_test_server.py start_nginx) \
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 53125ae554..775d290347 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -72,6 +72,7 @@ func (h *Handler) setup() {
 	mux.Handle("/_health/", &health.Handler{
 		Token:  h.Cluster.ManagementToken,
 		Prefix: "/_health/",
+		Routes: health.Routes{"ping": func() error { _, err := h.db(&http.Request{}); return err }},
 	})
 	hs := http.NotFoundHandler()
 	hs = prepend(hs, h.proxyRailsAPI)
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 4607365c55..79767c2fa5 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -413,6 +413,7 @@ def run_controller():
         f.write("""
 Clusters:
   zzzzz:
+    ManagementToken: e687950a23c3a9bceec28c6223a06c79
     HTTPRequestTimeout: 30s
     PostgreSQL:
       ConnectionPool: 32
@@ -632,6 +633,7 @@ def run_arv_git_httpd():
     agh = subprocess.Popen(
         ['arv-git-httpd',
          '-repo-root='+gitdir+'/test',
+         '-management-token=e687950a23c3a9bceec28c6223a06c79',
          '-address=:'+str(gitport)],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('arv-git-httpd'), 'w') as f:
@@ -657,6 +659,7 @@ def run_keep_web():
         ['keep-web',
          '-allow-anonymous',
          '-attachment-only-host=download',
+         '-management-token=e687950a23c3a9bceec28c6223a06c79',
          '-listen=:'+str(keepwebport)],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('keep-web'), 'w') as f: