15003: Check DB conn in controller health check. Check test services.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 9690cbbb938f854ce5ca4e251c5d413f471bcb0d..a33dab7c32048d86dfec5fe7c8b08b76d83964ad 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -379,6 +379,20 @@ checkpidfile() {
     echo "${svc} pid ${pid} ok"
 }
 
+checkhealth() {
+    svc="$1"
+    port="$(cat "$WORKSPACE/tmp/${svc}.port")"
+    scheme=http
+    if [[ ${svc} =~ -ssl$ || ${svc} = wss ]]; then
+        scheme=https
+    fi
+    url="$scheme://localhost:${port}/_health/ping"
+    if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then
+        echo "${url} failed"
+        return 1
+    fi
+}
+
 checkdiscoverydoc() {
     dd="https://${1}/discovery/v1/apis/arvados/v1/rest"
     if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then
@@ -412,12 +426,15 @@ start_services() {
         && checkdiscoverydoc $ARVADOS_API_HOST \
         && python sdk/python/tests/run_test_server.py start_controller \
         && checkpidfile controller \
+        && checkhealth controller \
         && python sdk/python/tests/run_test_server.py start_keep_proxy \
         && checkpidfile keepproxy \
         && python sdk/python/tests/run_test_server.py start_keep-web \
         && checkpidfile keep-web \
+        && checkhealth keep-web \
         && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
         && checkpidfile arv-git-httpd \
+        && checkhealth arv-git-httpd \
         && python sdk/python/tests/run_test_server.py start_ws \
         && checkpidfile ws \
         && eval $(python sdk/python/tests/run_test_server.py start_nginx) \

diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 53125ae5543b51287e5de80a8b442f2002972a86..775d2903475d6ad83eb368b77191cf479065cb57 100644 (file)
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -72,6 +72,7 @@ func (h *Handler) setup() {
        mux.Handle("/_health/", &health.Handler{
                Token:  h.Cluster.ManagementToken,
                Prefix: "/_health/",
+               Routes: health.Routes{"ping": func() error { _, err := h.db(&http.Request{}); return err }},
        })
        hs := http.NotFoundHandler()
        hs = prepend(hs, h.proxyRailsAPI)

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 4607365c55243eb44e4eefb6535710dc3c383612..79767c2fa5b007f267ad99a72c5445a909862d05 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -413,6 +413,7 @@ def run_controller():
         f.write("""
 Clusters:
   zzzzz:
+    ManagementToken: e687950a23c3a9bceec28c6223a06c79
     HTTPRequestTimeout: 30s
     PostgreSQL:
       ConnectionPool: 32
@@ -632,6 +633,7 @@ def run_arv_git_httpd():
     agh = subprocess.Popen(
         ['arv-git-httpd',
          '-repo-root='+gitdir+'/test',
+         '-management-token=e687950a23c3a9bceec28c6223a06c79',
          '-address=:'+str(gitport)],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('arv-git-httpd'), 'w') as f:
@@ -657,6 +659,7 @@ def run_keep_web():
         ['keep-web',
          '-allow-anonymous',
          '-attachment-only-host=download',
+         '-management-token=e687950a23c3a9bceec28c6223a06c79',
          '-listen=:'+str(keepwebport)],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('keep-web'), 'w') as f: