sdk/go/health/aggregator_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package health
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "io/ioutil"
  11         "net/http"
  12         "net/http/httptest"
  13         "strings"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/lib/config"
  17         "git.arvados.org/arvados.git/sdk/go/arvados"
  18         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  19         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  20         "github.com/ghodss/yaml"
  21         "gopkg.in/check.v1"
  22 )
  23
  24 type AggregatorSuite struct {
  25         handler *Aggregator
  26         req     *http.Request
  27         resp    *httptest.ResponseRecorder
  28 }
  29
  30 // Gocheck boilerplate
  31 var _ = check.Suite(&AggregatorSuite{})
  32
  33 func (s *AggregatorSuite) TestInterface(c *check.C) {
  34         var _ http.Handler = &Aggregator{}
  35 }
  36
  37 func (s *AggregatorSuite) SetUpTest(c *check.C) {
  38         ldr := config.NewLoader(bytes.NewBufferString(`Clusters: {zzzzz: {}}`), ctxlog.TestLogger(c))
  39         ldr.Path = "-"
  40         cfg, err := ldr.Load()
  41         c.Assert(err, check.IsNil)
  42         cluster, err := cfg.GetCluster("")
  43         c.Assert(err, check.IsNil)
  44         cluster.ManagementToken = arvadostest.ManagementToken
  45         cluster.SystemRootToken = arvadostest.SystemRootToken
  46         cluster.Collections.BlobSigningKey = arvadostest.BlobSigningKey
  47         cluster.Volumes["z"] = arvados.Volume{StorageClasses: map[string]bool{"default": true}}
  48         s.handler = &Aggregator{Cluster: cluster}
  49         s.req = httptest.NewRequest("GET", "/_health/all", nil)
  50         s.req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
  51         s.resp = httptest.NewRecorder()
  52 }
  53
  54 func (s *AggregatorSuite) TestNoAuth(c *check.C) {
  55         s.req.Header.Del("Authorization")
  56         s.handler.ServeHTTP(s.resp, s.req)
  57         s.checkError(c)
  58         c.Check(s.resp.Code, check.Equals, http.StatusUnauthorized)
  59 }
  60
  61 func (s *AggregatorSuite) TestBadAuth(c *check.C) {
  62         s.req.Header.Set("Authorization", "xyzzy")
  63         s.handler.ServeHTTP(s.resp, s.req)
  64         s.checkError(c)
  65         c.Check(s.resp.Code, check.Equals, http.StatusUnauthorized)
  66 }
  67
  68 func (s *AggregatorSuite) TestNoServicesConfigured(c *check.C) {
  69         s.handler.ServeHTTP(s.resp, s.req)
  70         s.checkUnhealthy(c)
  71 }
  72
  73 func (s *AggregatorSuite) stubServer(handler http.Handler) (*httptest.Server, string) {
  74         srv := httptest.NewServer(handler)
  75         var port string
  76         if parts := strings.Split(srv.URL, ":"); len(parts) < 3 {
  77                 panic(srv.URL)
  78         } else {
  79                 port = parts[len(parts)-1]
  80         }
  81         return srv, ":" + port
  82 }
  83
  84 func (s *AggregatorSuite) TestUnhealthy(c *check.C) {
  85         srv, listen := s.stubServer(&unhealthyHandler{})
  86         defer srv.Close()
  87         arvadostest.SetServiceURL(&s.handler.Cluster.Services.Keepstore, "http://localhost"+listen+"/")
  88         s.handler.ServeHTTP(s.resp, s.req)
  89         s.checkUnhealthy(c)
  90 }
  91
  92 func (s *AggregatorSuite) TestHealthy(c *check.C) {
  93         srv, listen := s.stubServer(&healthyHandler{})
  94         defer srv.Close()
  95         s.setAllServiceURLs(listen)
  96         s.handler.ServeHTTP(s.resp, s.req)
  97         resp := s.checkOK(c)
  98         svc := "keepstore+http://localhost" + listen + "/_health/ping"
  99         c.Logf("%#v", resp)
 100         ep := resp.Checks[svc]
 101         c.Check(ep.Health, check.Equals, "OK")
 102         c.Check(ep.HTTPStatusCode, check.Equals, 200)
 103 }
 104
 105 func (s *AggregatorSuite) TestHealthyAndUnhealthy(c *check.C) {
 106         srvH, listenH := s.stubServer(&healthyHandler{})
 107         defer srvH.Close()
 108         srvU, listenU := s.stubServer(&unhealthyHandler{})
 109         defer srvU.Close()
 110         s.setAllServiceURLs(listenH)
 111         arvadostest.SetServiceURL(&s.handler.Cluster.Services.Keepstore, "http://localhost"+listenH+"/", "http://127.0.0.1"+listenU+"/")
 112         s.handler.ServeHTTP(s.resp, s.req)
 113         resp := s.checkUnhealthy(c)
 114         ep := resp.Checks["keepstore+http://localhost"+listenH+"/_health/ping"]
 115         c.Check(ep.Health, check.Equals, "OK")
 116         c.Check(ep.HTTPStatusCode, check.Equals, 200)
 117         ep = resp.Checks["keepstore+http://127.0.0.1"+listenU+"/_health/ping"]
 118         c.Check(ep.Health, check.Equals, "ERROR")
 119         c.Check(ep.HTTPStatusCode, check.Equals, 200)
 120         c.Logf("%#v", ep)
 121 }
 122
 123 func (s *AggregatorSuite) TestPingTimeout(c *check.C) {
 124         s.handler.timeout = arvados.Duration(100 * time.Millisecond)
 125         srv, listen := s.stubServer(&slowHandler{})
 126         defer srv.Close()
 127         arvadostest.SetServiceURL(&s.handler.Cluster.Services.Keepstore, "http://localhost"+listen+"/")
 128         s.handler.ServeHTTP(s.resp, s.req)
 129         resp := s.checkUnhealthy(c)
 130         ep := resp.Checks["keepstore+http://localhost"+listen+"/_health/ping"]
 131         c.Check(ep.Health, check.Equals, "ERROR")
 132         c.Check(ep.HTTPStatusCode, check.Equals, 0)
 133         rt, err := ep.ResponseTime.Float64()
 134         c.Check(err, check.IsNil)
 135         c.Check(rt > 0.005, check.Equals, true)
 136 }
 137
 138 func (s *AggregatorSuite) TestCheckCommand(c *check.C) {
 139         srv, listen := s.stubServer(&healthyHandler{})
 140         defer srv.Close()
 141         s.setAllServiceURLs(listen)
 142         tmpdir := c.MkDir()
 143         confdata, err := yaml.Marshal(arvados.Config{Clusters: map[string]arvados.Cluster{s.handler.Cluster.ClusterID: *s.handler.Cluster}})
 144         c.Assert(err, check.IsNil)
 145         err = ioutil.WriteFile(tmpdir+"/config.yml", confdata, 0777)
 146         c.Assert(err, check.IsNil)
 147         var stdout, stderr bytes.Buffer
 148         exitcode := CheckCommand.RunCommand("check", []string{"-config=" + tmpdir + "/config.yml"}, &bytes.Buffer{}, &stdout, &stderr)
 149         c.Check(exitcode, check.Equals, 0)
 150         c.Check(stderr.String(), check.Equals, "")
 151         c.Check(stdout.String(), check.Matches, `(?ms).*(\n|^)health: OK\n.*`)
 152 }
 153
 154 func (s *AggregatorSuite) checkError(c *check.C) {
 155         c.Check(s.resp.Code, check.Not(check.Equals), http.StatusOK)
 156         var resp ClusterHealthResponse
 157         err := json.Unmarshal(s.resp.Body.Bytes(), &resp)
 158         c.Check(err, check.IsNil)
 159         c.Check(resp.Health, check.Not(check.Equals), "OK")
 160 }
 161
 162 func (s *AggregatorSuite) checkUnhealthy(c *check.C) ClusterHealthResponse {
 163         return s.checkResult(c, "ERROR")
 164 }
 165
 166 func (s *AggregatorSuite) checkOK(c *check.C) ClusterHealthResponse {
 167         return s.checkResult(c, "OK")
 168 }
 169
 170 func (s *AggregatorSuite) checkResult(c *check.C, health string) ClusterHealthResponse {
 171         c.Check(s.resp.Code, check.Equals, http.StatusOK)
 172         var resp ClusterHealthResponse
 173         c.Log(s.resp.Body.String())
 174         err := json.Unmarshal(s.resp.Body.Bytes(), &resp)
 175         c.Check(err, check.IsNil)
 176         c.Check(resp.Health, check.Equals, health)
 177         return resp
 178 }
 179
 180 func (s *AggregatorSuite) setAllServiceURLs(listen string) {
 181         svcs := &s.handler.Cluster.Services
 182         for _, svc := range []*arvados.Service{
 183                 &svcs.Controller,
 184                 &svcs.DispatchCloud,
 185                 &svcs.DispatchLSF,
 186                 &svcs.Keepbalance,
 187                 &svcs.Keepproxy,
 188                 &svcs.Keepstore,
 189                 &svcs.Health,
 190                 &svcs.RailsAPI,
 191                 &svcs.WebDAV,
 192                 &svcs.Websocket,
 193                 &svcs.Workbench1,
 194                 &svcs.Workbench2,
 195         } {
 196                 arvadostest.SetServiceURL(svc, "http://localhost"+listen+"/")
 197         }
 198 }
 199
 200 type unhealthyHandler struct{}
 201
 202 func (*unhealthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 203         if req.URL.Path == "/_health/ping" {
 204                 resp.Write([]byte(`{"health":"ERROR","error":"the bends"}`))
 205         } else {
 206                 http.Error(resp, "not found", http.StatusNotFound)
 207         }
 208 }
 209
 210 type healthyHandler struct{}
 211
 212 func (*healthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 213         if req.URL.Path == "/_health/ping" {
 214                 resp.Write([]byte(`{"health":"OK"}`))
 215         } else {
 216                 http.Error(resp, "not found", http.StatusNotFound)
 217         }
 218 }
 219
 220 type slowHandler struct{}
 221
 222 func (*slowHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 223         if req.URL.Path == "/_health/ping" {
 224                 time.Sleep(3 * time.Second)
 225                 resp.Write([]byte(`{"health":"OK"}`))
 226         } else {
 227                 http.Error(resp, "not found", http.StatusNotFound)
 228         }
 229 }