13198: Track & serve keep-web metrics.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/sdk/go/httpserver/metrics.go b/sdk/go/httpserver/metrics.go
index 1f7d44752bea34bb3cbf39092cb9c5b6d2707b54..77525a80f12ce269132cc932cd83e343b73d788b 100644 (file)
--- a/sdk/go/httpserver/metrics.go
+++ b/sdk/go/httpserver/metrics.go
@@ -19,6 +19,11 @@ import (
 
 type Handler interface {
        http.Handler
+
+       // Returns an http.Handler that serves the Handler's metrics
+       // data at /metrics and /metrics.json, and passes other
+       // requests through to next.
+       ServeAPI(next http.Handler) http.Handler
 }
 
 type metrics struct {
@@ -34,6 +39,8 @@ func (*metrics) Levels() []logrus.Level {
        return logrus.AllLevels
 }
 
+// Fire implements logrus.Hook in order to collect data points from
+// request logs.
 func (m *metrics) Fire(ent *logrus.Entry) error {
        if tts, ok := ent.Data["timeToStatus"].(stats.Duration); !ok {
        } else if method, ok := ent.Data["reqMethod"].(string); !ok {
@@ -57,19 +64,41 @@ func (m *metrics) exportJSON(w http.ResponseWriter, req *http.Request) {
        w.Write([]byte{']'})
 }
 
+// ServeHTTP implements http.Handler.
 func (m *metrics) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-       switch {
-       case req.Method != "GET" && req.Method != "HEAD":
-               m.next.ServeHTTP(w, req)
-       case req.URL.Path == "/metrics.json":
-               m.exportJSON(w, req)
-       case req.URL.Path == "/metrics":
-               m.exportProm.ServeHTTP(w, req)
-       default:
-               m.next.ServeHTTP(w, req)
-       }
+       m.next.ServeHTTP(w, req)
+}
+
+// ServeAPI returns a new http.Handler that serves current data at
+// metrics API endpoints (currently "GET /metrics(.json)?") and passes
+// other requests through to next.
+//
+// Typical example:
+//
+//     m := Instrument(...)
+//     srv := http.Server{Handler: m.ServeAPI(m)}
+func (m *metrics) ServeAPI(next http.Handler) http.Handler {
+       return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+               switch {
+               case req.Method != "GET" && req.Method != "HEAD":
+                       next.ServeHTTP(w, req)
+               case req.URL.Path == "/metrics.json":
+                       m.exportJSON(w, req)
+               case req.URL.Path == "/metrics":
+                       m.exportProm.ServeHTTP(w, req)
+               default:
+                       next.ServeHTTP(w, req)
+               }
+       })
 }
 
+// Instrument returns a new Handler that passes requests through to
+// the next handler in the stack, and tracks metrics of those
+// requests.
+//
+// For the metrics to be accurate, the caller must ensure every
+// request passed to the Handler also passes through
+// LogRequests(logger, ...), and vice versa.
 func Instrument(logger *logrus.Logger, next http.Handler) Handler {
        if logger == nil {
                logger = logrus.StandardLogger()

diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 7d17be6e7cfe8c59305b452c8d788bca5748acdc..d0ba431aa6312d64f44e518d5ca19d8826ad1c5c 100644 (file)
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -31,6 +31,7 @@ import (
 
 type handler struct {
        Config        *Config
+       MetricsAPI    http.Handler
        clientPool    *arvadosclient.ClientPool
        setupOnce     sync.Once
        healthHandler http.Handler
@@ -259,6 +260,9 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
        } else if r.URL.Path == "/status.json" {
                h.serveStatus(w, r)
                return
+       } else if strings.HasPrefix(r.URL.Path, "/metrics") {
+               h.MetricsAPI.ServeHTTP(w, r)
+               return
        } else if siteFSDir[pathParts[0]] {
                useSiteFS = true
        } else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {

diff --git a/services/keep-web/server.go b/services/keep-web/server.go
index 269f5d36896a6b7ea8f1bf40bd23b547cd8acadb..58ec348c882b88e6c92116dca947bd24e62c5877 100644 (file)
--- a/services/keep-web/server.go
+++ b/services/keep-web/server.go
@@ -5,6 +5,8 @@
 package main
 
 import (
+       "net/http"
+
        "git.curoverse.com/arvados.git/sdk/go/httpserver"
 )
 
@@ -14,7 +16,10 @@ type server struct {
 }
 
 func (srv *server) Start() error {
-       srv.Handler = httpserver.Instrument(nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, &handler{Config: srv.Config})))
+       h := &handler{Config: srv.Config}
+       mh := httpserver.Instrument(nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, h)))
+       h.MetricsAPI = mh.ServeAPI(http.NotFoundHandler())
+       srv.Handler = mh
        srv.Addr = srv.Config.Listen
        return srv.Server.Start()
 }

diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 63a84289c31b9bba94bfaa1215489320c387dd3f..6688cc2ee743ec53bf4f2ce15fdfcb4621f09253 100644 (file)
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -360,6 +360,7 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
        // must not intercept that route.
        req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
        req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        resp, err = http.DefaultClient.Do(req)
        c.Assert(err, check.IsNil)
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)

diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index 847486fbd74b75d679377ab056de7339f1ca7215..d19be61e9ade77958b376450169f5b2abdafc66b 100644 (file)
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -88,8 +88,9 @@ func MakeRESTRouter() http.Handler {
 
        rtr.limiter = httpserver.NewRequestLimiter(theConfig.MaxRequests, rtr)
 
-       return httpserver.Instrument(nil,
+       stack := httpserver.Instrument(nil,
                httpserver.AddRequestIDs(httpserver.LogRequests(nil, rtr.limiter)))
+       return stack.ServeAPI(stack)
 }
 
 // BadRequestHandler is a HandleFunc to address bad requests.