X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b16f84a8c5562cd5c3939e8c445dc86df2be2d4b..e3f324c979fe45803e2bd42aa8f6d715ea23edb5:/services/keepstore/handlers.go

diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index 2a1bbc972f..9a4d02df85 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -24,6 +24,7 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/health"
 	"git.curoverse.com/arvados.git/sdk/go/httpserver"
 	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 type router struct {
@@ -31,14 +32,16 @@ type router struct {
 	limiter     httpserver.RequestCounter
 	cluster     *arvados.Cluster
 	remoteProxy remoteProxy
+	metrics     *nodeMetrics
 }
 
 // MakeRESTRouter returns a new router that forwards all Keep requests
 // to the appropriate handlers.
-func MakeRESTRouter(cluster *arvados.Cluster) http.Handler {
+func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Handler {
 	rtr := &router{
 		Router:  mux.NewRouter(),
 		cluster: cluster,
+		metrics: &nodeMetrics{reg: reg},
 	}
 
 	rtr.HandleFunc(
@@ -85,8 +88,12 @@ func MakeRESTRouter(cluster *arvados.Cluster) http.Handler {
 	rtr.NotFoundHandler = http.HandlerFunc(BadRequestHandler)
 
 	rtr.limiter = httpserver.NewRequestLimiter(theConfig.MaxRequests, rtr)
+	rtr.metrics.setupBufferPoolMetrics(bufs)
+	rtr.metrics.setupWorkQueueMetrics(pullq, "pull")
+	rtr.metrics.setupWorkQueueMetrics(trashq, "trash")
+	rtr.metrics.setupRequestMetrics(rtr.limiter)
 
-	instrumented := httpserver.Instrument(nil, nil,
+	instrumented := httpserver.Instrument(rtr.metrics.reg, nil,
 		httpserver.AddRequestIDs(httpserver.LogRequests(nil, rtr.limiter)))
 	return instrumented.ServeAPI(theConfig.ManagementToken, instrumented)
 }
@@ -270,13 +277,19 @@ func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
 
 	for _, v := range vols {
 		if err := v.IndexTo(prefix, resp); err != nil {
-			// The only errors returned by IndexTo are
-			// write errors returned by resp.Write(),
-			// which probably means the client has
-			// disconnected and this error will never be
-			// reported to the client -- but it will
-			// appear in our own error log.
-			http.Error(resp, err.Error(), http.StatusInternalServerError)
+			// We can't send an error message to the
+			// client because we might have already sent
+			// headers and index content. All we can do is
+			// log the error in our own logs, and (in
+			// cases where headers haven't been sent yet)
+			// set a 500 status.
+			//
+			// If headers have already been sent, the
+			// client must notice the lack of trailing
+			// newline as an indication that the response
+			// is incomplete.
+			log.Printf("index error from volume %s: %s", v, err)
+			http.Error(resp, "", http.StatusInternalServerError)
 			return
 		}
 	}