13100: Handle writable collections mounted below output dir.

[arvados.git] / services / keepstore / config.go

diff --git a/services/keepstore/config.go b/services/keepstore/config.go
index 9c318d1245abcb285634ed46eaf17ddcabe300b2..17d6acdb68cca7463b9a7e9e49b9e1d9f3510229 100644 (file)
--- a/services/keepstore/config.go
+++ b/services/keepstore/config.go
@@ -1,3 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
@@ -5,16 +9,25 @@ import (
        "encoding/json"
        "fmt"
        "io/ioutil"
-       "log"
+       "net/http"
+       "strconv"
        "strings"
        "time"
 
        "git.curoverse.com/arvados.git/sdk/go/arvados"
+       "git.curoverse.com/arvados.git/sdk/go/stats"
+       "github.com/Sirupsen/logrus"
+       "github.com/golang/protobuf/jsonpb"
+       "github.com/prometheus/client_golang/prometheus"
+       "github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 type Config struct {
+       Debug  bool
        Listen string
 
+       LogFormat string
+
        PIDFile string
 
        MaxBuffers  int
@@ -32,14 +45,34 @@ type Config struct {
 
        blobSigningKey  []byte
        systemAuthToken string
+       debugLogf       func(string, ...interface{})
+
+       ManagementToken string
+
+       metrics
 }
 
-var theConfig = DefaultConfig()
+var (
+       theConfig = DefaultConfig()
+       formatter = map[string]logrus.Formatter{
+               "text": &logrus.TextFormatter{
+                       FullTimestamp:   true,
+                       TimestampFormat: rfc3339NanoFixed,
+               },
+               "json": &logrus.JSONFormatter{
+                       TimestampFormat: rfc3339NanoFixed,
+               },
+       }
+       log = logrus.StandardLogger()
+)
+
+const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
 
 // DefaultConfig returns the default configuration.
 func DefaultConfig() *Config {
        return &Config{
                Listen:             ":25107",
+               LogFormat:          "json",
                MaxBuffers:         128,
                RequireSignatures:  true,
                BlobSignatureTTL:   arvados.Duration(14 * 24 * time.Hour),
@@ -52,6 +85,21 @@ func DefaultConfig() *Config {
 // Start should be called exactly once: after setting all public
 // fields, and before using the config.
 func (cfg *Config) Start() error {
+       if cfg.Debug {
+               log.Level = logrus.DebugLevel
+               cfg.debugLogf = log.Printf
+               cfg.debugLogf("debugging enabled")
+       } else {
+               log.Level = logrus.InfoLevel
+               cfg.debugLogf = func(string, ...interface{}) {}
+       }
+
+       f := formatter[strings.ToLower(cfg.LogFormat)]
+       if f == nil {
+               return fmt.Errorf(`unsupported log format %q (try "text" or "json")`, cfg.LogFormat)
+       }
+       log.Formatter = f
+
        if cfg.MaxBuffers < 0 {
                return fmt.Errorf("MaxBuffers must be greater than zero")
        }
@@ -107,15 +155,71 @@ func (cfg *Config) Start() error {
        return nil
 }
 
+type metrics struct {
+       registry     *prometheus.Registry
+       reqDuration  *prometheus.SummaryVec
+       timeToStatus *prometheus.SummaryVec
+       exportProm   http.Handler
+}
+
+func (*metrics) Levels() []logrus.Level {
+       return logrus.AllLevels
+}
+
+func (m *metrics) Fire(ent *logrus.Entry) error {
+       if tts, ok := ent.Data["timeToStatus"].(stats.Duration); !ok {
+       } else if method, ok := ent.Data["reqMethod"].(string); !ok {
+       } else if code, ok := ent.Data["respStatusCode"].(int); !ok {
+       } else {
+               m.timeToStatus.WithLabelValues(strconv.Itoa(code), strings.ToLower(method)).Observe(time.Duration(tts).Seconds())
+       }
+       return nil
+}
+
+func (m *metrics) setup() {
+       m.registry = prometheus.NewRegistry()
+       m.timeToStatus = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+               Name: "time_to_status_seconds",
+               Help: "Summary of request TTFB.",
+       }, []string{"code", "method"})
+       m.reqDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+               Name: "request_duration_seconds",
+               Help: "Summary of request duration.",
+       }, []string{"code", "method"})
+       m.registry.MustRegister(m.timeToStatus)
+       m.registry.MustRegister(m.reqDuration)
+       m.exportProm = promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{
+               ErrorLog: log,
+       })
+       log.AddHook(m)
+}
+
+func (m *metrics) exportJSON(w http.ResponseWriter, req *http.Request) {
+       jm := jsonpb.Marshaler{Indent: "  "}
+       mfs, _ := m.registry.Gather()
+       w.Write([]byte{'['})
+       for i, mf := range mfs {
+               if i > 0 {
+                       w.Write([]byte{','})
+               }
+               jm.Marshal(w, mf)
+       }
+       w.Write([]byte{']'})
+}
+
+func (m *metrics) Instrument(next http.Handler) http.Handler {
+       return promhttp.InstrumentHandlerDuration(m.reqDuration, next)
+}
+
 // VolumeTypes is built up by init() funcs in the source files that
 // define the volume types.
 var VolumeTypes = []func() VolumeWithExamples{}
 
 type VolumeList []Volume
 
-// UnmarshalJSON, given an array of objects, deserializes each object
-// as the volume type indicated by the object's Type field.
-func (vols *VolumeList) UnmarshalJSON(data []byte) error {
+// UnmarshalJSON -- given an array of objects -- deserializes each
+// object as the volume type indicated by the object's Type field.
+func (vl *VolumeList) UnmarshalJSON(data []byte) error {
        typeMap := map[string]func() VolumeWithExamples{}
        for _, factory := range VolumeTypes {
                t := factory().Type()
@@ -148,7 +252,7 @@ func (vols *VolumeList) UnmarshalJSON(data []byte) error {
                if err != nil {
                        return err
                }
-               *vols = append(*vols, vol)
+               *vl = append(*vl, vol)
        }
        return nil
 }