From: Tom Clegg <tom@curii.com>
Date: Fri, 8 Apr 2022 21:48:21 +0000 (-0400)
Subject: 18794: Export config load time as prometheus metric.
X-Git-Tag: 2.5.0~184^2~22
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/8b6165d62d4e8354f31fea86dcd167b57650cf08

18794: Export config load time as prometheus metric.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>
---

diff --git a/lib/config/load.go b/lib/config/load.go
index d03ef5388d..1ad31402c3 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -53,6 +53,8 @@ type Loader struct {
 	// read configdata from, or the time when we read configdata
 	// from a pipe.
 	sourceTimestamp time.Time
+	// UTC time when configdata was read.
+	loadTimestamp time.Time
 }
 
 // NewLoader returns a new Loader with Stdin and Logger set to the
@@ -173,32 +175,36 @@ func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string,
 	return munged
 }
 
-func (ldr *Loader) loadBytes(path string) ([]byte, time.Time, error) {
+func (ldr *Loader) loadBytes(path string) (buf []byte, sourceTime, loadTime time.Time, err error) {
+	loadTime = time.Now().UTC()
 	if path == "-" {
-		buf, err := ioutil.ReadAll(ldr.Stdin)
-		return buf, time.Now().UTC(), err
+		buf, err = ioutil.ReadAll(ldr.Stdin)
+		sourceTime = loadTime
+		return
 	}
 	f, err := os.Open(path)
 	if err != nil {
-		return nil, time.Time{}, err
+		return
 	}
 	defer f.Close()
 	fi, err := f.Stat()
 	if err != nil {
-		return nil, time.Time{}, err
+		return
 	}
-	buf, err := ioutil.ReadAll(f)
-	return buf, fi.ModTime().UTC(), err
+	sourceTime = fi.ModTime().UTC()
+	buf, err = ioutil.ReadAll(f)
+	return
 }
 
 func (ldr *Loader) Load() (*arvados.Config, error) {
 	if ldr.configdata == nil {
-		buf, t, err := ldr.loadBytes(ldr.Path)
+		buf, sourceTime, loadTime, err := ldr.loadBytes(ldr.Path)
 		if err != nil {
 			return nil, err
 		}
 		ldr.configdata = buf
-		ldr.sourceTimestamp = t
+		ldr.sourceTimestamp = sourceTime
+		ldr.loadTimestamp = loadTime
 	}
 
 	// FIXME: We should reject YAML if the same key is used twice
@@ -579,12 +585,22 @@ func (ldr *Loader) autofillPreemptible(label string, cc *arvados.Cluster) {
 // called before Load(). Metrics are not updated by subsequent calls
 // to Load().
 func (ldr *Loader) RegisterMetrics(reg *prometheus.Registry) {
+	hash := fmt.Sprintf("%x", sha256.Sum256(ldr.configdata))
 	vec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: "arvados",
 		Subsystem: "config",
 		Name:      "source_timestamp_seconds",
 		Help:      "Timestamp of config file when it was loaded.",
 	}, []string{"sha256"})
-	vec.WithLabelValues(fmt.Sprintf("%x", sha256.Sum256(ldr.configdata))).Set(float64(ldr.sourceTimestamp.UnixNano()) / 1e9)
+	vec.WithLabelValues(hash).Set(float64(ldr.sourceTimestamp.UnixNano()) / 1e9)
+	reg.MustRegister(vec)
+
+	vec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "arvados",
+		Subsystem: "config",
+		Name:      "load_timestamp_seconds",
+		Help:      "Time when config file was loaded.",
+	}, []string{"sha256"})
+	vec.WithLabelValues(hash).Set(float64(ldr.loadTimestamp.UnixNano()) / 1e9)
 	reg.MustRegister(vec)
 }
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index b43cda38c8..abf3217056 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -799,7 +799,9 @@ func (s *LoadSuite) TestSourceTimestamp(c *check.C) {
 		cfg, err := ldr.Load()
 		c.Assert(err, check.IsNil)
 		c.Check(cfg.SourceTimestamp, check.Equals, cfg.SourceTimestamp.UTC())
+		c.Check(cfg.SourceTimestamp, check.Equals, ldr.sourceTimestamp)
 		c.Check(int(cfg.SourceTimestamp.Sub(trial.expectTime).Seconds()), check.Equals, 0)
+		c.Check(int(ldr.loadTimestamp.Sub(time.Now()).Seconds()), check.Equals, 0)
 
 		var buf bytes.Buffer
 		reg := prometheus.NewRegistry()
@@ -809,6 +811,12 @@ func (s *LoadSuite) TestSourceTimestamp(c *check.C) {
 		for _, mf := range got {
 			enc.Encode(mf)
 		}
-		c.Check(buf.String(), check.Matches, `# HELP .*\n# TYPE .*\narvados_config_source_timestamp_seconds{sha256="83aea5d82eb1d53372cd65c936c60acc1c6ef946e61977bbca7cfea709d201a8"} \Q`+fmt.Sprintf("%g", float64(cfg.SourceTimestamp.UnixNano())/1e9)+`\E\n`)
+		c.Check(buf.String(), check.Matches, `# HELP .*
+# TYPE .*
+arvados_config_load_timestamp_seconds{sha256="83aea5d82eb1d53372cd65c936c60acc1c6ef946e61977bbca7cfea709d201a8"} \Q`+fmt.Sprintf("%g", float64(ldr.loadTimestamp.UnixNano())/1e9)+`\E
+# HELP .*
+# TYPE .*
+arvados_config_source_timestamp_seconds{sha256="83aea5d82eb1d53372cd65c936c60acc1c6ef946e61977bbca7cfea709d201a8"} \Q`+fmt.Sprintf("%g", float64(cfg.SourceTimestamp.UnixNano())/1e9)+`\E
+`)
 	}
 }