18947: Merge branch 'main'

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --combined cmd/arvados-server/cmd.go
index 27c53eb5cb6393623f4e596e610006448e4d786a,ae1e3fbeee2d1e2d4310bb3f221b1f14bf77b17b..3a1fcd4c64e29b981ddb0234f1bf3eae6a14da7b
--- 1/cmd/arvados-server/cmd.go
--- 2/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@@ -5,7 -5,6 +5,7 @@@
  package main
  
  import (
 +      "context"
        "encoding/json"
        "fmt"
        "io"
@@@ -22,8 -21,6 +22,8 @@@
        "git.arvados.org/arvados.git/lib/install"
        "git.arvados.org/arvados.git/lib/lsf"
        "git.arvados.org/arvados.git/lib/recovercollection"
 +      "git.arvados.org/arvados.git/lib/service"
 +      "git.arvados.org/arvados.git/sdk/go/arvados"
        "git.arvados.org/arvados.git/sdk/go/health"
        "git.arvados.org/arvados.git/services/githttpd"
        keepbalance "git.arvados.org/arvados.git/services/keep-balance"
@@@ -31,7 -28,6 +31,7 @@@
        "git.arvados.org/arvados.git/services/keepproxy"
        "git.arvados.org/arvados.git/services/keepstore"
        "git.arvados.org/arvados.git/services/ws"
 +      "github.com/prometheus/client_golang/prometheus"
  )
  
  var (
@@@ -51,7 -47,6 +51,7 @@@
                "dispatch-cloud":     dispatchcloud.Command,
                "dispatch-lsf":       lsf.DispatchCommand,
                "git-httpd":          githttpd.Command,
 +              "health":             healthCommand,
                "install":            install.Command,
                "init":               install.InitCommand,
                "keep-balance":       keepbalance.Command,
@@@ -95,7 -90,3 +95,17 @@@ func (wb2command) RunCommand(prog strin
        }
        return 0
  }
- var healthCommand cmd.Handler = service.Command(arvados.ServiceNameHealth, func(ctx context.Context, cluster *arvados.Cluster, _ string, _ *prometheus.Registry) service.Handler {
-       return &health.Aggregator{Cluster: cluster}
 +
++var healthCommand cmd.Handler = service.Command(arvados.ServiceNameHealth, func(ctx context.Context, cluster *arvados.Cluster, _ string, reg *prometheus.Registry) service.Handler {
++      mClockSkew := prometheus.NewGauge(prometheus.GaugeOpts{
++              Namespace: "arvados",
++              Subsystem: "health",
++              Name:      "clock_skew_seconds",
++              Help:      "Clock skew observed in most recent health check",
++      })
++      reg.MustRegister(mClockSkew)
++      return &health.Aggregator{
++              Cluster:         cluster,
++              MetricClockSkew: mClockSkew,
++      }
 +})