20755: Report metrics for ec2 instances per subnet.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index b98e519d0a4a7852bf64db398ad984ec91373580..5e4df05f46f01c9c2305d968771282c3f34e2013 100644 (file)
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -113,6 +113,9 @@ type ec2InstanceSet struct {
        prices        map[priceKey][]cloud.InstancePrice
        pricesLock    sync.Mutex
        pricesUpdated map[priceKey]time.Time
+
+       mInstances      *prometheus.GaugeVec
+       mInstanceStarts *prometheus.CounterVec
 }
 
 func newEC2InstanceSet(config json.RawMessage, instanceSetID cloud.InstanceSetID, _ cloud.SharedResourceTags, logger logrus.FieldLogger, reg *prometheus.Registry) (prv cloud.InstanceSet, err error) {
@@ -142,6 +145,36 @@ func newEC2InstanceSet(config json.RawMessage, instanceSetID cloud.InstanceSetID
        if instanceSet.ec2config.EBSVolumeType == "" {
                instanceSet.ec2config.EBSVolumeType = "gp2"
        }
+
+       // Set up metrics
+       instanceSet.mInstances = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+               Namespace: "arvados",
+               Subsystem: "dispatchcloud",
+               Name:      "ec2_instances",
+               Help:      "Number of instances running",
+       }, []string{"subnet_id"})
+       instanceSet.mInstanceStarts = prometheus.NewCounterVec(prometheus.CounterOpts{
+               Namespace: "arvados",
+               Subsystem: "dispatchcloud",
+               Name:      "ec2_instance_starts_total",
+               Help:      "Number of attempts to start a new instance",
+       }, []string{"subnet_id", "success"})
+       // Initialize all of the series we'll be reporting.  Otherwise
+       // the {subnet=A, success=0} series doesn't appear in metrics
+       // at all until there's a failure in subnet A.
+       for _, subnet := range instanceSet.ec2config.SubnetID {
+               instanceSet.mInstanceStarts.WithLabelValues(subnet, "0").Add(0)
+               instanceSet.mInstanceStarts.WithLabelValues(subnet, "1").Add(0)
+       }
+       if len(instanceSet.ec2config.SubnetID) == 0 {
+               instanceSet.mInstanceStarts.WithLabelValues("", "0").Add(0)
+               instanceSet.mInstanceStarts.WithLabelValues("", "1").Add(0)
+       }
+       if reg != nil {
+               reg.MustRegister(instanceSet.mInstances)
+               reg.MustRegister(instanceSet.mInstanceStarts)
+       }
+
        return instanceSet, nil
 }
 
@@ -260,11 +293,14 @@ func (instanceSet *ec2InstanceSet) Create(
        currentSubnetIDIndex := int(atomic.LoadInt32(&instanceSet.currentSubnetIDIndex))
        for tryOffset := 0; ; tryOffset++ {
                tryIndex := 0
+               trySubnet := ""
                if len(subnets) > 0 {
                        tryIndex = (currentSubnetIDIndex + tryOffset) % len(subnets)
-                       rii.NetworkInterfaces[0].SubnetId = aws.String(subnets[tryIndex])
+                       trySubnet = subnets[tryIndex]
+                       rii.NetworkInterfaces[0].SubnetId = aws.String(trySubnet)
                }
                rsv, err = instanceSet.client.RunInstances(&rii)
+               instanceSet.mInstanceStarts.WithLabelValues(trySubnet, boolLabelValue[err == nil]).Add(1)
                if isErrorSubnetSpecific(err) &&
                        tryOffset < len(subnets)-1 {
                        instanceSet.logger.WithError(err).WithField("SubnetID", subnets[tryIndex]).
@@ -382,6 +418,24 @@ func (instanceSet *ec2InstanceSet) Instances(tags cloud.InstanceTags) (instances
                }
                instanceSet.updateSpotPrices(instances)
        }
+
+       // Count instances in each subnet, and report in metrics.
+       subnetInstances := map[string]int{"": 0}
+       for _, subnet := range instanceSet.ec2config.SubnetID {
+               subnetInstances[subnet] = 0
+       }
+       for _, inst := range instances {
+               subnet := inst.(*ec2Instance).instance.SubnetId
+               if subnet != nil {
+                       subnetInstances[*subnet]++
+               } else {
+                       subnetInstances[""]++
+               }
+       }
+       for subnet, count := range subnetInstances {
+               instanceSet.mInstances.WithLabelValues(subnet).Set(float64(count))
+       }
+
        return instances, err
 }
 
@@ -675,3 +729,5 @@ func wrapError(err error, throttleValue *atomic.Value) error {
        throttleValue.Store(time.Duration(0))
        return nil
 }
+
+var boolLabelValue = map[bool]string{false: "0", true: "1"}

diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index d8322e7af38ab145e9a23b4e01554dae44fc6760..6fde4bbbcafb9ba8e43f0ab55cbe9d141ecc2c79 100644 (file)
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -34,12 +34,14 @@ import (
        "git.arvados.org/arvados.git/lib/cloud"
        "git.arvados.org/arvados.git/lib/dispatchcloud/test"
        "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadostest"
        "git.arvados.org/arvados.git/sdk/go/config"
        "git.arvados.org/arvados.git/sdk/go/ctxlog"
        "github.com/aws/aws-sdk-go/aws"
        "github.com/aws/aws-sdk-go/aws/awserr"
        "github.com/aws/aws-sdk-go/service/ec2"
        "github.com/ghodss/yaml"
+       "github.com/prometheus/client_golang/prometheus"
        "github.com/sirupsen/logrus"
        check "gopkg.in/check.v1"
 )
@@ -210,7 +212,8 @@ func (err *ec2stubError) OrigErr() error  { return errors.New("stub OrigErr") }
 // Ensure ec2stubError satisfies the aws.Error interface
 var _ = awserr.Error(&ec2stubError{})
 
-func GetInstanceSet(c *check.C) (*ec2InstanceSet, cloud.ImageID, arvados.Cluster) {
+func GetInstanceSet(c *check.C, conf string) (*ec2InstanceSet, cloud.ImageID, arvados.Cluster, *prometheus.Registry) {
+       reg := prometheus.NewRegistry()
        cluster := arvados.Cluster{
                InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
                        "tiny": {
@@ -246,21 +249,19 @@ func GetInstanceSet(c *check.C) (*ec2InstanceSet, cloud.ImageID, arvados.Cluster
                err := config.LoadFile(&exampleCfg, *live)
                c.Assert(err, check.IsNil)
 
-               ap, err := newEC2InstanceSet(exampleCfg.DriverParameters, "test123", nil, logrus.StandardLogger(), nil)
+               is, err := newEC2InstanceSet(exampleCfg.DriverParameters, "test123", nil, logrus.StandardLogger(), reg)
                c.Assert(err, check.IsNil)
-               return ap.(*ec2InstanceSet), cloud.ImageID(exampleCfg.ImageIDForTestSuite), cluster
-       }
-       ap := ec2InstanceSet{
-               instanceSetID: "test123",
-               logger:        ctxlog.TestLogger(c),
-               client:        &ec2stub{c: c, reftime: time.Now().UTC()},
-               keys:          make(map[string]string),
+               return is.(*ec2InstanceSet), cloud.ImageID(exampleCfg.ImageIDForTestSuite), cluster, reg
+       } else {
+               is, err := newEC2InstanceSet(json.RawMessage(conf), "test123", nil, ctxlog.TestLogger(c), reg)
+               c.Assert(err, check.IsNil)
+               is.(*ec2InstanceSet).client = &ec2stub{c: c, reftime: time.Now().UTC()}
+               return is.(*ec2InstanceSet), cloud.ImageID("blob"), cluster, reg
        }
-       return &ap, cloud.ImageID("blob"), cluster
 }
 
 func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
-       ap, img, cluster := GetInstanceSet(c)
+       ap, img, cluster, _ := GetInstanceSet(c, "{}")
        pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
 
        inst, err := ap.Create(cluster.InstanceTypes["tiny"],
@@ -280,7 +281,7 @@ func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
 }
 
 func (*EC2InstanceSetSuite) TestCreateWithExtraScratch(c *check.C) {
-       ap, img, cluster := GetInstanceSet(c)
+       ap, img, cluster, _ := GetInstanceSet(c, "{}")
        inst, err := ap.Create(cluster.InstanceTypes["tiny-with-extra-scratch"],
                img, map[string]string{
                        "TestTagName": "test tag value",
@@ -301,7 +302,7 @@ func (*EC2InstanceSetSuite) TestCreateWithExtraScratch(c *check.C) {
 }
 
 func (*EC2InstanceSetSuite) TestCreatePreemptible(c *check.C) {
-       ap, img, cluster := GetInstanceSet(c)
+       ap, img, cluster, _ := GetInstanceSet(c, "{}")
        pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
 
        inst, err := ap.Create(cluster.InstanceTypes["tiny-preemptible"],
@@ -323,8 +324,7 @@ func (*EC2InstanceSetSuite) TestCreateFailoverSecondSubnet(c *check.C) {
                return
        }
 
-       ap, img, cluster := GetInstanceSet(c)
-       ap.ec2config.SubnetID = sliceOrSingleString{"subnet-full", "subnet-good"}
+       ap, img, cluster, reg := GetInstanceSet(c, `{"SubnetID":["subnet-full","subnet-good"]}`)
        ap.client.(*ec2stub).subnetErrorOnRunInstances = map[string]error{
                "subnet-full": &ec2stubError{
                        code:    "InsufficientFreeAddressesInSubnet",
@@ -335,12 +335,26 @@ func (*EC2InstanceSetSuite) TestCreateFailoverSecondSubnet(c *check.C) {
        c.Check(err, check.IsNil)
        c.Check(inst, check.NotNil)
        c.Check(ap.client.(*ec2stub).runInstancesCalls, check.HasLen, 2)
+       metrics := arvadostest.GatherMetricsAsString(reg)
+       c.Check(metrics, check.Matches, `(?ms).*`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="0"} 1\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="1"} 0\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-good",success="0"} 0\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-good",success="1"} 1\n`+
+               `.*`)
 
        // Next RunInstances call should try the working subnet first
        inst, err = ap.Create(cluster.InstanceTypes["tiny"], img, nil, "", nil)
        c.Check(err, check.IsNil)
        c.Check(inst, check.NotNil)
        c.Check(ap.client.(*ec2stub).runInstancesCalls, check.HasLen, 3)
+       metrics = arvadostest.GatherMetricsAsString(reg)
+       c.Check(metrics, check.Matches, `(?ms).*`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="0"} 1\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="1"} 0\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-good",success="0"} 0\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-good",success="1"} 2\n`+
+               `.*`)
 }
 
 func (*EC2InstanceSetSuite) TestCreateAllSubnetsFailing(c *check.C) {
@@ -349,8 +363,7 @@ func (*EC2InstanceSetSuite) TestCreateAllSubnetsFailing(c *check.C) {
                return
        }
 
-       ap, img, cluster := GetInstanceSet(c)
-       ap.ec2config.SubnetID = sliceOrSingleString{"subnet-full", "subnet-broken"}
+       ap, img, cluster, reg := GetInstanceSet(c, `{"SubnetID":["subnet-full","subnet-broken"]}`)
        ap.client.(*ec2stub).subnetErrorOnRunInstances = map[string]error{
                "subnet-full": &ec2stubError{
                        code:    "InsufficientFreeAddressesInSubnet",
@@ -365,15 +378,29 @@ func (*EC2InstanceSetSuite) TestCreateAllSubnetsFailing(c *check.C) {
        c.Check(err, check.NotNil)
        c.Check(err, check.ErrorMatches, `.*InvalidSubnetId\.NotFound.*`)
        c.Check(ap.client.(*ec2stub).runInstancesCalls, check.HasLen, 2)
+       metrics := arvadostest.GatherMetricsAsString(reg)
+       c.Check(metrics, check.Matches, `(?ms).*`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-broken",success="0"} 1\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-broken",success="1"} 0\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="0"} 1\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="1"} 0\n`+
+               `.*`)
 
        _, err = ap.Create(cluster.InstanceTypes["tiny"], img, nil, "", nil)
        c.Check(err, check.NotNil)
        c.Check(err, check.ErrorMatches, `.*InsufficientFreeAddressesInSubnet.*`)
        c.Check(ap.client.(*ec2stub).runInstancesCalls, check.HasLen, 4)
+       metrics = arvadostest.GatherMetricsAsString(reg)
+       c.Check(metrics, check.Matches, `(?ms).*`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-broken",success="0"} 2\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-broken",success="1"} 0\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="0"} 2\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="1"} 0\n`+
+               `.*`)
 }
 
 func (*EC2InstanceSetSuite) TestTagInstances(c *check.C) {
-       ap, _, _ := GetInstanceSet(c)
+       ap, _, _, _ := GetInstanceSet(c, "{}")
        l, err := ap.Instances(nil)
        c.Assert(err, check.IsNil)
 
@@ -385,7 +412,7 @@ func (*EC2InstanceSetSuite) TestTagInstances(c *check.C) {
 }
 
 func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
-       ap, _, _ := GetInstanceSet(c)
+       ap, _, _, reg := GetInstanceSet(c, "{}")
        l, err := ap.Instances(nil)
        c.Assert(err, check.IsNil)
 
@@ -393,10 +420,15 @@ func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
                tg := i.Tags()
                c.Logf("%v %v %v", i.String(), i.Address(), tg)
        }
+
+       metrics := arvadostest.GatherMetricsAsString(reg)
+       c.Check(metrics, check.Matches, `(?ms).*`+
+               `arvados_dispatchcloud_ec2_instances{subnet_id="[^"]*"} \d+\n`+
+               `.*`)
 }
 
 func (*EC2InstanceSetSuite) TestDestroyInstances(c *check.C) {
-       ap, _, _ := GetInstanceSet(c)
+       ap, _, _, _ := GetInstanceSet(c, "{}")
        l, err := ap.Instances(nil)
        c.Assert(err, check.IsNil)
 
@@ -406,7 +438,7 @@ func (*EC2InstanceSetSuite) TestDestroyInstances(c *check.C) {
 }
 
 func (*EC2InstanceSetSuite) TestInstancePriceHistory(c *check.C) {
-       ap, img, cluster := GetInstanceSet(c)
+       ap, img, cluster, _ := GetInstanceSet(c, "{}")
        pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
        tags := cloud.InstanceTags{"arvados-ec2-driver": "test"}
 

diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 9a041790864d6c22b1ac7eadf79e8c6dbe9325fa..75efc6a35aa67f103a33fbdd4eee2751df1da294 100644 (file)
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -19,10 +19,10 @@ import (
        "time"
 
        "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadostest"
        "git.arvados.org/arvados.git/sdk/go/ctxlog"
        "github.com/ghodss/yaml"
        "github.com/prometheus/client_golang/prometheus"
-       "github.com/prometheus/common/expfmt"
        "github.com/sirupsen/logrus"
        "golang.org/x/sys/unix"
        check "gopkg.in/check.v1"
@@ -882,15 +882,10 @@ func (s *LoadSuite) TestSourceTimestamp(c *check.C) {
                c.Check(int(cfg.SourceTimestamp.Sub(trial.expectTime).Seconds()), check.Equals, 0)
                c.Check(int(ldr.loadTimestamp.Sub(time.Now()).Seconds()), check.Equals, 0)
 
-               var buf bytes.Buffer
                reg := prometheus.NewRegistry()
                ldr.RegisterMetrics(reg)
-               enc := expfmt.NewEncoder(&buf, expfmt.FmtText)
-               got, _ := reg.Gather()
-               for _, mf := range got {
-                       enc.Encode(mf)
-               }
-               c.Check(buf.String(), check.Matches, `# HELP .*
+               metrics := arvadostest.GatherMetricsAsString(reg)
+               c.Check(metrics, check.Matches, `# HELP .*
 # TYPE .*
 arvados_config_load_timestamp_seconds{sha256="83aea5d82eb1d53372cd65c936c60acc1c6ef946e61977bbca7cfea709d201a8"} \Q`+fmt.Sprintf("%g", float64(ldr.loadTimestamp.UnixNano())/1e9)+`\E
 # HELP .*

diff --git a/sdk/go/arvadostest/metrics.go b/sdk/go/arvadostest/metrics.go
new file mode 100644 (file)
index 0000000..5fe1d60
--- /dev/null
+++ b/sdk/go/arvadostest/metrics.go
@@ -0,0 +1,22 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvadostest
+
+import (
+       "bytes"
+
+       "github.com/prometheus/client_golang/prometheus"
+       "github.com/prometheus/common/expfmt"
+)
+
+func GatherMetricsAsString(reg *prometheus.Registry) string {
+       buf := bytes.NewBuffer(nil)
+       enc := expfmt.NewEncoder(buf, expfmt.FmtText)
+       got, _ := reg.Gather()
+       for _, mf := range got {
+               enc.Encode(mf)
+       }
+       return buf.String()
+}

diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index aeed517d094e93bf2b67a4a65a6cae0396847874..db9a668481105e7b16940dcebcc25761b153fc89 100644 (file)
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -5,7 +5,6 @@
 package keepbalance
 
 import (
-       "bytes"
        "context"
        "encoding/json"
        "fmt"
@@ -24,7 +23,6 @@ import (
        "git.arvados.org/arvados.git/sdk/go/ctxlog"
        "github.com/jmoiron/sqlx"
        "github.com/prometheus/client_golang/prometheus"
-       "github.com/prometheus/common/expfmt"
        check "gopkg.in/check.v1"
 )
 
@@ -591,14 +589,12 @@ func (s *runSuite) TestCommit(c *check.C) {
        c.Assert(err, check.IsNil)
        c.Check(string(lost), check.Not(check.Matches), `(?ms).*acbd18db4cc2f85cedef654fccc4a4d8.*`)
 
-       buf, err := s.getMetrics(c, srv)
-       c.Check(err, check.IsNil)
-       bufstr := buf.String()
-       c.Check(bufstr, check.Matches, `(?ms).*\narvados_keep_total_bytes 15\n.*`)
-       c.Check(bufstr, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_sum [0-9\.]+\n.*`)
-       c.Check(bufstr, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count 1\n.*`)
-       c.Check(bufstr, check.Matches, `(?ms).*\narvados_keep_dedup_byte_ratio [1-9].*`)
-       c.Check(bufstr, check.Matches, `(?ms).*\narvados_keep_dedup_block_ratio [1-9].*`)
+       metrics := arvadostest.GatherMetricsAsString(srv.Metrics.reg)
+       c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_total_bytes 15\n.*`)
+       c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_sum [0-9\.]+\n.*`)
+       c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count 1\n.*`)
+       c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_dedup_byte_ratio [1-9].*`)
+       c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_dedup_block_ratio [1-9].*`)
 }
 
 func (s *runSuite) TestChunkPrefix(c *check.C) {
@@ -674,23 +670,6 @@ func (s *runSuite) TestRunForever(c *check.C) {
        c.Check(pullReqs.Count() >= 16, check.Equals, true)
        c.Check(trashReqs.Count(), check.Equals, pullReqs.Count()+4)
 
-       buf, err := s.getMetrics(c, srv)
-       c.Check(err, check.IsNil)
-       c.Check(buf, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count `+fmt.Sprintf("%d", pullReqs.Count()/4)+`\n.*`)
-}
-
-func (s *runSuite) getMetrics(c *check.C, srv *Server) (*bytes.Buffer, error) {
-       mfs, err := srv.Metrics.reg.Gather()
-       if err != nil {
-               return nil, err
-       }
-
-       var buf bytes.Buffer
-       for _, mf := range mfs {
-               if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil {
-                       return nil, err
-               }
-       }
-
-       return &buf, nil
+       metrics := arvadostest.GatherMetricsAsString(srv.Metrics.reg)
+       c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count `+fmt.Sprintf("%d", pullReqs.Count()/4)+`\n.*`)
 }

diff --git a/services/keep-web/cache_test.go b/services/keep-web/cache_test.go
index 010e29a0b876105d49756d736adefd316878a12e..e95ebcf8467c1d385710834300109c9b8dd86ecc 100644 (file)
--- a/services/keep-web/cache_test.go
+++ b/services/keep-web/cache_test.go
@@ -5,7 +5,6 @@
 package keepweb
 
 import (
-       "bytes"
        "net/http"
        "net/http/httptest"
        "regexp"
@@ -14,21 +13,12 @@ import (
 
        "git.arvados.org/arvados.git/sdk/go/arvados"
        "git.arvados.org/arvados.git/sdk/go/arvadostest"
-       "github.com/prometheus/common/expfmt"
        "gopkg.in/check.v1"
 )
 
 func (s *IntegrationSuite) checkCacheMetrics(c *check.C, regs ...string) {
        s.handler.Cache.updateGauges()
-       reg := s.handler.Cache.registry
-       mfs, err := reg.Gather()
-       c.Check(err, check.IsNil)
-       buf := &bytes.Buffer{}
-       enc := expfmt.NewEncoder(buf, expfmt.FmtText)
-       for _, mf := range mfs {
-               c.Check(enc.Encode(mf), check.IsNil)
-       }
-       mm := buf.String()
+       mm := arvadostest.GatherMetricsAsString(s.handler.Cache.registry)
        // Remove comments to make the "value vs. regexp" failure
        // output easier to read.
        mm = regexp.MustCompile(`(?m)^#.*\n`).ReplaceAllString(mm, "")