services/keep-balance/balance_run_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "os"
  16         "strings"
  17         "sync"
  18         "time"
  19
  20         "git.arvados.org/arvados.git/lib/config"
  21         "git.arvados.org/arvados.git/sdk/go/arvados"
  22         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  23         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  24         "github.com/jmoiron/sqlx"
  25         "github.com/prometheus/client_golang/prometheus"
  26         "github.com/prometheus/common/expfmt"
  27         check "gopkg.in/check.v1"
  28 )
  29
  30 var _ = check.Suite(&runSuite{})
  31
  32 type reqTracker struct {
  33         reqs []http.Request
  34         sync.Mutex
  35 }
  36
  37 func (rt *reqTracker) Count() int {
  38         rt.Lock()
  39         defer rt.Unlock()
  40         return len(rt.reqs)
  41 }
  42
  43 func (rt *reqTracker) Add(req *http.Request) int {
  44         rt.Lock()
  45         defer rt.Unlock()
  46         rt.reqs = append(rt.reqs, *req)
  47         return len(rt.reqs)
  48 }
  49
  50 var stubServices = []arvados.KeepService{
  51         {
  52                 UUID:           "zzzzz-bi6l4-000000000000000",
  53                 ServiceHost:    "keep0.zzzzz.arvadosapi.com",
  54                 ServicePort:    25107,
  55                 ServiceSSLFlag: false,
  56                 ServiceType:    "disk",
  57         },
  58         {
  59                 UUID:           "zzzzz-bi6l4-000000000000001",
  60                 ServiceHost:    "keep1.zzzzz.arvadosapi.com",
  61                 ServicePort:    25107,
  62                 ServiceSSLFlag: false,
  63                 ServiceType:    "disk",
  64         },
  65         {
  66                 UUID:           "zzzzz-bi6l4-000000000000002",
  67                 ServiceHost:    "keep2.zzzzz.arvadosapi.com",
  68                 ServicePort:    25107,
  69                 ServiceSSLFlag: false,
  70                 ServiceType:    "disk",
  71         },
  72         {
  73                 UUID:           "zzzzz-bi6l4-000000000000003",
  74                 ServiceHost:    "keep3.zzzzz.arvadosapi.com",
  75                 ServicePort:    25107,
  76                 ServiceSSLFlag: false,
  77                 ServiceType:    "disk",
  78         },
  79         {
  80                 UUID:           "zzzzz-bi6l4-h0a0xwut9qa6g3a",
  81                 ServiceHost:    "keep.zzzzz.arvadosapi.com",
  82                 ServicePort:    25333,
  83                 ServiceSSLFlag: true,
  84                 ServiceType:    "proxy",
  85         },
  86 }
  87
  88 var stubMounts = map[string][]arvados.KeepMount{
  89         "keep0.zzzzz.arvadosapi.com:25107": {{
  90                 UUID:     "zzzzz-ivpuk-000000000000000",
  91                 DeviceID: "keep0-vol0",
  92         }},
  93         "keep1.zzzzz.arvadosapi.com:25107": {{
  94                 UUID:     "zzzzz-ivpuk-100000000000000",
  95                 DeviceID: "keep1-vol0",
  96         }},
  97         "keep2.zzzzz.arvadosapi.com:25107": {{
  98                 UUID:     "zzzzz-ivpuk-200000000000000",
  99                 DeviceID: "keep2-vol0",
 100         }},
 101         "keep3.zzzzz.arvadosapi.com:25107": {{
 102                 UUID:     "zzzzz-ivpuk-300000000000000",
 103                 DeviceID: "keep3-vol0",
 104         }},
 105 }
 106
 107 // stubServer is an HTTP transport that intercepts and processes all
 108 // requests using its own handlers.
 109 type stubServer struct {
 110         mux      *http.ServeMux
 111         srv      *httptest.Server
 112         mutex    sync.Mutex
 113         Requests reqTracker
 114         logf     func(string, ...interface{})
 115 }
 116
 117 // Start initializes the stub server and returns an *http.Client that
 118 // uses the stub server to handle all requests.
 119 //
 120 // A stubServer that has been started should eventually be shut down
 121 // with Close().
 122 func (s *stubServer) Start() *http.Client {
 123         // Set up a config.Client that forwards all requests to s.mux
 124         // via s.srv. Test cases will attach handlers to s.mux to get
 125         // the desired responses.
 126         s.mux = http.NewServeMux()
 127         s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 128                 s.mutex.Lock()
 129                 s.Requests.Add(r)
 130                 s.mutex.Unlock()
 131                 w.Header().Set("Content-Type", "application/json")
 132                 s.mux.ServeHTTP(w, r)
 133         }))
 134         return &http.Client{Transport: s}
 135 }
 136
 137 func (s *stubServer) RoundTrip(req *http.Request) (*http.Response, error) {
 138         w := httptest.NewRecorder()
 139         s.mux.ServeHTTP(w, req)
 140         return &http.Response{
 141                 StatusCode: w.Code,
 142                 Status:     fmt.Sprintf("%d %s", w.Code, http.StatusText(w.Code)),
 143                 Header:     w.HeaderMap,
 144                 Body:       ioutil.NopCloser(w.Body)}, nil
 145 }
 146
 147 // Close releases resources used by the server.
 148 func (s *stubServer) Close() {
 149         s.srv.Close()
 150 }
 151
 152 func (s *stubServer) serveStatic(path, data string) *reqTracker {
 153         rt := &reqTracker{}
 154         s.mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
 155                 rt.Add(r)
 156                 if r.Body != nil {
 157                         ioutil.ReadAll(r.Body)
 158                         r.Body.Close()
 159                 }
 160                 io.WriteString(w, data)
 161         })
 162         return rt
 163 }
 164
 165 func (s *stubServer) serveCurrentUserAdmin() *reqTracker {
 166         return s.serveStatic("/arvados/v1/users/current",
 167                 `{"uuid":"zzzzz-tpzed-000000000000000","is_admin":true,"is_active":true}`)
 168 }
 169
 170 func (s *stubServer) serveCurrentUserNotAdmin() *reqTracker {
 171         return s.serveStatic("/arvados/v1/users/current",
 172                 `{"uuid":"zzzzz-tpzed-000000000000000","is_admin":false,"is_active":true}`)
 173 }
 174
 175 func (s *stubServer) serveDiscoveryDoc() *reqTracker {
 176         return s.serveStatic("/discovery/v1/apis/arvados/v1/rest",
 177                 `{"defaultCollectionReplication":2}`)
 178 }
 179
 180 func (s *stubServer) serveZeroCollections() *reqTracker {
 181         return s.serveStatic("/arvados/v1/collections",
 182                 `{"items":[],"items_available":0}`)
 183 }
 184
 185 func (s *stubServer) serveFooBarFileCollections() *reqTracker {
 186         rt := &reqTracker{}
 187         s.mux.HandleFunc("/arvados/v1/collections", func(w http.ResponseWriter, r *http.Request) {
 188                 r.ParseForm()
 189                 rt.Add(r)
 190                 if strings.Contains(r.Form.Get("filters"), `modified_at`) {
 191                         io.WriteString(w, `{"items_available":0,"items":[]}`)
 192                 } else {
 193                         io.WriteString(w, `{"items_available":3,"items":[
 194                                 {"uuid":"zzzzz-4zz18-aaaaaaaaaaaaaaa","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
 195                                 {"uuid":"zzzzz-4zz18-ehbhgtheo8909or","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
 196                                 {"uuid":"zzzzz-4zz18-znfnqtbbv4spc3w","portable_data_hash":"1f4b0bc7583c2a7f9102c395f4ffc5e3+45","manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n","modified_at":"2014-02-03T17:22:54Z"}]}`)
 197                 }
 198         })
 199         return rt
 200 }
 201
 202 func (s *stubServer) serveCollectionsButSkipOne() *reqTracker {
 203         rt := &reqTracker{}
 204         s.mux.HandleFunc("/arvados/v1/collections", func(w http.ResponseWriter, r *http.Request) {
 205                 r.ParseForm()
 206                 rt.Add(r)
 207                 if strings.Contains(r.Form.Get("filters"), `"modified_at","\u003c="`) {
 208                         io.WriteString(w, `{"items_available":3,"items":[]}`)
 209                 } else if strings.Contains(r.Form.Get("filters"), `"modified_at","\u003e`) {
 210                         io.WriteString(w, `{"items_available":0,"items":[]}`)
 211                 } else if strings.Contains(r.Form.Get("filters"), `"modified_at","="`) && strings.Contains(r.Form.Get("filters"), `"uuid","\u003e"`) {
 212                         io.WriteString(w, `{"items_available":0,"items":[]}`)
 213                 } else if strings.Contains(r.Form.Get("filters"), `"modified_at","=",null`) {
 214                         io.WriteString(w, `{"items_available":0,"items":[]}`)
 215                 } else {
 216                         io.WriteString(w, `{"items_available":2,"items":[
 217                                 {"uuid":"zzzzz-4zz18-ehbhgtheo8909or","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
 218                                 {"uuid":"zzzzz-4zz18-znfnqtbbv4spc3w","portable_data_hash":"1f4b0bc7583c2a7f9102c395f4ffc5e3+45","manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n","modified_at":"2014-02-03T17:22:54Z"}]}`)
 219                 }
 220         })
 221         return rt
 222 }
 223
 224 func (s *stubServer) serveZeroKeepServices() *reqTracker {
 225         return s.serveJSON("/arvados/v1/keep_services", arvados.KeepServiceList{})
 226 }
 227
 228 func (s *stubServer) serveKeepServices(svcs []arvados.KeepService) *reqTracker {
 229         return s.serveJSON("/arvados/v1/keep_services", arvados.KeepServiceList{
 230                 ItemsAvailable: len(svcs),
 231                 Items:          svcs,
 232         })
 233 }
 234
 235 func (s *stubServer) serveJSON(path string, resp interface{}) *reqTracker {
 236         rt := &reqTracker{}
 237         s.mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
 238                 rt.Add(r)
 239                 json.NewEncoder(w).Encode(resp)
 240         })
 241         return rt
 242 }
 243
 244 func (s *stubServer) serveKeepstoreMounts() *reqTracker {
 245         rt := &reqTracker{}
 246         s.mux.HandleFunc("/mounts", func(w http.ResponseWriter, r *http.Request) {
 247                 rt.Add(r)
 248                 json.NewEncoder(w).Encode(stubMounts[r.Host])
 249         })
 250         return rt
 251 }
 252
 253 func (s *stubServer) serveKeepstoreIndexFoo4Bar1() *reqTracker {
 254         rt := &reqTracker{}
 255         s.mux.HandleFunc("/index/", func(w http.ResponseWriter, r *http.Request) {
 256                 count := rt.Add(r)
 257                 if r.Host == "keep0.zzzzz.arvadosapi.com:25107" {
 258                         io.WriteString(w, "37b51d194a7513e45b56f6524f2d51f2+3 12345678\n")
 259                 }
 260                 fmt.Fprintf(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 %d\n\n", 12345678+count)
 261         })
 262         for _, mounts := range stubMounts {
 263                 for i, mnt := range mounts {
 264                         i := i
 265                         s.mux.HandleFunc(fmt.Sprintf("/mounts/%s/blocks", mnt.UUID), func(w http.ResponseWriter, r *http.Request) {
 266                                 count := rt.Add(r)
 267                                 if i == 0 && r.Host == "keep0.zzzzz.arvadosapi.com:25107" {
 268                                         io.WriteString(w, "37b51d194a7513e45b56f6524f2d51f2+3 12345678\n")
 269                                 }
 270                                 if i == 0 {
 271                                         fmt.Fprintf(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 %d\n", 12345678+count)
 272                                 }
 273                                 fmt.Fprintf(w, "\n")
 274                         })
 275                 }
 276         }
 277         return rt
 278 }
 279
 280 func (s *stubServer) serveKeepstoreIndexFoo1() *reqTracker {
 281         rt := &reqTracker{}
 282         s.mux.HandleFunc("/index/", func(w http.ResponseWriter, r *http.Request) {
 283                 rt.Add(r)
 284                 io.WriteString(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 12345678\n\n")
 285         })
 286         for _, mounts := range stubMounts {
 287                 for i, mnt := range mounts {
 288                         i := i
 289                         s.mux.HandleFunc(fmt.Sprintf("/mounts/%s/blocks", mnt.UUID), func(w http.ResponseWriter, r *http.Request) {
 290                                 rt.Add(r)
 291                                 if i == 0 {
 292                                         io.WriteString(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 12345678\n\n")
 293                                 } else {
 294                                         io.WriteString(w, "\n")
 295                                 }
 296                         })
 297                 }
 298         }
 299         return rt
 300 }
 301
 302 func (s *stubServer) serveKeepstoreTrash() *reqTracker {
 303         return s.serveStatic("/trash", `{}`)
 304 }
 305
 306 func (s *stubServer) serveKeepstorePull() *reqTracker {
 307         return s.serveStatic("/pull", `{}`)
 308 }
 309
 310 type runSuite struct {
 311         stub   stubServer
 312         config *arvados.Cluster
 313         db     *sqlx.DB
 314         client *arvados.Client
 315 }
 316
 317 func (s *runSuite) newServer(options *RunOptions) *Server {
 318         srv := &Server{
 319                 Cluster:    s.config,
 320                 ArvClient:  s.client,
 321                 RunOptions: *options,
 322                 Metrics:    newMetrics(prometheus.NewRegistry()),
 323                 Logger:     options.Logger,
 324                 Dumper:     options.Dumper,
 325                 DB:         s.db,
 326         }
 327         return srv
 328 }
 329
 330 func (s *runSuite) SetUpTest(c *check.C) {
 331         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
 332         c.Assert(err, check.Equals, nil)
 333         s.config, err = cfg.GetCluster("")
 334         c.Assert(err, check.Equals, nil)
 335         s.db, err = sqlx.Open("postgres", s.config.PostgreSQL.Connection.String())
 336         c.Assert(err, check.IsNil)
 337
 338         s.config.Collections.BalancePeriod = arvados.Duration(time.Second)
 339         arvadostest.SetServiceURL(&s.config.Services.Keepbalance, "http://localhost:/")
 340
 341         s.client = &arvados.Client{
 342                 AuthToken: "xyzzy",
 343                 APIHost:   "zzzzz.arvadosapi.com",
 344                 Client:    s.stub.Start()}
 345
 346         s.stub.serveDiscoveryDoc()
 347         s.stub.logf = c.Logf
 348 }
 349
 350 func (s *runSuite) TearDownTest(c *check.C) {
 351         s.stub.Close()
 352 }
 353
 354 func (s *runSuite) TestRefuseZeroCollections(c *check.C) {
 355         opts := RunOptions{
 356                 CommitPulls: true,
 357                 CommitTrash: true,
 358                 Logger:      ctxlog.TestLogger(c),
 359         }
 360         s.stub.serveCurrentUserAdmin()
 361         s.stub.serveZeroCollections()
 362         s.stub.serveKeepServices(stubServices)
 363         s.stub.serveKeepstoreMounts()
 364         s.stub.serveKeepstoreIndexFoo4Bar1()
 365         trashReqs := s.stub.serveKeepstoreTrash()
 366         pullReqs := s.stub.serveKeepstorePull()
 367         srv := s.newServer(&opts)
 368         _, err := srv.runOnce()
 369         c.Check(err, check.ErrorMatches, "received zero collections")
 370         c.Check(trashReqs.Count(), check.Equals, 4)
 371         c.Check(pullReqs.Count(), check.Equals, 0)
 372 }
 373
 374 func (s *runSuite) TestRefuseNonAdmin(c *check.C) {
 375         opts := RunOptions{
 376                 CommitPulls: true,
 377                 CommitTrash: true,
 378                 Logger:      ctxlog.TestLogger(c),
 379         }
 380         s.stub.serveCurrentUserNotAdmin()
 381         s.stub.serveZeroCollections()
 382         s.stub.serveKeepServices(stubServices)
 383         s.stub.serveKeepstoreMounts()
 384         trashReqs := s.stub.serveKeepstoreTrash()
 385         pullReqs := s.stub.serveKeepstorePull()
 386         srv := s.newServer(&opts)
 387         _, err := srv.runOnce()
 388         c.Check(err, check.ErrorMatches, "current user .* is not .* admin user")
 389         c.Check(trashReqs.Count(), check.Equals, 0)
 390         c.Check(pullReqs.Count(), check.Equals, 0)
 391 }
 392
 393 func (s *runSuite) TestDetectSkippedCollections(c *check.C) {
 394         opts := RunOptions{
 395                 CommitPulls: true,
 396                 CommitTrash: true,
 397                 Logger:      ctxlog.TestLogger(c),
 398         }
 399         s.stub.serveCurrentUserAdmin()
 400         s.stub.serveCollectionsButSkipOne()
 401         s.stub.serveKeepServices(stubServices)
 402         s.stub.serveKeepstoreMounts()
 403         s.stub.serveKeepstoreIndexFoo4Bar1()
 404         trashReqs := s.stub.serveKeepstoreTrash()
 405         pullReqs := s.stub.serveKeepstorePull()
 406         srv := s.newServer(&opts)
 407         _, err := srv.runOnce()
 408         c.Check(err, check.ErrorMatches, `Retrieved 2 collections with modtime <= .* but server now reports there are 3 collections.*`)
 409         c.Check(trashReqs.Count(), check.Equals, 4)
 410         c.Check(pullReqs.Count(), check.Equals, 0)
 411 }
 412
 413 func (s *runSuite) TestWriteLostBlocks(c *check.C) {
 414         lostf, err := ioutil.TempFile("", "keep-balance-lost-blocks-test-")
 415         c.Assert(err, check.IsNil)
 416         s.config.Collections.BlobMissingReport = lostf.Name()
 417         defer os.Remove(lostf.Name())
 418         opts := RunOptions{
 419                 CommitPulls: true,
 420                 CommitTrash: true,
 421                 Logger:      ctxlog.TestLogger(c),
 422         }
 423         s.stub.serveCurrentUserAdmin()
 424         s.stub.serveFooBarFileCollections()
 425         s.stub.serveKeepServices(stubServices)
 426         s.stub.serveKeepstoreMounts()
 427         s.stub.serveKeepstoreIndexFoo1()
 428         s.stub.serveKeepstoreTrash()
 429         s.stub.serveKeepstorePull()
 430         srv := s.newServer(&opts)
 431         c.Assert(err, check.IsNil)
 432         _, err = srv.runOnce()
 433         c.Check(err, check.IsNil)
 434         lost, err := ioutil.ReadFile(lostf.Name())
 435         c.Assert(err, check.IsNil)
 436         c.Check(string(lost), check.Equals, "37b51d194a7513e45b56f6524f2d51f2 fa7aeb5140e2848d39b416daeef4ffc5+45\n")
 437 }
 438
 439 func (s *runSuite) TestDryRun(c *check.C) {
 440         opts := RunOptions{
 441                 CommitPulls: false,
 442                 CommitTrash: false,
 443                 Logger:      ctxlog.TestLogger(c),
 444         }
 445         s.stub.serveCurrentUserAdmin()
 446         collReqs := s.stub.serveFooBarFileCollections()
 447         s.stub.serveKeepServices(stubServices)
 448         s.stub.serveKeepstoreMounts()
 449         s.stub.serveKeepstoreIndexFoo4Bar1()
 450         trashReqs := s.stub.serveKeepstoreTrash()
 451         pullReqs := s.stub.serveKeepstorePull()
 452         srv := s.newServer(&opts)
 453         bal, err := srv.runOnce()
 454         c.Check(err, check.IsNil)
 455         for _, req := range collReqs.reqs {
 456                 c.Check(req.Form.Get("include_trash"), check.Equals, "true")
 457                 c.Check(req.Form.Get("include_old_versions"), check.Equals, "true")
 458         }
 459         c.Check(trashReqs.Count(), check.Equals, 0)
 460         c.Check(pullReqs.Count(), check.Equals, 0)
 461         c.Check(bal.stats.pulls, check.Not(check.Equals), 0)
 462         c.Check(bal.stats.underrep.replicas, check.Not(check.Equals), 0)
 463         c.Check(bal.stats.overrep.replicas, check.Not(check.Equals), 0)
 464 }
 465
 466 func (s *runSuite) TestCommit(c *check.C) {
 467         lostf, err := ioutil.TempFile("", "keep-balance-lost-blocks-test-")
 468         c.Assert(err, check.IsNil)
 469         s.config.Collections.BlobMissingReport = lostf.Name()
 470         defer os.Remove(lostf.Name())
 471
 472         s.config.ManagementToken = "xyzzy"
 473         opts := RunOptions{
 474                 CommitPulls: true,
 475                 CommitTrash: true,
 476                 Logger:      ctxlog.TestLogger(c),
 477                 Dumper:      ctxlog.TestLogger(c),
 478         }
 479         s.stub.serveCurrentUserAdmin()
 480         s.stub.serveFooBarFileCollections()
 481         s.stub.serveKeepServices(stubServices)
 482         s.stub.serveKeepstoreMounts()
 483         s.stub.serveKeepstoreIndexFoo4Bar1()
 484         trashReqs := s.stub.serveKeepstoreTrash()
 485         pullReqs := s.stub.serveKeepstorePull()
 486         srv := s.newServer(&opts)
 487         bal, err := srv.runOnce()
 488         c.Check(err, check.IsNil)
 489         c.Check(trashReqs.Count(), check.Equals, 8)
 490         c.Check(pullReqs.Count(), check.Equals, 4)
 491         // "foo" block is overreplicated by 2
 492         c.Check(bal.stats.trashes, check.Equals, 2)
 493         // "bar" block is underreplicated by 1, and its only copy is
 494         // in a poor rendezvous position
 495         c.Check(bal.stats.pulls, check.Equals, 2)
 496
 497         lost, err := ioutil.ReadFile(lostf.Name())
 498         c.Assert(err, check.IsNil)
 499         c.Check(string(lost), check.Equals, "")
 500
 501         buf, err := s.getMetrics(c, srv)
 502         c.Check(err, check.IsNil)
 503         c.Check(buf, check.Matches, `(?ms).*\narvados_keep_total_bytes 15\n.*`)
 504         c.Check(buf, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_sum [0-9\.]+\n.*`)
 505         c.Check(buf, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count 1\n.*`)
 506         c.Check(buf, check.Matches, `(?ms).*\narvados_keep_dedup_byte_ratio 1\.5\n.*`)
 507         c.Check(buf, check.Matches, `(?ms).*\narvados_keep_dedup_block_ratio 1\.5\n.*`)
 508 }
 509
 510 func (s *runSuite) TestRunForever(c *check.C) {
 511         s.config.ManagementToken = "xyzzy"
 512         opts := RunOptions{
 513                 CommitPulls: true,
 514                 CommitTrash: true,
 515                 Logger:      ctxlog.TestLogger(c),
 516                 Dumper:      ctxlog.TestLogger(c),
 517         }
 518         s.stub.serveCurrentUserAdmin()
 519         s.stub.serveFooBarFileCollections()
 520         s.stub.serveKeepServices(stubServices)
 521         s.stub.serveKeepstoreMounts()
 522         s.stub.serveKeepstoreIndexFoo4Bar1()
 523         trashReqs := s.stub.serveKeepstoreTrash()
 524         pullReqs := s.stub.serveKeepstorePull()
 525
 526         stop := make(chan interface{})
 527         s.config.Collections.BalancePeriod = arvados.Duration(time.Millisecond)
 528         srv := s.newServer(&opts)
 529
 530         done := make(chan bool)
 531         go func() {
 532                 srv.runForever(stop)
 533                 close(done)
 534         }()
 535
 536         // Each run should send 4 pull lists + 4 trash lists. The
 537         // first run should also send 4 empty trash lists at
 538         // startup. We should complete all four runs in much less than
 539         // a second.
 540         for t0 := time.Now(); pullReqs.Count() < 16 && time.Since(t0) < 10*time.Second; {
 541                 time.Sleep(time.Millisecond)
 542         }
 543         stop <- true
 544         <-done
 545         c.Check(pullReqs.Count() >= 16, check.Equals, true)
 546         c.Check(trashReqs.Count(), check.Equals, pullReqs.Count()+4)
 547
 548         buf, err := s.getMetrics(c, srv)
 549         c.Check(err, check.IsNil)
 550         c.Check(buf, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count `+fmt.Sprintf("%d", pullReqs.Count()/4)+`\n.*`)
 551 }
 552
 553 func (s *runSuite) getMetrics(c *check.C, srv *Server) (*bytes.Buffer, error) {
 554         mfs, err := srv.Metrics.reg.Gather()
 555         if err != nil {
 556                 return nil, err
 557         }
 558
 559         var buf bytes.Buffer
 560         for _, mf := range mfs {
 561                 if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil {
 562                         return nil, err
 563                 }
 564         }
 565
 566         return &buf, nil
 567 }