X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ae03057c03ca6b8543c93a4b3938ba06dad41571..c3647ea9542ced1dfc9239c845f4b118b6743df0:/services/keep-balance/balance_run_test.go diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go index 3ef3b00077..0d1b6b5912 100644 --- a/services/keep-balance/balance_run_test.go +++ b/services/keep-balance/balance_run_test.go @@ -5,6 +5,7 @@ package main import ( + "bytes" "encoding/json" "fmt" "io" @@ -16,10 +17,13 @@ import ( "sync" "time" - "git.curoverse.com/arvados.git/lib/config" - "git.curoverse.com/arvados.git/sdk/go/arvados" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - "github.com/sirupsen/logrus" + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/jmoiron/sqlx" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" check "gopkg.in/check.v1" ) @@ -83,20 +87,24 @@ var stubServices = []arvados.KeepService{ var stubMounts = map[string][]arvados.KeepMount{ "keep0.zzzzz.arvadosapi.com:25107": {{ - UUID: "zzzzz-ivpuk-000000000000000", - DeviceID: "keep0-vol0", + UUID: "zzzzz-ivpuk-000000000000000", + DeviceID: "keep0-vol0", + StorageClasses: map[string]bool{"default": true}, }}, "keep1.zzzzz.arvadosapi.com:25107": {{ - UUID: "zzzzz-ivpuk-100000000000000", - DeviceID: "keep1-vol0", + UUID: "zzzzz-ivpuk-100000000000000", + DeviceID: "keep1-vol0", + StorageClasses: map[string]bool{"default": true}, }}, "keep2.zzzzz.arvadosapi.com:25107": {{ - UUID: "zzzzz-ivpuk-200000000000000", - DeviceID: "keep2-vol0", + UUID: "zzzzz-ivpuk-200000000000000", + DeviceID: "keep2-vol0", + StorageClasses: map[string]bool{"default": true}, }}, "keep3.zzzzz.arvadosapi.com:25107": {{ - UUID: "zzzzz-ivpuk-300000000000000", - DeviceID: "keep3-vol0", + UUID: "zzzzz-ivpuk-300000000000000", + DeviceID: "keep3-vol0", + StorageClasses: map[string]bool{"default": true}, }}, } @@ -306,6 +314,7 @@ func (s *stubServer) serveKeepstorePull() *reqTracker { type runSuite struct { stub stubServer config *arvados.Cluster + db *sqlx.DB client *arvados.Client } @@ -314,42 +323,21 @@ func (s *runSuite) newServer(options *RunOptions) *Server { Cluster: s.config, ArvClient: s.client, RunOptions: *options, - Metrics: newMetrics(), + Metrics: newMetrics(prometheus.NewRegistry()), Logger: options.Logger, Dumper: options.Dumper, + DB: s.db, } - srv.setup() return srv } -// make a log.Logger that writes to the current test's c.Log(). -func (s *runSuite) logger(c *check.C) *logrus.Logger { - r, w := io.Pipe() - go func() { - buf := make([]byte, 10000) - for { - n, err := r.Read(buf) - if n > 0 { - if buf[n-1] == '\n' { - n-- - } - c.Log(string(buf[:n])) - } - if err != nil { - break - } - } - }() - logger := logrus.New() - logger.Out = w - return logger -} - func (s *runSuite) SetUpTest(c *check.C) { - cfg, err := config.NewLoader(nil, nil).Load() + cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load() c.Assert(err, check.Equals, nil) s.config, err = cfg.GetCluster("") c.Assert(err, check.Equals, nil) + s.db, err = sqlx.Open("postgres", s.config.PostgreSQL.Connection.String()) + c.Assert(err, check.IsNil) s.config.Collections.BalancePeriod = arvados.Duration(time.Second) arvadostest.SetServiceURL(&s.config.Services.Keepbalance, "http://localhost:/") @@ -368,10 +356,13 @@ func (s *runSuite) TearDownTest(c *check.C) { } func (s *runSuite) TestRefuseZeroCollections(c *check.C) { + defer arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil) + _, err := s.db.Exec(`delete from collections`) + c.Assert(err, check.IsNil) opts := RunOptions{ CommitPulls: true, CommitTrash: true, - Logger: s.logger(c), + Logger: ctxlog.TestLogger(c), } s.stub.serveCurrentUserAdmin() s.stub.serveZeroCollections() @@ -381,7 +372,7 @@ func (s *runSuite) TestRefuseZeroCollections(c *check.C) { trashReqs := s.stub.serveKeepstoreTrash() pullReqs := s.stub.serveKeepstorePull() srv := s.newServer(&opts) - _, err := srv.runOnce() + _, err = srv.runOnce() c.Check(err, check.ErrorMatches, "received zero collections") c.Check(trashReqs.Count(), check.Equals, 4) c.Check(pullReqs.Count(), check.Equals, 0) @@ -391,7 +382,7 @@ func (s *runSuite) TestRefuseNonAdmin(c *check.C) { opts := RunOptions{ CommitPulls: true, CommitTrash: true, - Logger: s.logger(c), + Logger: ctxlog.TestLogger(c), } s.stub.serveCurrentUserNotAdmin() s.stub.serveZeroCollections() @@ -406,23 +397,29 @@ func (s *runSuite) TestRefuseNonAdmin(c *check.C) { c.Check(pullReqs.Count(), check.Equals, 0) } -func (s *runSuite) TestDetectSkippedCollections(c *check.C) { +func (s *runSuite) TestRefuseSameDeviceDifferentVolumes(c *check.C) { opts := RunOptions{ CommitPulls: true, CommitTrash: true, - Logger: s.logger(c), + Logger: ctxlog.TestLogger(c), } s.stub.serveCurrentUserAdmin() - s.stub.serveCollectionsButSkipOne() + s.stub.serveZeroCollections() s.stub.serveKeepServices(stubServices) - s.stub.serveKeepstoreMounts() - s.stub.serveKeepstoreIndexFoo4Bar1() + s.stub.mux.HandleFunc("/mounts", func(w http.ResponseWriter, r *http.Request) { + hostid := r.Host[:5] // "keep0.zzzzz.arvadosapi.com:25107" => "keep0" + json.NewEncoder(w).Encode([]arvados.KeepMount{{ + UUID: "zzzzz-ivpuk-0000000000" + hostid, + DeviceID: "keep0-vol0", + StorageClasses: map[string]bool{"default": true}, + }}) + }) trashReqs := s.stub.serveKeepstoreTrash() pullReqs := s.stub.serveKeepstorePull() srv := s.newServer(&opts) _, err := srv.runOnce() - c.Check(err, check.ErrorMatches, `Retrieved 2 collections with modtime <= .* but server now reports there are 3 collections.*`) - c.Check(trashReqs.Count(), check.Equals, 4) + c.Check(err, check.ErrorMatches, "cannot continue with config errors.*") + c.Check(trashReqs.Count(), check.Equals, 0) c.Check(pullReqs.Count(), check.Equals, 0) } @@ -434,7 +431,7 @@ func (s *runSuite) TestWriteLostBlocks(c *check.C) { opts := RunOptions{ CommitPulls: true, CommitTrash: true, - Logger: s.logger(c), + Logger: ctxlog.TestLogger(c), } s.stub.serveCurrentUserAdmin() s.stub.serveFooBarFileCollections() @@ -449,14 +446,14 @@ func (s *runSuite) TestWriteLostBlocks(c *check.C) { c.Check(err, check.IsNil) lost, err := ioutil.ReadFile(lostf.Name()) c.Assert(err, check.IsNil) - c.Check(string(lost), check.Equals, "37b51d194a7513e45b56f6524f2d51f2 fa7aeb5140e2848d39b416daeef4ffc5+45\n") + c.Check(string(lost), check.Matches, `(?ms).*37b51d194a7513e45b56f6524f2d51f2.* fa7aeb5140e2848d39b416daeef4ffc5\+45.*`) } func (s *runSuite) TestDryRun(c *check.C) { opts := RunOptions{ CommitPulls: false, CommitTrash: false, - Logger: s.logger(c), + Logger: ctxlog.TestLogger(c), } s.stub.serveCurrentUserAdmin() collReqs := s.stub.serveFooBarFileCollections() @@ -480,17 +477,13 @@ func (s *runSuite) TestDryRun(c *check.C) { } func (s *runSuite) TestCommit(c *check.C) { - lostf, err := ioutil.TempFile("", "keep-balance-lost-blocks-test-") - c.Assert(err, check.IsNil) - s.config.Collections.BlobMissingReport = lostf.Name() - defer os.Remove(lostf.Name()) - + s.config.Collections.BlobMissingReport = c.MkDir() + "/keep-balance-lost-blocks-test-" s.config.ManagementToken = "xyzzy" opts := RunOptions{ CommitPulls: true, CommitTrash: true, - Logger: s.logger(c), - Dumper: s.logger(c), + Logger: ctxlog.TestLogger(c), + Dumper: ctxlog.TestLogger(c), } s.stub.serveCurrentUserAdmin() s.stub.serveFooBarFileCollections() @@ -510,16 +503,18 @@ func (s *runSuite) TestCommit(c *check.C) { // in a poor rendezvous position c.Check(bal.stats.pulls, check.Equals, 2) - lost, err := ioutil.ReadFile(lostf.Name()) + lost, err := ioutil.ReadFile(s.config.Collections.BlobMissingReport) c.Assert(err, check.IsNil) - c.Check(string(lost), check.Equals, "") + c.Check(string(lost), check.Not(check.Matches), `(?ms).*acbd18db4cc2f85cedef654fccc4a4d8.*`) - metrics := s.getMetrics(c, srv) - c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_total_bytes 15\n.*`) - c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_sum [0-9\.]+\n.*`) - c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count 1\n.*`) - c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_dedup_byte_ratio 1\.5\n.*`) - c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_dedup_block_ratio 1\.5\n.*`) + buf, err := s.getMetrics(c, srv) + c.Check(err, check.IsNil) + bufstr := buf.String() + c.Check(bufstr, check.Matches, `(?ms).*\narvados_keep_total_bytes 15\n.*`) + c.Check(bufstr, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_sum [0-9\.]+\n.*`) + c.Check(bufstr, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count 1\n.*`) + c.Check(bufstr, check.Matches, `(?ms).*\narvados_keep_dedup_byte_ratio [1-9].*`) + c.Check(bufstr, check.Matches, `(?ms).*\narvados_keep_dedup_block_ratio [1-9].*`) } func (s *runSuite) TestRunForever(c *check.C) { @@ -527,8 +522,8 @@ func (s *runSuite) TestRunForever(c *check.C) { opts := RunOptions{ CommitPulls: true, CommitTrash: true, - Logger: s.logger(c), - Dumper: s.logger(c), + Logger: ctxlog.TestLogger(c), + Dumper: ctxlog.TestLogger(c), } s.stub.serveCurrentUserAdmin() s.stub.serveFooBarFileCollections() @@ -559,21 +554,24 @@ func (s *runSuite) TestRunForever(c *check.C) { <-done c.Check(pullReqs.Count() >= 16, check.Equals, true) c.Check(trashReqs.Count(), check.Equals, pullReqs.Count()+4) - c.Check(s.getMetrics(c, srv), check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count `+fmt.Sprintf("%d", pullReqs.Count()/4)+`\n.*`) + + buf, err := s.getMetrics(c, srv) + c.Check(err, check.IsNil) + c.Check(buf, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count `+fmt.Sprintf("%d", pullReqs.Count()/4)+`\n.*`) } -func (s *runSuite) getMetrics(c *check.C, srv *Server) string { - req := httptest.NewRequest("GET", "/metrics", nil) - resp := httptest.NewRecorder() - srv.ServeHTTP(resp, req) - c.Check(resp.Code, check.Equals, http.StatusUnauthorized) +func (s *runSuite) getMetrics(c *check.C, srv *Server) (*bytes.Buffer, error) { + mfs, err := srv.Metrics.reg.Gather() + if err != nil { + return nil, err + } - req = httptest.NewRequest("GET", "/metrics?api_token=xyzzy", nil) - resp = httptest.NewRecorder() - srv.ServeHTTP(resp, req) - c.Check(resp.Code, check.Equals, http.StatusOK) + var buf bytes.Buffer + for _, mf := range mfs { + if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil { + return nil, err + } + } - buf, err := ioutil.ReadAll(resp.Body) - c.Check(err, check.IsNil) - return string(buf) + return &buf, nil }