X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a95f899d7ac84f29b3d019aa410d265bb40833e5..562130229d7e85cbf813179a315772594e298127:/services/keep-balance/balance_run_test.go diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go index a138d911a3..28776abc47 100644 --- a/services/keep-balance/balance_run_test.go +++ b/services/keep-balance/balance_run_test.go @@ -1,7 +1,11 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( - _ "encoding/json" + "encoding/json" "fmt" "io" "io/ioutil" @@ -37,6 +41,63 @@ func (rt *reqTracker) Add(req *http.Request) int { return len(rt.reqs) } +var stubServices = []arvados.KeepService{ + { + UUID: "zzzzz-bi6l4-000000000000000", + ServiceHost: "keep0.zzzzz.arvadosapi.com", + ServicePort: 25107, + ServiceSSLFlag: false, + ServiceType: "disk", + }, + { + UUID: "zzzzz-bi6l4-000000000000001", + ServiceHost: "keep1.zzzzz.arvadosapi.com", + ServicePort: 25107, + ServiceSSLFlag: false, + ServiceType: "disk", + }, + { + UUID: "zzzzz-bi6l4-000000000000002", + ServiceHost: "keep2.zzzzz.arvadosapi.com", + ServicePort: 25107, + ServiceSSLFlag: false, + ServiceType: "disk", + }, + { + UUID: "zzzzz-bi6l4-000000000000003", + ServiceHost: "keep3.zzzzz.arvadosapi.com", + ServicePort: 25107, + ServiceSSLFlag: false, + ServiceType: "disk", + }, + { + UUID: "zzzzz-bi6l4-h0a0xwut9qa6g3a", + ServiceHost: "keep.zzzzz.arvadosapi.com", + ServicePort: 25333, + ServiceSSLFlag: true, + ServiceType: "proxy", + }, +} + +var stubMounts = map[string][]arvados.KeepMount{ + "keep0.zzzzz.arvadosapi.com:25107": {{ + UUID: "zzzzz-ivpuk-000000000000000", + DeviceID: "keep0-vol0", + }}, + "keep1.zzzzz.arvadosapi.com:25107": {{ + UUID: "zzzzz-ivpuk-100000000000000", + DeviceID: "keep1-vol0", + }}, + "keep2.zzzzz.arvadosapi.com:25107": {{ + UUID: "zzzzz-ivpuk-200000000000000", + DeviceID: "keep2-vol0", + }}, + "keep3.zzzzz.arvadosapi.com:25107": {{ + UUID: "zzzzz-ivpuk-300000000000000", + DeviceID: "keep3-vol0", + }}, +} + // stubServer is an HTTP transport that intercepts and processes all // requests using its own handlers. type stubServer struct { @@ -138,7 +199,9 @@ func (s *stubServer) serveCollectionsButSkipOne() *reqTracker { rt.Add(r) if strings.Contains(r.Form.Get("filters"), `"modified_at","\u003c="`) { io.WriteString(w, `{"items_available":3,"items":[]}`) - } else if strings.Contains(r.Form.Get("filters"), `"modified_at","\u003e="`) { + } else if strings.Contains(r.Form.Get("filters"), `"modified_at","\u003e`) { + io.WriteString(w, `{"items_available":0,"items":[]}`) + } else if strings.Contains(r.Form.Get("filters"), `"modified_at","="`) && strings.Contains(r.Form.Get("filters"), `"uuid","\u003e"`) { io.WriteString(w, `{"items_available":0,"items":[]}`) } else { io.WriteString(w, `{"items_available":2,"items":[ @@ -150,17 +213,32 @@ func (s *stubServer) serveCollectionsButSkipOne() *reqTracker { } func (s *stubServer) serveZeroKeepServices() *reqTracker { - return s.serveStatic("/arvados/v1/keep_services", - `{"items":[],"items_available":0}`) + return s.serveJSON("/arvados/v1/keep_services", arvados.KeepServiceList{}) +} + +func (s *stubServer) serveKeepServices(svcs []arvados.KeepService) *reqTracker { + return s.serveJSON("/arvados/v1/keep_services", arvados.KeepServiceList{ + ItemsAvailable: len(svcs), + Items: svcs, + }) +} + +func (s *stubServer) serveJSON(path string, resp interface{}) *reqTracker { + rt := &reqTracker{} + s.mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + rt.Add(r) + json.NewEncoder(w).Encode(resp) + }) + return rt } -func (s *stubServer) serveFourDiskKeepServices() *reqTracker { - return s.serveStatic("/arvados/v1/keep_services", `{"items_available":5,"items":[ - {"uuid":"zzzzz-bi6l4-000000000000000","service_host":"keep0.zzzzz.arvadosapi.com","service_port":25107,"service_ssl_flag":false,"service_type":"disk"}, - {"uuid":"zzzzz-bi6l4-000000000000001","service_host":"keep1.zzzzz.arvadosapi.com","service_port":25107,"service_ssl_flag":false,"service_type":"disk"}, - {"uuid":"zzzzz-bi6l4-000000000000002","service_host":"keep2.zzzzz.arvadosapi.com","service_port":25107,"service_ssl_flag":false,"service_type":"disk"}, - {"uuid":"zzzzz-bi6l4-000000000000003","service_host":"keep3.zzzzz.arvadosapi.com","service_port":25107,"service_ssl_flag":false,"service_type":"disk"}, - {"uuid":"zzzzz-bi6l4-h0a0xwut9qa6g3a","service_host":"keep.zzzzz.arvadosapi.com","service_port":25333,"service_ssl_flag":true,"service_type":"proxy"}]}`) +func (s *stubServer) serveKeepstoreMounts() *reqTracker { + rt := &reqTracker{} + s.mux.HandleFunc("/mounts", func(w http.ResponseWriter, r *http.Request) { + rt.Add(r) + json.NewEncoder(w).Encode(stubMounts[r.Host]) + }) + return rt } func (s *stubServer) serveKeepstoreIndexFoo4Bar1() *reqTracker { @@ -172,6 +250,21 @@ func (s *stubServer) serveKeepstoreIndexFoo4Bar1() *reqTracker { } fmt.Fprintf(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 %d\n\n", 12345678+count) }) + for _, mounts := range stubMounts { + for i, mnt := range mounts { + i := i + s.mux.HandleFunc(fmt.Sprintf("/mounts/%s/blocks", mnt.UUID), func(w http.ResponseWriter, r *http.Request) { + count := rt.Add(r) + if i == 0 && r.Host == "keep0.zzzzz.arvadosapi.com:25107" { + io.WriteString(w, "37b51d194a7513e45b56f6524f2d51f2+3 12345678\n") + } + if i == 0 { + fmt.Fprintf(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 %d\n", 12345678+count) + } + fmt.Fprintf(w, "\n") + }) + } + } return rt } @@ -232,11 +325,12 @@ func (s *runSuite) TestRefuseZeroCollections(c *check.C) { } s.stub.serveCurrentUserAdmin() s.stub.serveZeroCollections() - s.stub.serveFourDiskKeepServices() + s.stub.serveKeepServices(stubServices) + s.stub.serveKeepstoreMounts() s.stub.serveKeepstoreIndexFoo4Bar1() trashReqs := s.stub.serveKeepstoreTrash() pullReqs := s.stub.serveKeepstorePull() - err := (&Balancer{}).Run(s.config, opts) + _, err := (&Balancer{}).Run(s.config, opts) c.Check(err, check.ErrorMatches, "received zero collections") c.Check(trashReqs.Count(), check.Equals, 4) c.Check(pullReqs.Count(), check.Equals, 0) @@ -251,10 +345,11 @@ func (s *runSuite) TestServiceTypes(c *check.C) { s.config.KeepServiceTypes = []string{"unlisted-type"} s.stub.serveCurrentUserAdmin() s.stub.serveFooBarFileCollections() - s.stub.serveFourDiskKeepServices() + s.stub.serveKeepServices(stubServices) + s.stub.serveKeepstoreMounts() indexReqs := s.stub.serveKeepstoreIndexFoo4Bar1() trashReqs := s.stub.serveKeepstoreTrash() - err := (&Balancer{}).Run(s.config, opts) + _, err := (&Balancer{}).Run(s.config, opts) c.Check(err, check.IsNil) c.Check(indexReqs.Count(), check.Equals, 0) c.Check(trashReqs.Count(), check.Equals, 0) @@ -268,10 +363,11 @@ func (s *runSuite) TestRefuseNonAdmin(c *check.C) { } s.stub.serveCurrentUserNotAdmin() s.stub.serveZeroCollections() - s.stub.serveFourDiskKeepServices() + s.stub.serveKeepServices(stubServices) + s.stub.serveKeepstoreMounts() trashReqs := s.stub.serveKeepstoreTrash() pullReqs := s.stub.serveKeepstorePull() - err := (&Balancer{}).Run(s.config, opts) + _, err := (&Balancer{}).Run(s.config, opts) c.Check(err, check.ErrorMatches, "current user .* is not .* admin user") c.Check(trashReqs.Count(), check.Equals, 0) c.Check(pullReqs.Count(), check.Equals, 0) @@ -285,11 +381,12 @@ func (s *runSuite) TestDetectSkippedCollections(c *check.C) { } s.stub.serveCurrentUserAdmin() s.stub.serveCollectionsButSkipOne() - s.stub.serveFourDiskKeepServices() + s.stub.serveKeepServices(stubServices) + s.stub.serveKeepstoreMounts() s.stub.serveKeepstoreIndexFoo4Bar1() trashReqs := s.stub.serveKeepstoreTrash() pullReqs := s.stub.serveKeepstorePull() - err := (&Balancer{}).Run(s.config, opts) + _, err := (&Balancer{}).Run(s.config, opts) c.Check(err, check.ErrorMatches, `Retrieved 2 collections with modtime <= .* but server now reports there are 3 collections.*`) c.Check(trashReqs.Count(), check.Equals, 4) c.Check(pullReqs.Count(), check.Equals, 0) @@ -302,20 +399,23 @@ func (s *runSuite) TestDryRun(c *check.C) { Logger: s.logger(c), } s.stub.serveCurrentUserAdmin() - s.stub.serveFooBarFileCollections() - s.stub.serveFourDiskKeepServices() + collReqs := s.stub.serveFooBarFileCollections() + s.stub.serveKeepServices(stubServices) + s.stub.serveKeepstoreMounts() s.stub.serveKeepstoreIndexFoo4Bar1() trashReqs := s.stub.serveKeepstoreTrash() pullReqs := s.stub.serveKeepstorePull() var bal Balancer - err := bal.Run(s.config, opts) + _, err := bal.Run(s.config, opts) c.Check(err, check.IsNil) + for _, req := range collReqs.reqs { + c.Check(req.Form.Get("include_trash"), check.Equals, "true") + } c.Check(trashReqs.Count(), check.Equals, 0) c.Check(pullReqs.Count(), check.Equals, 0) - stats := bal.getStatistics() - c.Check(stats.pulls, check.Not(check.Equals), 0) - c.Check(stats.underrep.replicas, check.Not(check.Equals), 0) - c.Check(stats.overrep.replicas, check.Not(check.Equals), 0) + c.Check(bal.stats.pulls, check.Not(check.Equals), 0) + c.Check(bal.stats.underrep.replicas, check.Not(check.Equals), 0) + c.Check(bal.stats.overrep.replicas, check.Not(check.Equals), 0) } func (s *runSuite) TestCommit(c *check.C) { @@ -327,21 +427,21 @@ func (s *runSuite) TestCommit(c *check.C) { } s.stub.serveCurrentUserAdmin() s.stub.serveFooBarFileCollections() - s.stub.serveFourDiskKeepServices() + s.stub.serveKeepServices(stubServices) + s.stub.serveKeepstoreMounts() s.stub.serveKeepstoreIndexFoo4Bar1() trashReqs := s.stub.serveKeepstoreTrash() pullReqs := s.stub.serveKeepstorePull() var bal Balancer - err := bal.Run(s.config, opts) + _, err := bal.Run(s.config, opts) c.Check(err, check.IsNil) c.Check(trashReqs.Count(), check.Equals, 8) c.Check(pullReqs.Count(), check.Equals, 4) - stats := bal.getStatistics() // "foo" block is overreplicated by 2 - c.Check(stats.trashes, check.Equals, 2) + c.Check(bal.stats.trashes, check.Equals, 2) // "bar" block is underreplicated by 1, and its only copy is // in a poor rendezvous position - c.Check(stats.pulls, check.Equals, 2) + c.Check(bal.stats.pulls, check.Equals, 2) } func (s *runSuite) TestRunForever(c *check.C) { @@ -353,7 +453,8 @@ func (s *runSuite) TestRunForever(c *check.C) { } s.stub.serveCurrentUserAdmin() s.stub.serveFooBarFileCollections() - s.stub.serveFourDiskKeepServices() + s.stub.serveKeepServices(stubServices) + s.stub.serveKeepstoreMounts() s.stub.serveKeepstoreIndexFoo4Bar1() trashReqs := s.stub.serveKeepstoreTrash() pullReqs := s.stub.serveKeepstorePull() @@ -362,13 +463,14 @@ func (s *runSuite) TestRunForever(c *check.C) { s.config.RunPeriod = arvados.Duration(time.Millisecond) go RunForever(s.config, opts, stop) - // Each run should send 4 clear trash lists + 4 pull lists + 4 - // trash lists. We should complete four runs in much less than + // Each run should send 4 pull lists + 4 trash lists. The + // first run should also send 4 empty trash lists at + // startup. We should complete all four runs in much less than // a second. for t0 := time.Now(); pullReqs.Count() < 16 && time.Since(t0) < 10*time.Second; { time.Sleep(time.Millisecond) } stop <- true c.Check(pullReqs.Count() >= 16, check.Equals, true) - c.Check(trashReqs.Count(), check.Equals, 2*pullReqs.Count()) + c.Check(trashReqs.Count(), check.Equals, pullReqs.Count()+4) }