1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
20 "git.arvados.org/arvados.git/lib/config"
21 "git.arvados.org/arvados.git/sdk/go/arvados"
22 "git.arvados.org/arvados.git/sdk/go/arvadostest"
23 "git.arvados.org/arvados.git/sdk/go/ctxlog"
24 "github.com/jmoiron/sqlx"
25 "github.com/prometheus/client_golang/prometheus"
26 "github.com/prometheus/common/expfmt"
27 check "gopkg.in/check.v1"
30 var _ = check.Suite(&runSuite{})
32 type reqTracker struct {
37 func (rt *reqTracker) Count() int {
43 func (rt *reqTracker) Add(req *http.Request) int {
46 rt.reqs = append(rt.reqs, *req)
50 var stubServices = []arvados.KeepService{
52 UUID: "zzzzz-bi6l4-000000000000000",
53 ServiceHost: "keep0.zzzzz.arvadosapi.com",
55 ServiceSSLFlag: false,
59 UUID: "zzzzz-bi6l4-000000000000001",
60 ServiceHost: "keep1.zzzzz.arvadosapi.com",
62 ServiceSSLFlag: false,
66 UUID: "zzzzz-bi6l4-000000000000002",
67 ServiceHost: "keep2.zzzzz.arvadosapi.com",
69 ServiceSSLFlag: false,
73 UUID: "zzzzz-bi6l4-000000000000003",
74 ServiceHost: "keep3.zzzzz.arvadosapi.com",
76 ServiceSSLFlag: false,
80 UUID: "zzzzz-bi6l4-h0a0xwut9qa6g3a",
81 ServiceHost: "keep.zzzzz.arvadosapi.com",
88 var stubMounts = map[string][]arvados.KeepMount{
89 "keep0.zzzzz.arvadosapi.com:25107": {{
90 UUID: "zzzzz-ivpuk-000000000000000",
91 DeviceID: "keep0-vol0",
92 StorageClasses: map[string]bool{"default": true},
94 "keep1.zzzzz.arvadosapi.com:25107": {{
95 UUID: "zzzzz-ivpuk-100000000000000",
96 DeviceID: "keep1-vol0",
97 StorageClasses: map[string]bool{"default": true},
99 "keep2.zzzzz.arvadosapi.com:25107": {{
100 UUID: "zzzzz-ivpuk-200000000000000",
101 DeviceID: "keep2-vol0",
102 StorageClasses: map[string]bool{"default": true},
104 "keep3.zzzzz.arvadosapi.com:25107": {{
105 UUID: "zzzzz-ivpuk-300000000000000",
106 DeviceID: "keep3-vol0",
107 StorageClasses: map[string]bool{"default": true},
111 // stubServer is an HTTP transport that intercepts and processes all
112 // requests using its own handlers.
113 type stubServer struct {
118 logf func(string, ...interface{})
121 // Start initializes the stub server and returns an *http.Client that
122 // uses the stub server to handle all requests.
124 // A stubServer that has been started should eventually be shut down
126 func (s *stubServer) Start() *http.Client {
127 // Set up a config.Client that forwards all requests to s.mux
128 // via s.srv. Test cases will attach handlers to s.mux to get
129 // the desired responses.
130 s.mux = http.NewServeMux()
131 s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
135 w.Header().Set("Content-Type", "application/json")
136 s.mux.ServeHTTP(w, r)
138 return &http.Client{Transport: s}
141 func (s *stubServer) RoundTrip(req *http.Request) (*http.Response, error) {
142 w := httptest.NewRecorder()
143 s.mux.ServeHTTP(w, req)
144 return &http.Response{
146 Status: fmt.Sprintf("%d %s", w.Code, http.StatusText(w.Code)),
148 Body: ioutil.NopCloser(w.Body)}, nil
151 // Close releases resources used by the server.
152 func (s *stubServer) Close() {
156 func (s *stubServer) serveStatic(path, data string) *reqTracker {
158 s.mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
161 ioutil.ReadAll(r.Body)
164 io.WriteString(w, data)
169 func (s *stubServer) serveCurrentUserAdmin() *reqTracker {
170 return s.serveStatic("/arvados/v1/users/current",
171 `{"uuid":"zzzzz-tpzed-000000000000000","is_admin":true,"is_active":true}`)
174 func (s *stubServer) serveCurrentUserNotAdmin() *reqTracker {
175 return s.serveStatic("/arvados/v1/users/current",
176 `{"uuid":"zzzzz-tpzed-000000000000000","is_admin":false,"is_active":true}`)
179 func (s *stubServer) serveDiscoveryDoc() *reqTracker {
180 return s.serveStatic("/discovery/v1/apis/arvados/v1/rest",
181 `{"defaultCollectionReplication":2}`)
184 func (s *stubServer) serveZeroCollections() *reqTracker {
185 return s.serveStatic("/arvados/v1/collections",
186 `{"items":[],"items_available":0}`)
189 func (s *stubServer) serveFooBarFileCollections() *reqTracker {
191 s.mux.HandleFunc("/arvados/v1/collections", func(w http.ResponseWriter, r *http.Request) {
194 if strings.Contains(r.Form.Get("filters"), `modified_at`) {
195 io.WriteString(w, `{"items_available":0,"items":[]}`)
197 io.WriteString(w, `{"items_available":3,"items":[
198 {"uuid":"zzzzz-4zz18-aaaaaaaaaaaaaaa","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
199 {"uuid":"zzzzz-4zz18-ehbhgtheo8909or","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
200 {"uuid":"zzzzz-4zz18-znfnqtbbv4spc3w","portable_data_hash":"1f4b0bc7583c2a7f9102c395f4ffc5e3+45","manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n","modified_at":"2014-02-03T17:22:54Z"}]}`)
206 func (s *stubServer) serveCollectionsButSkipOne() *reqTracker {
208 s.mux.HandleFunc("/arvados/v1/collections", func(w http.ResponseWriter, r *http.Request) {
211 if strings.Contains(r.Form.Get("filters"), `"modified_at","\u003c="`) {
212 io.WriteString(w, `{"items_available":3,"items":[]}`)
213 } else if strings.Contains(r.Form.Get("filters"), `"modified_at","\u003e`) {
214 io.WriteString(w, `{"items_available":0,"items":[]}`)
215 } else if strings.Contains(r.Form.Get("filters"), `"modified_at","="`) && strings.Contains(r.Form.Get("filters"), `"uuid","\u003e"`) {
216 io.WriteString(w, `{"items_available":0,"items":[]}`)
217 } else if strings.Contains(r.Form.Get("filters"), `"modified_at","=",null`) {
218 io.WriteString(w, `{"items_available":0,"items":[]}`)
220 io.WriteString(w, `{"items_available":2,"items":[
221 {"uuid":"zzzzz-4zz18-ehbhgtheo8909or","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
222 {"uuid":"zzzzz-4zz18-znfnqtbbv4spc3w","portable_data_hash":"1f4b0bc7583c2a7f9102c395f4ffc5e3+45","manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n","modified_at":"2014-02-03T17:22:54Z"}]}`)
228 func (s *stubServer) serveZeroKeepServices() *reqTracker {
229 return s.serveJSON("/arvados/v1/keep_services", arvados.KeepServiceList{})
232 func (s *stubServer) serveKeepServices(svcs []arvados.KeepService) *reqTracker {
233 return s.serveJSON("/arvados/v1/keep_services", arvados.KeepServiceList{
234 ItemsAvailable: len(svcs),
239 func (s *stubServer) serveJSON(path string, resp interface{}) *reqTracker {
241 s.mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
243 json.NewEncoder(w).Encode(resp)
248 func (s *stubServer) serveKeepstoreMounts() *reqTracker {
250 s.mux.HandleFunc("/mounts", func(w http.ResponseWriter, r *http.Request) {
252 json.NewEncoder(w).Encode(stubMounts[r.Host])
257 func (s *stubServer) serveKeepstoreIndexFoo4Bar1() *reqTracker {
259 s.mux.HandleFunc("/index/", func(w http.ResponseWriter, r *http.Request) {
261 if r.Host == "keep0.zzzzz.arvadosapi.com:25107" {
262 io.WriteString(w, "37b51d194a7513e45b56f6524f2d51f2+3 12345678\n")
264 fmt.Fprintf(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 %d\n\n", 12345678+count)
266 for _, mounts := range stubMounts {
267 for i, mnt := range mounts {
269 s.mux.HandleFunc(fmt.Sprintf("/mounts/%s/blocks", mnt.UUID), func(w http.ResponseWriter, r *http.Request) {
271 if i == 0 && r.Host == "keep0.zzzzz.arvadosapi.com:25107" {
272 io.WriteString(w, "37b51d194a7513e45b56f6524f2d51f2+3 12345678\n")
275 fmt.Fprintf(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 %d\n", 12345678+count)
284 func (s *stubServer) serveKeepstoreIndexFoo1() *reqTracker {
286 s.mux.HandleFunc("/index/", func(w http.ResponseWriter, r *http.Request) {
288 io.WriteString(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 12345678\n\n")
290 for _, mounts := range stubMounts {
291 for i, mnt := range mounts {
293 s.mux.HandleFunc(fmt.Sprintf("/mounts/%s/blocks", mnt.UUID), func(w http.ResponseWriter, r *http.Request) {
296 io.WriteString(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 12345678\n\n")
298 io.WriteString(w, "\n")
306 func (s *stubServer) serveKeepstoreTrash() *reqTracker {
307 return s.serveStatic("/trash", `{}`)
310 func (s *stubServer) serveKeepstorePull() *reqTracker {
311 return s.serveStatic("/pull", `{}`)
314 type runSuite struct {
316 config *arvados.Cluster
318 client *arvados.Client
321 func (s *runSuite) newServer(options *RunOptions) *Server {
325 RunOptions: *options,
326 Metrics: newMetrics(prometheus.NewRegistry()),
327 Logger: options.Logger,
328 Dumper: options.Dumper,
334 func (s *runSuite) SetUpTest(c *check.C) {
335 cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
336 c.Assert(err, check.Equals, nil)
337 s.config, err = cfg.GetCluster("")
338 c.Assert(err, check.Equals, nil)
339 s.db, err = sqlx.Open("postgres", s.config.PostgreSQL.Connection.String())
340 c.Assert(err, check.IsNil)
342 s.config.Collections.BalancePeriod = arvados.Duration(time.Second)
343 arvadostest.SetServiceURL(&s.config.Services.Keepbalance, "http://localhost:/")
345 s.client = &arvados.Client{
347 APIHost: "zzzzz.arvadosapi.com",
348 Client: s.stub.Start()}
350 s.stub.serveDiscoveryDoc()
354 func (s *runSuite) TearDownTest(c *check.C) {
358 func (s *runSuite) TestRefuseZeroCollections(c *check.C) {
359 defer arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil)
360 _, err := s.db.Exec(`delete from collections`)
361 c.Assert(err, check.IsNil)
365 Logger: ctxlog.TestLogger(c),
367 s.stub.serveCurrentUserAdmin()
368 s.stub.serveZeroCollections()
369 s.stub.serveKeepServices(stubServices)
370 s.stub.serveKeepstoreMounts()
371 s.stub.serveKeepstoreIndexFoo4Bar1()
372 trashReqs := s.stub.serveKeepstoreTrash()
373 pullReqs := s.stub.serveKeepstorePull()
374 srv := s.newServer(&opts)
375 _, err = srv.runOnce()
376 c.Check(err, check.ErrorMatches, "received zero collections")
377 c.Check(trashReqs.Count(), check.Equals, 4)
378 c.Check(pullReqs.Count(), check.Equals, 0)
381 func (s *runSuite) TestRefuseNonAdmin(c *check.C) {
385 Logger: ctxlog.TestLogger(c),
387 s.stub.serveCurrentUserNotAdmin()
388 s.stub.serveZeroCollections()
389 s.stub.serveKeepServices(stubServices)
390 s.stub.serveKeepstoreMounts()
391 trashReqs := s.stub.serveKeepstoreTrash()
392 pullReqs := s.stub.serveKeepstorePull()
393 srv := s.newServer(&opts)
394 _, err := srv.runOnce()
395 c.Check(err, check.ErrorMatches, "current user .* is not .* admin user")
396 c.Check(trashReqs.Count(), check.Equals, 0)
397 c.Check(pullReqs.Count(), check.Equals, 0)
400 func (s *runSuite) TestWriteLostBlocks(c *check.C) {
401 lostf, err := ioutil.TempFile("", "keep-balance-lost-blocks-test-")
402 c.Assert(err, check.IsNil)
403 s.config.Collections.BlobMissingReport = lostf.Name()
404 defer os.Remove(lostf.Name())
408 Logger: ctxlog.TestLogger(c),
410 s.stub.serveCurrentUserAdmin()
411 s.stub.serveFooBarFileCollections()
412 s.stub.serveKeepServices(stubServices)
413 s.stub.serveKeepstoreMounts()
414 s.stub.serveKeepstoreIndexFoo1()
415 s.stub.serveKeepstoreTrash()
416 s.stub.serveKeepstorePull()
417 srv := s.newServer(&opts)
418 c.Assert(err, check.IsNil)
419 _, err = srv.runOnce()
420 c.Check(err, check.IsNil)
421 lost, err := ioutil.ReadFile(lostf.Name())
422 c.Assert(err, check.IsNil)
423 c.Check(string(lost), check.Matches, `(?ms).*37b51d194a7513e45b56f6524f2d51f2.* fa7aeb5140e2848d39b416daeef4ffc5\+45.*`)
426 func (s *runSuite) TestDryRun(c *check.C) {
430 Logger: ctxlog.TestLogger(c),
432 s.stub.serveCurrentUserAdmin()
433 collReqs := s.stub.serveFooBarFileCollections()
434 s.stub.serveKeepServices(stubServices)
435 s.stub.serveKeepstoreMounts()
436 s.stub.serveKeepstoreIndexFoo4Bar1()
437 trashReqs := s.stub.serveKeepstoreTrash()
438 pullReqs := s.stub.serveKeepstorePull()
439 srv := s.newServer(&opts)
440 bal, err := srv.runOnce()
441 c.Check(err, check.IsNil)
442 for _, req := range collReqs.reqs {
443 c.Check(req.Form.Get("include_trash"), check.Equals, "true")
444 c.Check(req.Form.Get("include_old_versions"), check.Equals, "true")
446 c.Check(trashReqs.Count(), check.Equals, 0)
447 c.Check(pullReqs.Count(), check.Equals, 0)
448 c.Check(bal.stats.pulls, check.Not(check.Equals), 0)
449 c.Check(bal.stats.underrep.replicas, check.Not(check.Equals), 0)
450 c.Check(bal.stats.overrep.replicas, check.Not(check.Equals), 0)
453 func (s *runSuite) TestCommit(c *check.C) {
454 s.config.Collections.BlobMissingReport = c.MkDir() + "/keep-balance-lost-blocks-test-"
455 s.config.ManagementToken = "xyzzy"
459 Logger: ctxlog.TestLogger(c),
460 Dumper: ctxlog.TestLogger(c),
462 s.stub.serveCurrentUserAdmin()
463 s.stub.serveFooBarFileCollections()
464 s.stub.serveKeepServices(stubServices)
465 s.stub.serveKeepstoreMounts()
466 s.stub.serveKeepstoreIndexFoo4Bar1()
467 trashReqs := s.stub.serveKeepstoreTrash()
468 pullReqs := s.stub.serveKeepstorePull()
469 srv := s.newServer(&opts)
470 bal, err := srv.runOnce()
471 c.Check(err, check.IsNil)
472 c.Check(trashReqs.Count(), check.Equals, 8)
473 c.Check(pullReqs.Count(), check.Equals, 4)
474 // "foo" block is overreplicated by 2
475 c.Check(bal.stats.trashes, check.Equals, 2)
476 // "bar" block is underreplicated by 1, and its only copy is
477 // in a poor rendezvous position
478 c.Check(bal.stats.pulls, check.Equals, 2)
480 lost, err := ioutil.ReadFile(s.config.Collections.BlobMissingReport)
481 c.Assert(err, check.IsNil)
482 c.Check(string(lost), check.Not(check.Matches), `(?ms).*acbd18db4cc2f85cedef654fccc4a4d8.*`)
484 buf, err := s.getMetrics(c, srv)
485 c.Check(err, check.IsNil)
486 bufstr := buf.String()
487 c.Check(bufstr, check.Matches, `(?ms).*\narvados_keep_total_bytes 15\n.*`)
488 c.Check(bufstr, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_sum [0-9\.]+\n.*`)
489 c.Check(bufstr, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count 1\n.*`)
490 c.Check(bufstr, check.Matches, `(?ms).*\narvados_keep_dedup_byte_ratio [1-9].*`)
491 c.Check(bufstr, check.Matches, `(?ms).*\narvados_keep_dedup_block_ratio [1-9].*`)
494 func (s *runSuite) TestRunForever(c *check.C) {
495 s.config.ManagementToken = "xyzzy"
499 Logger: ctxlog.TestLogger(c),
500 Dumper: ctxlog.TestLogger(c),
502 s.stub.serveCurrentUserAdmin()
503 s.stub.serveFooBarFileCollections()
504 s.stub.serveKeepServices(stubServices)
505 s.stub.serveKeepstoreMounts()
506 s.stub.serveKeepstoreIndexFoo4Bar1()
507 trashReqs := s.stub.serveKeepstoreTrash()
508 pullReqs := s.stub.serveKeepstorePull()
510 stop := make(chan interface{})
511 s.config.Collections.BalancePeriod = arvados.Duration(time.Millisecond)
512 srv := s.newServer(&opts)
514 done := make(chan bool)
520 // Each run should send 4 pull lists + 4 trash lists. The
521 // first run should also send 4 empty trash lists at
522 // startup. We should complete all four runs in much less than
524 for t0 := time.Now(); pullReqs.Count() < 16 && time.Since(t0) < 10*time.Second; {
525 time.Sleep(time.Millisecond)
529 c.Check(pullReqs.Count() >= 16, check.Equals, true)
530 c.Check(trashReqs.Count(), check.Equals, pullReqs.Count()+4)
532 buf, err := s.getMetrics(c, srv)
533 c.Check(err, check.IsNil)
534 c.Check(buf, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count `+fmt.Sprintf("%d", pullReqs.Count()/4)+`\n.*`)
537 func (s *runSuite) getMetrics(c *check.C, srv *Server) (*bytes.Buffer, error) {
538 mfs, err := srv.Metrics.reg.Gather()
544 for _, mf := range mfs {
545 if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil {