1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
19 "git.curoverse.com/arvados.git/sdk/go/arvados"
20 "github.com/sirupsen/logrus"
21 check "gopkg.in/check.v1"
24 var _ = check.Suite(&runSuite{})
26 type reqTracker struct {
31 func (rt *reqTracker) Count() int {
37 func (rt *reqTracker) Add(req *http.Request) int {
40 rt.reqs = append(rt.reqs, *req)
44 var stubServices = []arvados.KeepService{
46 UUID: "zzzzz-bi6l4-000000000000000",
47 ServiceHost: "keep0.zzzzz.arvadosapi.com",
49 ServiceSSLFlag: false,
53 UUID: "zzzzz-bi6l4-000000000000001",
54 ServiceHost: "keep1.zzzzz.arvadosapi.com",
56 ServiceSSLFlag: false,
60 UUID: "zzzzz-bi6l4-000000000000002",
61 ServiceHost: "keep2.zzzzz.arvadosapi.com",
63 ServiceSSLFlag: false,
67 UUID: "zzzzz-bi6l4-000000000000003",
68 ServiceHost: "keep3.zzzzz.arvadosapi.com",
70 ServiceSSLFlag: false,
74 UUID: "zzzzz-bi6l4-h0a0xwut9qa6g3a",
75 ServiceHost: "keep.zzzzz.arvadosapi.com",
82 var stubMounts = map[string][]arvados.KeepMount{
83 "keep0.zzzzz.arvadosapi.com:25107": {{
84 UUID: "zzzzz-ivpuk-000000000000000",
85 DeviceID: "keep0-vol0",
87 "keep1.zzzzz.arvadosapi.com:25107": {{
88 UUID: "zzzzz-ivpuk-100000000000000",
89 DeviceID: "keep1-vol0",
91 "keep2.zzzzz.arvadosapi.com:25107": {{
92 UUID: "zzzzz-ivpuk-200000000000000",
93 DeviceID: "keep2-vol0",
95 "keep3.zzzzz.arvadosapi.com:25107": {{
96 UUID: "zzzzz-ivpuk-300000000000000",
97 DeviceID: "keep3-vol0",
101 // stubServer is an HTTP transport that intercepts and processes all
102 // requests using its own handlers.
103 type stubServer struct {
108 logf func(string, ...interface{})
111 // Start initializes the stub server and returns an *http.Client that
112 // uses the stub server to handle all requests.
114 // A stubServer that has been started should eventually be shut down
116 func (s *stubServer) Start() *http.Client {
117 // Set up a config.Client that forwards all requests to s.mux
118 // via s.srv. Test cases will attach handlers to s.mux to get
119 // the desired responses.
120 s.mux = http.NewServeMux()
121 s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
125 w.Header().Set("Content-Type", "application/json")
126 s.mux.ServeHTTP(w, r)
128 return &http.Client{Transport: s}
131 func (s *stubServer) RoundTrip(req *http.Request) (*http.Response, error) {
132 w := httptest.NewRecorder()
133 s.mux.ServeHTTP(w, req)
134 return &http.Response{
136 Status: fmt.Sprintf("%d %s", w.Code, http.StatusText(w.Code)),
138 Body: ioutil.NopCloser(w.Body)}, nil
141 // Close releases resources used by the server.
142 func (s *stubServer) Close() {
146 func (s *stubServer) serveStatic(path, data string) *reqTracker {
148 s.mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
151 ioutil.ReadAll(r.Body)
154 io.WriteString(w, data)
159 func (s *stubServer) serveCurrentUserAdmin() *reqTracker {
160 return s.serveStatic("/arvados/v1/users/current",
161 `{"uuid":"zzzzz-tpzed-000000000000000","is_admin":true,"is_active":true}`)
164 func (s *stubServer) serveCurrentUserNotAdmin() *reqTracker {
165 return s.serveStatic("/arvados/v1/users/current",
166 `{"uuid":"zzzzz-tpzed-000000000000000","is_admin":false,"is_active":true}`)
169 func (s *stubServer) serveDiscoveryDoc() *reqTracker {
170 return s.serveStatic("/discovery/v1/apis/arvados/v1/rest",
171 `{"defaultCollectionReplication":2}`)
174 func (s *stubServer) serveZeroCollections() *reqTracker {
175 return s.serveStatic("/arvados/v1/collections",
176 `{"items":[],"items_available":0}`)
179 func (s *stubServer) serveFooBarFileCollections() *reqTracker {
181 s.mux.HandleFunc("/arvados/v1/collections", func(w http.ResponseWriter, r *http.Request) {
184 if strings.Contains(r.Form.Get("filters"), `modified_at`) {
185 io.WriteString(w, `{"items_available":0,"items":[]}`)
187 io.WriteString(w, `{"items_available":3,"items":[
188 {"uuid":"zzzzz-4zz18-aaaaaaaaaaaaaaa","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
189 {"uuid":"zzzzz-4zz18-ehbhgtheo8909or","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
190 {"uuid":"zzzzz-4zz18-znfnqtbbv4spc3w","portable_data_hash":"1f4b0bc7583c2a7f9102c395f4ffc5e3+45","manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n","modified_at":"2014-02-03T17:22:54Z"}]}`)
196 func (s *stubServer) serveCollectionsButSkipOne() *reqTracker {
198 s.mux.HandleFunc("/arvados/v1/collections", func(w http.ResponseWriter, r *http.Request) {
201 if strings.Contains(r.Form.Get("filters"), `"modified_at","\u003c="`) {
202 io.WriteString(w, `{"items_available":3,"items":[]}`)
203 } else if strings.Contains(r.Form.Get("filters"), `"modified_at","\u003e`) {
204 io.WriteString(w, `{"items_available":0,"items":[]}`)
205 } else if strings.Contains(r.Form.Get("filters"), `"modified_at","="`) && strings.Contains(r.Form.Get("filters"), `"uuid","\u003e"`) {
206 io.WriteString(w, `{"items_available":0,"items":[]}`)
207 } else if strings.Contains(r.Form.Get("filters"), `"modified_at","=",null`) {
208 io.WriteString(w, `{"items_available":0,"items":[]}`)
210 io.WriteString(w, `{"items_available":2,"items":[
211 {"uuid":"zzzzz-4zz18-ehbhgtheo8909or","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
212 {"uuid":"zzzzz-4zz18-znfnqtbbv4spc3w","portable_data_hash":"1f4b0bc7583c2a7f9102c395f4ffc5e3+45","manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n","modified_at":"2014-02-03T17:22:54Z"}]}`)
218 func (s *stubServer) serveZeroKeepServices() *reqTracker {
219 return s.serveJSON("/arvados/v1/keep_services", arvados.KeepServiceList{})
222 func (s *stubServer) serveKeepServices(svcs []arvados.KeepService) *reqTracker {
223 return s.serveJSON("/arvados/v1/keep_services", arvados.KeepServiceList{
224 ItemsAvailable: len(svcs),
229 func (s *stubServer) serveJSON(path string, resp interface{}) *reqTracker {
231 s.mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
233 json.NewEncoder(w).Encode(resp)
238 func (s *stubServer) serveKeepstoreMounts() *reqTracker {
240 s.mux.HandleFunc("/mounts", func(w http.ResponseWriter, r *http.Request) {
242 json.NewEncoder(w).Encode(stubMounts[r.Host])
247 func (s *stubServer) serveKeepstoreIndexFoo4Bar1() *reqTracker {
249 s.mux.HandleFunc("/index/", func(w http.ResponseWriter, r *http.Request) {
251 if r.Host == "keep0.zzzzz.arvadosapi.com:25107" {
252 io.WriteString(w, "37b51d194a7513e45b56f6524f2d51f2+3 12345678\n")
254 fmt.Fprintf(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 %d\n\n", 12345678+count)
256 for _, mounts := range stubMounts {
257 for i, mnt := range mounts {
259 s.mux.HandleFunc(fmt.Sprintf("/mounts/%s/blocks", mnt.UUID), func(w http.ResponseWriter, r *http.Request) {
261 if i == 0 && r.Host == "keep0.zzzzz.arvadosapi.com:25107" {
262 io.WriteString(w, "37b51d194a7513e45b56f6524f2d51f2+3 12345678\n")
265 fmt.Fprintf(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 %d\n", 12345678+count)
274 func (s *stubServer) serveKeepstoreIndexFoo1() *reqTracker {
276 s.mux.HandleFunc("/index/", func(w http.ResponseWriter, r *http.Request) {
278 io.WriteString(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 12345678\n\n")
280 for _, mounts := range stubMounts {
281 for i, mnt := range mounts {
283 s.mux.HandleFunc(fmt.Sprintf("/mounts/%s/blocks", mnt.UUID), func(w http.ResponseWriter, r *http.Request) {
286 io.WriteString(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 12345678\n\n")
288 io.WriteString(w, "\n")
296 func (s *stubServer) serveKeepstoreTrash() *reqTracker {
297 return s.serveStatic("/trash", `{}`)
300 func (s *stubServer) serveKeepstorePull() *reqTracker {
301 return s.serveStatic("/pull", `{}`)
304 type runSuite struct {
309 // make a log.Logger that writes to the current test's c.Log().
310 func (s *runSuite) logger(c *check.C) *logrus.Logger {
313 buf := make([]byte, 10000)
315 n, err := r.Read(buf)
317 if buf[n-1] == '\n' {
320 c.Log(string(buf[:n]))
327 logger := logrus.New()
332 func (s *runSuite) SetUpTest(c *check.C) {
334 Client: arvados.Client{
336 APIHost: "zzzzz.arvadosapi.com",
337 Client: s.stub.Start()},
338 KeepServiceTypes: []string{"disk"},
339 RunPeriod: arvados.Duration(time.Second),
341 s.stub.serveDiscoveryDoc()
345 func (s *runSuite) TearDownTest(c *check.C) {
349 func (s *runSuite) TestRefuseZeroCollections(c *check.C) {
355 s.stub.serveCurrentUserAdmin()
356 s.stub.serveZeroCollections()
357 s.stub.serveKeepServices(stubServices)
358 s.stub.serveKeepstoreMounts()
359 s.stub.serveKeepstoreIndexFoo4Bar1()
360 trashReqs := s.stub.serveKeepstoreTrash()
361 pullReqs := s.stub.serveKeepstorePull()
362 srv, err := NewServer(s.config, opts)
363 c.Assert(err, check.IsNil)
365 c.Check(err, check.ErrorMatches, "received zero collections")
366 c.Check(trashReqs.Count(), check.Equals, 4)
367 c.Check(pullReqs.Count(), check.Equals, 0)
370 func (s *runSuite) TestServiceTypes(c *check.C) {
376 s.config.KeepServiceTypes = []string{"unlisted-type"}
377 s.stub.serveCurrentUserAdmin()
378 s.stub.serveFooBarFileCollections()
379 s.stub.serveKeepServices(stubServices)
380 s.stub.serveKeepstoreMounts()
381 indexReqs := s.stub.serveKeepstoreIndexFoo4Bar1()
382 trashReqs := s.stub.serveKeepstoreTrash()
383 srv, err := NewServer(s.config, opts)
384 c.Assert(err, check.IsNil)
386 c.Check(err, check.IsNil)
387 c.Check(indexReqs.Count(), check.Equals, 0)
388 c.Check(trashReqs.Count(), check.Equals, 0)
391 func (s *runSuite) TestRefuseNonAdmin(c *check.C) {
397 s.stub.serveCurrentUserNotAdmin()
398 s.stub.serveZeroCollections()
399 s.stub.serveKeepServices(stubServices)
400 s.stub.serveKeepstoreMounts()
401 trashReqs := s.stub.serveKeepstoreTrash()
402 pullReqs := s.stub.serveKeepstorePull()
403 srv, err := NewServer(s.config, opts)
404 c.Assert(err, check.IsNil)
406 c.Check(err, check.ErrorMatches, "current user .* is not .* admin user")
407 c.Check(trashReqs.Count(), check.Equals, 0)
408 c.Check(pullReqs.Count(), check.Equals, 0)
411 func (s *runSuite) TestDetectSkippedCollections(c *check.C) {
417 s.stub.serveCurrentUserAdmin()
418 s.stub.serveCollectionsButSkipOne()
419 s.stub.serveKeepServices(stubServices)
420 s.stub.serveKeepstoreMounts()
421 s.stub.serveKeepstoreIndexFoo4Bar1()
422 trashReqs := s.stub.serveKeepstoreTrash()
423 pullReqs := s.stub.serveKeepstorePull()
424 srv, err := NewServer(s.config, opts)
425 c.Assert(err, check.IsNil)
427 c.Check(err, check.ErrorMatches, `Retrieved 2 collections with modtime <= .* but server now reports there are 3 collections.*`)
428 c.Check(trashReqs.Count(), check.Equals, 4)
429 c.Check(pullReqs.Count(), check.Equals, 0)
432 func (s *runSuite) TestWriteLostBlocks(c *check.C) {
433 lostf, err := ioutil.TempFile("", "keep-balance-lost-blocks-test-")
434 c.Assert(err, check.IsNil)
435 s.config.LostBlocksFile = lostf.Name()
436 defer os.Remove(lostf.Name())
442 s.stub.serveCurrentUserAdmin()
443 s.stub.serveFooBarFileCollections()
444 s.stub.serveKeepServices(stubServices)
445 s.stub.serveKeepstoreMounts()
446 s.stub.serveKeepstoreIndexFoo1()
447 s.stub.serveKeepstoreTrash()
448 s.stub.serveKeepstorePull()
449 srv, err := NewServer(s.config, opts)
450 c.Assert(err, check.IsNil)
452 c.Check(err, check.IsNil)
453 lost, err := ioutil.ReadFile(lostf.Name())
454 c.Assert(err, check.IsNil)
455 c.Check(string(lost), check.Equals, "37b51d194a7513e45b56f6524f2d51f2\n")
458 func (s *runSuite) TestDryRun(c *check.C) {
464 s.stub.serveCurrentUserAdmin()
465 collReqs := s.stub.serveFooBarFileCollections()
466 s.stub.serveKeepServices(stubServices)
467 s.stub.serveKeepstoreMounts()
468 s.stub.serveKeepstoreIndexFoo4Bar1()
469 trashReqs := s.stub.serveKeepstoreTrash()
470 pullReqs := s.stub.serveKeepstorePull()
471 srv, err := NewServer(s.config, opts)
472 c.Assert(err, check.IsNil)
473 bal, err := srv.Run()
474 c.Check(err, check.IsNil)
475 for _, req := range collReqs.reqs {
476 c.Check(req.Form.Get("include_trash"), check.Equals, "true")
477 c.Check(req.Form.Get("include_old_versions"), check.Equals, "true")
479 c.Check(trashReqs.Count(), check.Equals, 0)
480 c.Check(pullReqs.Count(), check.Equals, 0)
481 c.Check(bal.stats.pulls, check.Not(check.Equals), 0)
482 c.Check(bal.stats.underrep.replicas, check.Not(check.Equals), 0)
483 c.Check(bal.stats.overrep.replicas, check.Not(check.Equals), 0)
486 func (s *runSuite) TestCommit(c *check.C) {
487 lostf, err := ioutil.TempFile("", "keep-balance-lost-blocks-test-")
488 c.Assert(err, check.IsNil)
489 s.config.LostBlocksFile = lostf.Name()
490 defer os.Remove(lostf.Name())
492 s.config.Listen = ":"
493 s.config.ManagementToken = "xyzzy"
500 s.stub.serveCurrentUserAdmin()
501 s.stub.serveFooBarFileCollections()
502 s.stub.serveKeepServices(stubServices)
503 s.stub.serveKeepstoreMounts()
504 s.stub.serveKeepstoreIndexFoo4Bar1()
505 trashReqs := s.stub.serveKeepstoreTrash()
506 pullReqs := s.stub.serveKeepstorePull()
507 srv, err := NewServer(s.config, opts)
508 c.Assert(err, check.IsNil)
509 bal, err := srv.Run()
510 c.Check(err, check.IsNil)
511 c.Check(trashReqs.Count(), check.Equals, 8)
512 c.Check(pullReqs.Count(), check.Equals, 4)
513 // "foo" block is overreplicated by 2
514 c.Check(bal.stats.trashes, check.Equals, 2)
515 // "bar" block is underreplicated by 1, and its only copy is
516 // in a poor rendezvous position
517 c.Check(bal.stats.pulls, check.Equals, 2)
519 lost, err := ioutil.ReadFile(lostf.Name())
520 c.Assert(err, check.IsNil)
521 c.Check(string(lost), check.Equals, "")
523 metrics := s.getMetrics(c, srv)
524 c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_total_bytes 15\n.*`)
525 c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_sum [0-9\.]+\n.*`)
526 c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count 1\n.*`)
527 c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_dedup_byte_ratio 1\.5\n.*`)
528 c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_dedup_block_ratio 1\.5\n.*`)
531 func (s *runSuite) TestRunForever(c *check.C) {
532 s.config.Listen = ":"
533 s.config.ManagementToken = "xyzzy"
540 s.stub.serveCurrentUserAdmin()
541 s.stub.serveFooBarFileCollections()
542 s.stub.serveKeepServices(stubServices)
543 s.stub.serveKeepstoreMounts()
544 s.stub.serveKeepstoreIndexFoo4Bar1()
545 trashReqs := s.stub.serveKeepstoreTrash()
546 pullReqs := s.stub.serveKeepstorePull()
548 stop := make(chan interface{})
549 s.config.RunPeriod = arvados.Duration(time.Millisecond)
550 srv, err := NewServer(s.config, opts)
551 c.Assert(err, check.IsNil)
553 done := make(chan bool)
559 // Each run should send 4 pull lists + 4 trash lists. The
560 // first run should also send 4 empty trash lists at
561 // startup. We should complete all four runs in much less than
563 for t0 := time.Now(); pullReqs.Count() < 16 && time.Since(t0) < 10*time.Second; {
564 time.Sleep(time.Millisecond)
568 c.Check(pullReqs.Count() >= 16, check.Equals, true)
569 c.Check(trashReqs.Count(), check.Equals, pullReqs.Count()+4)
570 c.Check(s.getMetrics(c, srv), check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count `+fmt.Sprintf("%d", pullReqs.Count()/4)+`\n.*`)
573 func (s *runSuite) getMetrics(c *check.C, srv *Server) string {
574 resp, err := http.Get("http://" + srv.listening + "/metrics")
575 c.Assert(err, check.IsNil)
576 c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
578 resp, err = http.Get("http://" + srv.listening + "/metrics?api_token=xyzzy")
579 c.Assert(err, check.IsNil)
580 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
581 buf, err := ioutil.ReadAll(resp.Body)
582 c.Check(err, check.IsNil)