1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
21 "git.arvados.org/arvados.git/lib/config"
22 "git.arvados.org/arvados.git/sdk/go/arvados"
23 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
24 "git.arvados.org/arvados.git/sdk/go/arvadostest"
25 "git.arvados.org/arvados.git/sdk/go/ctxlog"
26 "git.arvados.org/arvados.git/sdk/go/keepclient"
27 log "github.com/sirupsen/logrus"
32 // Gocheck boilerplate
33 func Test(t *testing.T) {
37 // Gocheck boilerplate
38 var _ = Suite(&ServerRequiredSuite{})
40 // Tests that require the Keep server running
41 type ServerRequiredSuite struct{}
43 // Gocheck boilerplate
44 var _ = Suite(&ServerRequiredConfigYmlSuite{})
46 // Tests that require the Keep servers running as defined in config.yml
47 type ServerRequiredConfigYmlSuite struct{}
49 // Gocheck boilerplate
50 var _ = Suite(&NoKeepServerSuite{})
52 // Test with no keepserver to simulate errors
53 type NoKeepServerSuite struct{}
55 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
57 // Wait (up to 1 second) for keepproxy to listen on a port. This
58 // avoids a race condition where we hit a "connection refused" error
59 // because we start testing the proxy too soon.
60 func waitForListener() {
64 for i := 0; listener == nil && i < 10000; i += ms {
65 time.Sleep(ms * time.Millisecond)
68 panic("Timed out waiting for listener to start")
72 func closeListener() {
78 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
79 arvadostest.StartAPI()
80 arvadostest.StartKeep(2, false)
83 func (s *ServerRequiredSuite) SetUpTest(c *C) {
84 arvadostest.ResetEnv()
87 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
88 arvadostest.StopKeep(2)
92 func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
93 arvadostest.StartAPI()
94 // config.yml defines 4 keepstores
95 arvadostest.StartKeep(4, false)
98 func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
99 arvadostest.ResetEnv()
102 func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
103 arvadostest.StopKeep(4)
104 arvadostest.StopAPI()
107 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
108 arvadostest.StartAPI()
109 // We need API to have some keep services listed, but the
110 // services themselves should be unresponsive.
111 arvadostest.StartKeep(2, false)
112 arvadostest.StopKeep(2)
115 func (s *NoKeepServerSuite) SetUpTest(c *C) {
116 arvadostest.ResetEnv()
119 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
120 arvadostest.StopAPI()
123 func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool) *keepclient.KeepClient {
124 cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
125 c.Assert(err, Equals, nil)
126 cluster, err := cfg.GetCluster("")
127 c.Assert(err, Equals, nil)
129 if !loadKeepstoresFromConfig {
130 // Do not load Keepstore InternalURLs from the config file
131 cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
134 cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: ":0"}: {}}
138 run(log.New(), cluster)
139 defer closeListener()
143 client := arvados.NewClientFromEnv()
144 arv, err := arvadosclient.New(client)
145 c.Assert(err, Equals, nil)
146 if bogusClientToken {
147 arv.ApiToken = "bogus-token"
149 kc := keepclient.New(arv)
150 sr := map[string]string{
151 TestProxyUUID: "http://" + listener.Addr().String(),
153 kc.SetServiceRoots(sr, sr, sr)
154 kc.Arvados.External = true
159 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
160 runProxy(c, false, false)
161 defer closeListener()
163 req, err := http.NewRequest("POST",
164 "http://"+listener.Addr().String()+"/",
165 strings.NewReader("TestViaHeader"))
166 c.Assert(err, Equals, nil)
167 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
168 resp, err := (&http.Client{}).Do(req)
169 c.Assert(err, Equals, nil)
170 c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
171 c.Assert(resp.StatusCode, Equals, http.StatusOK)
172 locator, err := ioutil.ReadAll(resp.Body)
173 c.Assert(err, Equals, nil)
176 req, err = http.NewRequest("GET",
177 "http://"+listener.Addr().String()+"/"+string(locator),
179 c.Assert(err, Equals, nil)
180 resp, err = (&http.Client{}).Do(req)
181 c.Assert(err, Equals, nil)
182 c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
186 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
187 kc := runProxy(c, false, false)
188 defer closeListener()
190 sr := map[string]string{
191 TestProxyUUID: "http://" + listener.Addr().String(),
193 router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
195 content := []byte("TestLoopDetection")
196 _, _, err := kc.PutB(content)
197 c.Check(err, ErrorMatches, `.*loop detected.*`)
199 hash := fmt.Sprintf("%x", md5.Sum(content))
200 _, _, _, err = kc.Get(hash)
201 c.Check(err, ErrorMatches, `.*loop detected.*`)
204 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
205 kc := runProxy(c, false, false)
206 defer closeListener()
208 // Set up fake keepstore to record request headers
210 ts := httptest.NewServer(http.HandlerFunc(
211 func(w http.ResponseWriter, r *http.Request) {
213 http.Error(w, "Error", http.StatusInternalServerError)
217 // Point keepproxy router's keepclient to the fake keepstore
218 sr := map[string]string{
219 TestProxyUUID: ts.URL,
221 router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
223 // Set up client to ask for storage classes to keepproxy
224 kc.StorageClasses = []string{"secure"}
225 content := []byte("Very important data")
226 _, _, err := kc.PutB(content)
228 c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
231 func (s *ServerRequiredSuite) TestStorageClassesConfirmedHeader(c *C) {
232 runProxy(c, false, false)
233 defer closeListener()
235 content := []byte("foo")
236 hash := fmt.Sprintf("%x", md5.Sum(content))
237 client := &http.Client{}
239 req, err := http.NewRequest("PUT",
240 fmt.Sprintf("http://%s/%s", listener.Addr().String(), hash),
241 bytes.NewReader(content))
243 req.Header.Set("X-Keep-Storage-Classes", "default")
244 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
245 req.Header.Set("Content-Type", "application/octet-stream")
247 resp, err := client.Do(req)
249 c.Assert(resp.StatusCode, Equals, http.StatusOK)
250 c.Assert(resp.Header.Get("X-Keep-Storage-Classes-Confirmed"), Equals, "default=2")
253 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
254 kc := runProxy(c, false, false)
255 defer closeListener()
257 content := []byte("TestDesiredReplicas")
258 hash := fmt.Sprintf("%x", md5.Sum(content))
260 for _, kc.Want_replicas = range []int{0, 1, 2} {
261 locator, rep, err := kc.PutB(content)
262 c.Check(err, Equals, nil)
263 c.Check(rep, Equals, kc.Want_replicas)
265 c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
270 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
271 kc := runProxy(c, false, false)
272 defer closeListener()
274 content := []byte("TestPutWrongContentLength")
275 hash := fmt.Sprintf("%x", md5.Sum(content))
277 // If we use http.Client to send these requests to the network
278 // server we just started, the Go http library automatically
279 // fixes the invalid Content-Length header. In order to test
280 // our server behavior, we have to call the handler directly
281 // using an httptest.ResponseRecorder.
282 rtr := MakeRESTRouter(kc, 10*time.Second, "")
284 type testcase struct {
289 for _, t := range []testcase{
290 {"1", http.StatusBadRequest},
291 {"", http.StatusLengthRequired},
292 {"-1", http.StatusLengthRequired},
293 {"abcdef", http.StatusLengthRequired},
295 req, err := http.NewRequest("PUT",
296 fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
297 bytes.NewReader(content))
299 req.Header.Set("Content-Length", t.sendLength)
300 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
301 req.Header.Set("Content-Type", "application/octet-stream")
303 resp := httptest.NewRecorder()
304 rtr.ServeHTTP(resp, req)
305 c.Check(resp.Code, Equals, t.expectStatus)
309 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
310 kc := runProxy(c, false, false)
311 defer closeListener()
312 router.(*proxyHandler).timeout = time.Nanosecond
314 buf := make([]byte, 1<<20)
316 var wg sync.WaitGroup
317 for i := 0; i < 128; i++ {
324 done := make(chan bool)
331 case <-time.After(10 * time.Second):
336 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
337 kc := runProxy(c, false, false)
338 defer closeListener()
340 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
344 _, _, err := kc.Ask(hash)
345 c.Check(err, Equals, keepclient.BlockNotFound)
346 c.Log("Finished Ask (expected BlockNotFound)")
350 reader, _, _, err := kc.Get(hash)
351 c.Check(reader, Equals, nil)
352 c.Check(err, Equals, keepclient.BlockNotFound)
353 c.Log("Finished Get (expected BlockNotFound)")
356 // Note in bug #5309 among other errors keepproxy would set
357 // Content-Length incorrectly on the 404 BlockNotFound response, this
358 // would result in a protocol violation that would prevent reuse of the
359 // connection, which would manifest by the next attempt to use the
360 // connection (in this case the PutB below) failing. So to test for
361 // that bug it's necessary to trigger an error response (such as
362 // BlockNotFound) and then do something else with the same httpClient
368 hash2, rep, err = kc.PutB([]byte("foo"))
369 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
370 c.Check(rep, Equals, 2)
371 c.Check(err, Equals, nil)
372 c.Log("Finished PutB (expected success)")
376 blocklen, _, err := kc.Ask(hash2)
377 c.Assert(err, Equals, nil)
378 c.Check(blocklen, Equals, int64(3))
379 c.Log("Finished Ask (expected success)")
383 reader, blocklen, _, err := kc.Get(hash2)
384 c.Assert(err, Equals, nil)
385 all, err := ioutil.ReadAll(reader)
387 c.Check(all, DeepEquals, []byte("foo"))
388 c.Check(blocklen, Equals, int64(3))
389 c.Log("Finished Get (expected success)")
395 hash2, rep, err = kc.PutB([]byte(""))
396 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
397 c.Check(rep, Equals, 2)
398 c.Check(err, Equals, nil)
399 c.Log("Finished PutB zero block")
403 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
404 c.Assert(err, Equals, nil)
405 all, err := ioutil.ReadAll(reader)
407 c.Check(all, DeepEquals, []byte(""))
408 c.Check(blocklen, Equals, int64(0))
409 c.Log("Finished Get zero block")
413 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
414 kc := runProxy(c, true, false)
415 defer closeListener()
417 hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
419 _, _, err := kc.Ask(hash)
420 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
422 hash2, rep, err := kc.PutB([]byte("bar"))
423 c.Check(hash2, Equals, "")
424 c.Check(rep, Equals, 0)
425 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
427 blocklen, _, err := kc.Ask(hash)
428 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
429 c.Check(err, ErrorMatches, ".*not found.*")
430 c.Check(blocklen, Equals, int64(0))
432 _, blocklen, _, err = kc.Get(hash)
433 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
434 c.Check(err, ErrorMatches, ".*not found.*")
435 c.Check(blocklen, Equals, int64(0))
439 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
440 runProxy(c, false, false)
441 defer closeListener()
444 client := http.Client{}
445 req, err := http.NewRequest("OPTIONS",
446 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
449 req.Header.Add("Access-Control-Request-Method", "PUT")
450 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
451 resp, err := client.Do(req)
452 c.Check(err, Equals, nil)
453 c.Check(resp.StatusCode, Equals, 200)
454 body, err := ioutil.ReadAll(resp.Body)
456 c.Check(string(body), Equals, "")
457 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
458 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
462 resp, err := http.Get(
463 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
464 c.Check(err, Equals, nil)
465 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
466 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
470 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
471 runProxy(c, false, false)
472 defer closeListener()
475 client := http.Client{}
476 req, err := http.NewRequest("POST",
477 "http://"+listener.Addr().String()+"/",
478 strings.NewReader("qux"))
480 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
481 req.Header.Add("Content-Type", "application/octet-stream")
482 resp, err := client.Do(req)
483 c.Check(err, Equals, nil)
484 body, err := ioutil.ReadAll(resp.Body)
485 c.Check(err, Equals, nil)
486 c.Check(string(body), Matches,
487 fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
491 func (s *ServerRequiredSuite) TestStripHint(c *C) {
492 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
494 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
495 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
497 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
498 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
500 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
501 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
503 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
508 // Put one block, with 2 replicas
509 // With no prefix (expect the block locator, twice)
510 // With an existing prefix (expect the block locator, twice)
511 // With a valid but non-existing prefix (expect "\n")
512 // With an invalid prefix (expect error)
513 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
514 getIndexWorker(c, false)
519 // Put one block, with 2 replicas
520 // With no prefix (expect the block locator, twice)
521 // With an existing prefix (expect the block locator, twice)
522 // With a valid but non-existing prefix (expect "\n")
523 // With an invalid prefix (expect error)
524 func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
525 getIndexWorker(c, true)
528 func getIndexWorker(c *C, useConfig bool) {
529 kc := runProxy(c, false, useConfig)
530 defer closeListener()
532 // Put "index-data" blocks
533 data := []byte("index-data")
534 hash := fmt.Sprintf("%x", md5.Sum(data))
536 hash2, rep, err := kc.PutB(data)
537 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
538 c.Check(rep, Equals, 2)
539 c.Check(err, Equals, nil)
541 reader, blocklen, _, err := kc.Get(hash)
543 c.Check(blocklen, Equals, int64(10))
544 all, err := ioutil.ReadAll(reader)
546 c.Check(all, DeepEquals, data)
548 // Put some more blocks
549 _, _, err = kc.PutB([]byte("some-more-index-data"))
552 kc.Arvados.ApiToken = arvadostest.SystemRootToken
555 for _, spec := range []struct {
560 {"", true, true}, // with no prefix
561 {hash[:3], true, false}, // with matching prefix
562 {"abcdef", false, false}, // with no such prefix
564 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
565 c.Assert(err, Equals, nil)
566 indexResp, err := ioutil.ReadAll(indexReader)
567 c.Assert(err, Equals, nil)
568 locators := strings.Split(string(indexResp), "\n")
571 for _, locator := range locators {
575 c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
576 if locator[:32] == hash {
582 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
583 c.Check(gotOther > 0, Equals, spec.expectOther)
586 // GetIndex with invalid prefix
587 _, err = kc.GetIndex(TestProxyUUID, "xyz")
588 c.Assert((err != nil), Equals, true)
591 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
592 kc := runProxy(c, false, false)
593 defer closeListener()
594 hash, _, err := kc.PutB([]byte("shareddata"))
596 kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
597 rdr, _, _, err := kc.Get(hash)
599 data, err := ioutil.ReadAll(rdr)
601 c.Check(data, DeepEquals, []byte("shareddata"))
604 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
605 kc := runProxy(c, false, false)
606 defer closeListener()
609 hash, rep, err := kc.PutB([]byte("foo"))
611 c.Check(rep, Equals, 2)
613 for _, badToken := range []string{
615 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
617 kc.Arvados.ApiToken = badToken
619 // Ask and Get will fail only if the upstream
620 // keepstore server checks for valid signatures.
621 // Without knowing the blob signing key, there is no
622 // way for keepproxy to know whether a given token is
623 // permitted to read a block. So these tests fail:
625 _, _, err = kc.Ask(hash)
626 c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
627 c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
628 c.Check(err, ErrorMatches, ".*HTTP 403.*")
630 _, _, _, err = kc.Get(hash)
631 c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
632 c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
633 c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
636 _, _, err = kc.PutB([]byte("foo"))
637 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
641 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
642 kc := runProxy(c, false, false)
643 defer closeListener()
645 // Point keepproxy at a non-existent keepstore
646 locals := map[string]string{
647 TestProxyUUID: "http://localhost:12345",
649 router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
651 // Ask should result in temporary bad gateway error
652 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
653 _, _, err := kc.Ask(hash)
655 errNotFound, _ := err.(*keepclient.ErrNotFound)
656 c.Check(errNotFound.Temporary(), Equals, true)
657 c.Assert(err, ErrorMatches, ".*HTTP 502.*")
659 // Get should result in temporary bad gateway error
660 _, _, _, err = kc.Get(hash)
662 errNotFound, _ = err.(*keepclient.ErrNotFound)
663 c.Check(errNotFound.Temporary(), Equals, true)
664 c.Assert(err, ErrorMatches, ".*HTTP 502.*")
667 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
668 kc := runProxy(c, false, false)
669 defer closeListener()
671 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
672 for _, f := range []func() error{
674 _, _, err := kc.Ask(hash)
678 _, _, _, err := kc.Get(hash)
683 c.Assert(err, NotNil)
684 errNotFound, _ := err.(*keepclient.ErrNotFound)
685 c.Check(errNotFound.Temporary(), Equals, true)
686 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
690 func (s *ServerRequiredSuite) TestPing(c *C) {
691 kc := runProxy(c, false, false)
692 defer closeListener()
694 rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
696 req, err := http.NewRequest("GET",
697 "http://"+listener.Addr().String()+"/_health/ping",
700 req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
702 resp := httptest.NewRecorder()
703 rtr.ServeHTTP(resp, req)
704 c.Check(resp.Code, Equals, 200)
705 c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)