X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/4ad120ed9f8977dff6ab333b4223f323d04a33d7..7533276a8b444ac3d49f74c8da0fb31d8caf2650:/services/keepproxy/keepproxy_test.go diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go index 1b9f627b55..f02971d61b 100644 --- a/services/keepproxy/keepproxy_test.go +++ b/services/keepproxy/keepproxy_test.go @@ -13,16 +13,20 @@ import ( "math/rand" "net/http" "net/http/httptest" - "os" "strings" "sync" "testing" "time" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - "git.curoverse.com/arvados.git/sdk/go/keepclient" + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "git.arvados.org/arvados.git/sdk/go/keepclient" + log "github.com/sirupsen/logrus" + "gopkg.in/check.v1" . "gopkg.in/check.v1" ) @@ -37,6 +41,12 @@ var _ = Suite(&ServerRequiredSuite{}) // Tests that require the Keep server running type ServerRequiredSuite struct{} +// Gocheck boilerplate +var _ = Suite(&ServerRequiredConfigYmlSuite{}) + +// Tests that require the Keep servers running as defined in config.yml +type ServerRequiredConfigYmlSuite struct{} + // Gocheck boilerplate var _ = Suite(&NoKeepServerSuite{}) @@ -80,6 +90,21 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) { arvadostest.StopAPI() } +func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) { + arvadostest.StartAPI() + // config.yml defines 4 keepstores + arvadostest.StartKeep(4, false) +} + +func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) { + arvadostest.ResetEnv() +} + +func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) { + arvadostest.StopKeep(4) + arvadostest.StopAPI() +} + func (s *NoKeepServerSuite) SetUpSuite(c *C) { arvadostest.StartAPI() // We need API to have some keep services listed, but the @@ -96,14 +121,35 @@ func (s *NoKeepServerSuite) TearDownSuite(c *C) { arvadostest.StopAPI() } -func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient { - args = append([]string{"keepproxy"}, args...) - os.Args = append(args, "-listen=:0") +func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool, kp *arvados.UploadDownloadRolePermissions) (*keepclient.KeepClient, *bytes.Buffer) { + cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load() + c.Assert(err, Equals, nil) + cluster, err := cfg.GetCluster("") + c.Assert(err, Equals, nil) + + if !loadKeepstoresFromConfig { + // Do not load Keepstore InternalURLs from the config file + cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance) + } + + cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: ":0"}: {}} + + if kp != nil { + cluster.Collections.KeepproxyPermission = *kp + } + listener = nil - go main() + logbuf := &bytes.Buffer{} + logger := log.New() + logger.Out = logbuf + go func() { + run(logger, cluster) + defer closeListener() + }() waitForListener() - arv, err := arvadosclient.MakeArvadosClient() + client := arvados.NewClientFromEnv() + arv, err := arvadosclient.New(client) c.Assert(err, Equals, nil) if bogusClientToken { arv.ApiToken = "bogus-token" @@ -115,20 +161,22 @@ func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient kc.SetServiceRoots(sr, sr, sr) kc.Arvados.External = true - return kc + return kc, logbuf } func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) { - runProxy(c, nil, false) + runProxy(c, false, false, nil) defer closeListener() req, err := http.NewRequest("POST", "http://"+listener.Addr().String()+"/", strings.NewReader("TestViaHeader")) + c.Assert(err, Equals, nil) req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken) resp, err := (&http.Client{}).Do(req) c.Assert(err, Equals, nil) c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy") + c.Assert(resp.StatusCode, Equals, http.StatusOK) locator, err := ioutil.ReadAll(resp.Body) c.Assert(err, Equals, nil) resp.Body.Close() @@ -144,7 +192,7 @@ func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) { } func (s *ServerRequiredSuite) TestLoopDetection(c *C) { - kc := runProxy(c, nil, false) + kc, _ := runProxy(c, false, false, nil) defer closeListener() sr := map[string]string{ @@ -161,25 +209,78 @@ func (s *ServerRequiredSuite) TestLoopDetection(c *C) { c.Check(err, ErrorMatches, `.*loop detected.*`) } +func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) { + kc, _ := runProxy(c, false, false, nil) + defer closeListener() + + // Set up fake keepstore to record request headers + var hdr http.Header + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + hdr = r.Header + http.Error(w, "Error", http.StatusInternalServerError) + })) + defer ts.Close() + + // Point keepproxy router's keepclient to the fake keepstore + sr := map[string]string{ + TestProxyUUID: ts.URL, + } + router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr) + + // Set up client to ask for storage classes to keepproxy + kc.StorageClasses = []string{"secure"} + content := []byte("Very important data") + _, _, err := kc.PutB(content) + c.Check(err, NotNil) + c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure") +} + +func (s *ServerRequiredSuite) TestStorageClassesConfirmedHeader(c *C) { + runProxy(c, false, false, nil) + defer closeListener() + + content := []byte("foo") + hash := fmt.Sprintf("%x", md5.Sum(content)) + client := &http.Client{} + + req, err := http.NewRequest("PUT", + fmt.Sprintf("http://%s/%s", listener.Addr().String(), hash), + bytes.NewReader(content)) + c.Assert(err, IsNil) + req.Header.Set("X-Keep-Storage-Classes", "default") + req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken) + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := client.Do(req) + c.Assert(err, IsNil) + c.Assert(resp.StatusCode, Equals, http.StatusOK) + c.Assert(resp.Header.Get("X-Keep-Storage-Classes-Confirmed"), Equals, "default=2") +} + func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) { - kc := runProxy(c, nil, false) + kc, _ := runProxy(c, false, false, nil) defer closeListener() content := []byte("TestDesiredReplicas") hash := fmt.Sprintf("%x", md5.Sum(content)) - for _, kc.Want_replicas = range []int{0, 1, 2} { + for _, kc.Want_replicas = range []int{0, 1, 2, 3} { locator, rep, err := kc.PutB(content) - c.Check(err, Equals, nil) - c.Check(rep, Equals, kc.Want_replicas) - if rep > 0 { - c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content))) + if kc.Want_replicas < 3 { + c.Check(err, Equals, nil) + c.Check(rep, Equals, kc.Want_replicas) + if rep > 0 { + c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content))) + } + } else { + c.Check(err, ErrorMatches, ".*503.*") } } } func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) { - kc := runProxy(c, nil, false) + kc, _ := runProxy(c, false, false, nil) defer closeListener() content := []byte("TestPutWrongContentLength") @@ -190,7 +291,8 @@ func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) { // fixes the invalid Content-Length header. In order to test // our server behavior, we have to call the handler directly // using an httptest.ResponseRecorder. - rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "") + rtr, err := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{}, log.New()) + c.Assert(err, check.IsNil) type testcase struct { sendLength string @@ -218,7 +320,7 @@ func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) { } func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) { - kc := runProxy(c, nil, false) + kc, _ := runProxy(c, false, false, nil) defer closeListener() router.(*proxyHandler).timeout = time.Nanosecond @@ -245,7 +347,7 @@ func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) { } func (s *ServerRequiredSuite) TestPutAskGet(c *C) { - kc := runProxy(c, nil, false) + kc, logbuf := runProxy(c, false, false, nil) defer closeListener() hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) @@ -281,6 +383,9 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { c.Check(rep, Equals, 2) c.Check(err, Equals, nil) c.Log("Finished PutB (expected success)") + + c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`) + logbuf.Reset() } { @@ -288,6 +393,8 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { c.Assert(err, Equals, nil) c.Check(blocklen, Equals, int64(3)) c.Log("Finished Ask (expected success)") + c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`) + logbuf.Reset() } { @@ -298,6 +405,8 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { c.Check(all, DeepEquals, []byte("foo")) c.Check(blocklen, Equals, int64(3)) c.Log("Finished Get (expected success)") + c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`) + logbuf.Reset() } { @@ -322,7 +431,7 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { } func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) { - kc := runProxy(c, nil, true) + kc, _ := runProxy(c, true, false, nil) defer closeListener() hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar"))) @@ -337,69 +446,116 @@ func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) { blocklen, _, err := kc.Ask(hash) c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{}) - c.Check(err, ErrorMatches, ".*not found.*") + c.Check(err, ErrorMatches, ".*HTTP 403.*") c.Check(blocklen, Equals, int64(0)) _, blocklen, _, err = kc.Get(hash) c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{}) - c.Check(err, ErrorMatches, ".*not found.*") + c.Check(err, ErrorMatches, ".*HTTP 403.*") c.Check(blocklen, Equals, int64(0)) - } -func (s *ServerRequiredSuite) TestGetDisabled(c *C) { - kc := runProxy(c, []string{"-no-get"}, false) +func testPermission(c *C, admin bool, perm arvados.UploadDownloadPermission) { + kp := arvados.UploadDownloadRolePermissions{} + if admin { + kp.Admin = perm + kp.User = arvados.UploadDownloadPermission{Upload: true, Download: true} + } else { + kp.Admin = arvados.UploadDownloadPermission{Upload: true, Download: true} + kp.User = perm + } + + kc, logbuf := runProxy(c, false, false, &kp) defer closeListener() + if admin { + kc.Arvados.ApiToken = arvadostest.AdminToken + } else { + kc.Arvados.ApiToken = arvadostest.ActiveToken + } - hash := fmt.Sprintf("%x", md5.Sum([]byte("baz"))) + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + var hash2 string { - _, _, err := kc.Ask(hash) - errNotFound, _ := err.(keepclient.ErrNotFound) - c.Check(errNotFound, NotNil) - c.Assert(err, ErrorMatches, `.*HTTP 405.*`) - c.Log("Ask 1") - } + var rep int + var err error + hash2, rep, err = kc.PutB([]byte("foo")) - { - hash2, rep, err := kc.PutB([]byte("baz")) - c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash)) - c.Check(rep, Equals, 2) - c.Check(err, Equals, nil) - c.Log("PutB") - } + if perm.Upload { + c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash)) + c.Check(rep, Equals, 2) + c.Check(err, Equals, nil) + c.Log("Finished PutB (expected success)") + if admin { + c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`) + } else { - { - blocklen, _, err := kc.Ask(hash) - errNotFound, _ := err.(keepclient.ErrNotFound) - c.Check(errNotFound, NotNil) - c.Assert(err, ErrorMatches, `.*HTTP 405.*`) - c.Check(blocklen, Equals, int64(0)) - c.Log("Ask 2") + c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`) + } + } else { + c.Check(hash2, Equals, "") + c.Check(rep, Equals, 0) + c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New(""))) + } + logbuf.Reset() } + if perm.Upload { + // can't test download without upload. - { - _, blocklen, _, err := kc.Get(hash) - errNotFound, _ := err.(keepclient.ErrNotFound) - c.Check(errNotFound, NotNil) - c.Assert(err, ErrorMatches, `.*HTTP 405.*`) - c.Check(blocklen, Equals, int64(0)) - c.Log("Get") + reader, blocklen, _, err := kc.Get(hash2) + if perm.Download { + c.Assert(err, Equals, nil) + all, err := ioutil.ReadAll(reader) + c.Check(err, IsNil) + c.Check(all, DeepEquals, []byte("foo")) + c.Check(blocklen, Equals, int64(3)) + c.Log("Finished Get (expected success)") + if admin { + c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`) + } else { + c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`) + } + } else { + c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{}) + c.Check(err, ErrorMatches, ".*Missing or invalid Authorization header, or method not allowed.*") + c.Check(blocklen, Equals, int64(0)) + } + logbuf.Reset() } -} -func (s *ServerRequiredSuite) TestPutDisabled(c *C) { - kc := runProxy(c, []string{"-no-put"}, false) - defer closeListener() +} - hash2, rep, err := kc.PutB([]byte("quux")) - c.Check(hash2, Equals, "") - c.Check(rep, Equals, 0) - c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New(""))) +func (s *ServerRequiredSuite) TestPutGetPermission(c *C) { + + for _, adminperm := range []bool{true, false} { + for _, userperm := range []bool{true, false} { + + testPermission(c, true, + arvados.UploadDownloadPermission{ + Upload: adminperm, + Download: true, + }) + testPermission(c, true, + arvados.UploadDownloadPermission{ + Upload: true, + Download: adminperm, + }) + testPermission(c, false, + arvados.UploadDownloadPermission{ + Upload: true, + Download: userperm, + }) + testPermission(c, false, + arvados.UploadDownloadPermission{ + Upload: true, + Download: userperm, + }) + } + } } func (s *ServerRequiredSuite) TestCorsHeaders(c *C) { - runProxy(c, nil, false) + runProxy(c, false, false, nil) defer closeListener() { @@ -407,6 +563,7 @@ func (s *ServerRequiredSuite) TestCorsHeaders(c *C) { req, err := http.NewRequest("OPTIONS", fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))), nil) + c.Assert(err, IsNil) req.Header.Add("Access-Control-Request-Method", "PUT") req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas") resp, err := client.Do(req) @@ -429,7 +586,7 @@ func (s *ServerRequiredSuite) TestCorsHeaders(c *C) { } func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) { - runProxy(c, nil, false) + runProxy(c, false, false, nil) defer closeListener() { @@ -437,6 +594,7 @@ func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) { req, err := http.NewRequest("POST", "http://"+listener.Addr().String()+"/", strings.NewReader("qux")) + c.Check(err, IsNil) req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken) req.Header.Add("Content-Type", "application/octet-stream") resp, err := client.Do(req) @@ -471,7 +629,22 @@ func (s *ServerRequiredSuite) TestStripHint(c *C) { // With a valid but non-existing prefix (expect "\n") // With an invalid prefix (expect error) func (s *ServerRequiredSuite) TestGetIndex(c *C) { - kc := runProxy(c, nil, false) + getIndexWorker(c, false) +} + +// Test GetIndex +// Uses config.yml +// Put one block, with 2 replicas +// With no prefix (expect the block locator, twice) +// With an existing prefix (expect the block locator, twice) +// With a valid but non-existing prefix (expect "\n") +// With an invalid prefix (expect error) +func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) { + getIndexWorker(c, true) +} + +func getIndexWorker(c *C, useConfig bool) { + kc, _ := runProxy(c, false, useConfig, nil) defer closeListener() // Put "index-data" blocks @@ -484,16 +657,17 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) { c.Check(err, Equals, nil) reader, blocklen, _, err := kc.Get(hash) - c.Assert(err, Equals, nil) + c.Assert(err, IsNil) c.Check(blocklen, Equals, int64(10)) all, err := ioutil.ReadAll(reader) + c.Assert(err, IsNil) c.Check(all, DeepEquals, data) // Put some more blocks _, _, err = kc.PutB([]byte("some-more-index-data")) c.Check(err, IsNil) - kc.Arvados.ApiToken = arvadostest.DataManagerToken + kc.Arvados.ApiToken = arvadostest.SystemRootToken // Invoke GetIndex for _, spec := range []struct { @@ -533,7 +707,7 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) { } func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) { - kc := runProxy(c, nil, false) + kc, _ := runProxy(c, false, false, nil) defer closeListener() hash, _, err := kc.PutB([]byte("shareddata")) c.Check(err, IsNil) @@ -546,7 +720,7 @@ func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) { } func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) { - kc := runProxy(c, nil, false) + kc, _ := runProxy(c, false, false, nil) defer closeListener() // Put a test block @@ -574,43 +748,42 @@ func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) { _, _, _, err = kc.Get(hash) c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{}) c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false) - c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*") + c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header, or method not allowed\".*") } _, _, err = kc.PutB([]byte("foo")) - c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header") + c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header, or method not allowed") } } func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) { - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, Equals, nil) + kc, _ := runProxy(c, false, false, nil) + defer closeListener() - // keepclient with no such keep server - kc := keepclient.New(arv) + // Point keepproxy at a non-existent keepstore locals := map[string]string{ TestProxyUUID: "http://localhost:12345", } - kc.SetServiceRoots(locals, nil, nil) + router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil) - // Ask should result in temporary connection refused error + // Ask should result in temporary bad gateway error hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) - _, _, err = kc.Ask(hash) + _, _, err := kc.Ask(hash) c.Check(err, NotNil) errNotFound, _ := err.(*keepclient.ErrNotFound) c.Check(errNotFound.Temporary(), Equals, true) - c.Assert(err, ErrorMatches, ".*connection refused.*") + c.Assert(err, ErrorMatches, ".*HTTP 502.*") - // Get should result in temporary connection refused error + // Get should result in temporary bad gateway error _, _, _, err = kc.Get(hash) c.Check(err, NotNil) errNotFound, _ = err.(*keepclient.ErrNotFound) c.Check(errNotFound.Temporary(), Equals, true) - c.Assert(err, ErrorMatches, ".*connection refused.*") + c.Assert(err, ErrorMatches, ".*HTTP 502.*") } func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) { - kc := runProxy(c, nil, false) + kc, _ := runProxy(c, false, false, nil) defer closeListener() hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) @@ -633,10 +806,11 @@ func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) { } func (s *ServerRequiredSuite) TestPing(c *C) { - kc := runProxy(c, nil, false) + kc, _ := runProxy(c, false, false, nil) defer closeListener() - rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken) + rtr, err := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{ManagementToken: arvadostest.ManagementToken}, log.New()) + c.Assert(err, check.IsNil) req, err := http.NewRequest("GET", "http://"+listener.Addr().String()+"/_health/ping",