X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/6207681aa301ad12d164aab24f52a210945af04b..7466d0c1af1f7d9a0c1b4ae54e9f1bbc951f2711:/services/keepproxy/keepproxy_test.go diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go index 997163eca4..65e22e3b3e 100644 --- a/services/keepproxy/keepproxy_test.go +++ b/services/keepproxy/keepproxy_test.go @@ -1,22 +1,29 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( + "bytes" "crypto/md5" - "crypto/tls" + "errors" "fmt" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - "git.curoverse.com/arvados.git/sdk/go/keepclient" - . "gopkg.in/check.v1" - "io" "io/ioutil" - "log" + "math/rand" "net/http" - "net/url" + "net/http/httptest" "os" "strings" + "sync" "testing" "time" + + "git.curoverse.com/arvados.git/sdk/go/arvadosclient" + "git.curoverse.com/arvados.git/sdk/go/arvadostest" + "git.curoverse.com/arvados.git/sdk/go/keepclient" + + . "gopkg.in/check.v1" ) // Gocheck boilerplate @@ -36,6 +43,8 @@ var _ = Suite(&NoKeepServerSuite{}) // Test with no keepserver to simulate errors type NoKeepServerSuite struct{} +var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz" + // Wait (up to 1 second) for keepproxy to listen on a port. This // avoids a race condition where we hit a "connection refused" error // because we start testing the proxy too soon. @@ -43,11 +52,11 @@ func waitForListener() { const ( ms = 5 ) - for i := 0; listener == nil && i < 1000; i += ms { + for i := 0; listener == nil && i < 10000; i += ms { time.Sleep(ms * time.Millisecond) } if listener == nil { - log.Fatalf("Timed out waiting for listener to start") + panic("Timed out waiting for listener to start") } } @@ -73,6 +82,10 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) { func (s *NoKeepServerSuite) SetUpSuite(c *C) { arvadostest.StartAPI() + // We need API to have some keep services listed, but the + // services themselves should be unresponsive. + arvadostest.StartKeep(2, false) + arvadostest.StopKeep(2) } func (s *NoKeepServerSuite) SetUpTest(c *C) { @@ -83,106 +96,157 @@ func (s *NoKeepServerSuite) TearDownSuite(c *C) { arvadostest.StopAPI() } -func setupProxyService() { - - client := &http.Client{Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}} +func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient { + args = append([]string{"keepproxy"}, args...) + os.Args = append(args, "-listen=:0") + listener = nil + go main() + waitForListener() - var req *http.Request - var err error - if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil { - panic(err.Error()) + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, Equals, nil) + if bogusClientToken { + arv.ApiToken = "bogus-token" } - req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN"))) + kc := keepclient.New(arv) + sr := map[string]string{ + TestProxyUUID: "http://" + listener.Addr().String(), + } + kc.SetServiceRoots(sr, sr, sr) + kc.Arvados.External = true - reader, writer := io.Pipe() + return kc +} - req.Body = reader +func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) { + runProxy(c, nil, false) + defer closeListener() - go func() { - data := url.Values{} - data.Set("keep_service", `{ - "service_host": "localhost", - "service_port": 29950, - "service_ssl_flag": false, - "service_type": "proxy" -}`) - - writer.Write([]byte(data.Encode())) - writer.Close() - }() + req, err := http.NewRequest("POST", + "http://"+listener.Addr().String()+"/", + strings.NewReader("TestViaHeader")) + c.Assert(err, Equals, nil) + req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken) + resp, err := (&http.Client{}).Do(req) + c.Assert(err, Equals, nil) + c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy") + locator, err := ioutil.ReadAll(resp.Body) + c.Assert(err, Equals, nil) + resp.Body.Close() - var resp *http.Response - if resp, err = client.Do(req); err != nil { - panic(err.Error()) - } - if resp.StatusCode != 200 { - panic(resp.Status) - } + req, err = http.NewRequest("GET", + "http://"+listener.Addr().String()+"/"+string(locator), + nil) + c.Assert(err, Equals, nil) + resp, err = (&http.Client{}).Do(req) + c.Assert(err, Equals, nil) + c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy") + resp.Body.Close() } -func runProxy(c *C, args []string, port int, bogusClientToken bool) keepclient.KeepClient { - if bogusClientToken { - os.Setenv("ARVADOS_API_TOKEN", "bogus-token") - } - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, Equals, nil) - kc := keepclient.KeepClient{ - Arvados: &arv, - Want_replicas: 2, - Using_proxy: true, - Client: &http.Client{}, - } - locals := map[string]string{ - "proxy": fmt.Sprintf("http://localhost:%v", port), - } - writableLocals := map[string]string{ - "proxy": fmt.Sprintf("http://localhost:%v", port), - } - kc.SetServiceRoots(locals, writableLocals, nil) - c.Check(kc.Using_proxy, Equals, true) - c.Check(len(kc.LocalRoots()), Equals, 1) - for _, root := range kc.LocalRoots() { - c.Check(root, Equals, fmt.Sprintf("http://localhost:%v", port)) - } - log.Print("keepclient created") - if bogusClientToken { - arvadostest.ResetEnv() - } +func (s *ServerRequiredSuite) TestLoopDetection(c *C) { + kc := runProxy(c, nil, false) + defer closeListener() - { - os.Args = append(args, fmt.Sprintf("-listen=:%v", port)) - listener = nil - go main() + sr := map[string]string{ + TestProxyUUID: "http://" + listener.Addr().String(), } + router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr) - return kc + content := []byte("TestLoopDetection") + _, _, err := kc.PutB(content) + c.Check(err, ErrorMatches, `.*loop detected.*`) + + hash := fmt.Sprintf("%x", md5.Sum(content)) + _, _, _, err = kc.Get(hash) + c.Check(err, ErrorMatches, `.*loop detected.*`) } -func (s *ServerRequiredSuite) TestPutAskGet(c *C) { - log.Print("TestPutAndGet start") +func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) { + kc := runProxy(c, nil, false) + defer closeListener() - os.Args = []string{"keepproxy", "-listen=:29950"} - listener = nil - go main() - time.Sleep(100 * time.Millisecond) + content := []byte("TestDesiredReplicas") + hash := fmt.Sprintf("%x", md5.Sum(content)) - setupProxyService() + for _, kc.Want_replicas = range []int{0, 1, 2} { + locator, rep, err := kc.PutB(content) + c.Check(err, Equals, nil) + c.Check(rep, Equals, kc.Want_replicas) + if rep > 0 { + c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content))) + } + } +} - os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true") - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, Equals, nil) - kc, err := keepclient.MakeKeepClient(&arv) - c.Assert(err, Equals, nil) - c.Check(kc.Arvados.External, Equals, true) - c.Check(kc.Using_proxy, Equals, true) - c.Check(len(kc.LocalRoots()), Equals, 1) - for _, root := range kc.LocalRoots() { - c.Check(root, Equals, "http://localhost:29950") +func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) { + kc := runProxy(c, nil, false) + defer closeListener() + + content := []byte("TestPutWrongContentLength") + hash := fmt.Sprintf("%x", md5.Sum(content)) + + // If we use http.Client to send these requests to the network + // server we just started, the Go http library automatically + // fixes the invalid Content-Length header. In order to test + // our server behavior, we have to call the handler directly + // using an httptest.ResponseRecorder. + rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "") + + type testcase struct { + sendLength string + expectStatus int } - os.Setenv("ARVADOS_EXTERNAL_CLIENT", "") - waitForListener() + for _, t := range []testcase{ + {"1", http.StatusBadRequest}, + {"", http.StatusLengthRequired}, + {"-1", http.StatusLengthRequired}, + {"abcdef", http.StatusLengthRequired}, + } { + req, err := http.NewRequest("PUT", + fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)), + bytes.NewReader(content)) + c.Assert(err, IsNil) + req.Header.Set("Content-Length", t.sendLength) + req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken) + req.Header.Set("Content-Type", "application/octet-stream") + + resp := httptest.NewRecorder() + rtr.ServeHTTP(resp, req) + c.Check(resp.Code, Equals, t.expectStatus) + } +} + +func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) { + kc := runProxy(c, nil, false) + defer closeListener() + router.(*proxyHandler).timeout = time.Nanosecond + + buf := make([]byte, 1<<20) + rand.Read(buf) + var wg sync.WaitGroup + for i := 0; i < 128; i++ { + wg.Add(1) + go func() { + defer wg.Done() + kc.PutB(buf) + }() + } + done := make(chan bool) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + c.Error("timeout") + } +} + +func (s *ServerRequiredSuite) TestPutAskGet(c *C) { + kc := runProxy(c, nil, false) defer closeListener() hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) @@ -191,14 +255,14 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { { _, _, err := kc.Ask(hash) c.Check(err, Equals, keepclient.BlockNotFound) - log.Print("Finished Ask (expected BlockNotFound)") + c.Log("Finished Ask (expected BlockNotFound)") } { reader, _, _, err := kc.Get(hash) c.Check(reader, Equals, nil) c.Check(err, Equals, keepclient.BlockNotFound) - log.Print("Finished Get (expected BlockNotFound)") + c.Log("Finished Get (expected BlockNotFound)") } // Note in bug #5309 among other errors keepproxy would set @@ -217,23 +281,24 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash)) c.Check(rep, Equals, 2) c.Check(err, Equals, nil) - log.Print("Finished PutB (expected success)") + c.Log("Finished PutB (expected success)") } { blocklen, _, err := kc.Ask(hash2) c.Assert(err, Equals, nil) c.Check(blocklen, Equals, int64(3)) - log.Print("Finished Ask (expected success)") + c.Log("Finished Ask (expected success)") } { reader, blocklen, _, err := kc.Get(hash2) c.Assert(err, Equals, nil) all, err := ioutil.ReadAll(reader) + c.Check(err, IsNil) c.Check(all, DeepEquals, []byte("foo")) c.Check(blocklen, Equals, int64(3)) - log.Print("Finished Get (expected success)") + c.Log("Finished Get (expected success)") } { @@ -243,72 +308,48 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`) c.Check(rep, Equals, 2) c.Check(err, Equals, nil) - log.Print("Finished PutB zero block") + c.Log("Finished PutB zero block") } { reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e") c.Assert(err, Equals, nil) all, err := ioutil.ReadAll(reader) + c.Check(err, IsNil) c.Check(all, DeepEquals, []byte("")) c.Check(blocklen, Equals, int64(0)) - log.Print("Finished Get zero block") + c.Log("Finished Get zero block") } - - log.Print("TestPutAndGet done") } func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) { - log.Print("TestPutAskGetForbidden start") - - kc := runProxy(c, []string{"keepproxy"}, 29951, true) - waitForListener() + kc := runProxy(c, nil, true) defer closeListener() - hash := fmt.Sprintf("%x", md5.Sum([]byte("bar"))) + hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar"))) - { - _, _, err := kc.Ask(hash) - errNotFound, _ := err.(keepclient.ErrNotFound) - c.Check(errNotFound, NotNil) - c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true) - log.Print("Ask 1") - } + _, _, err := kc.Ask(hash) + c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{}) - { - hash2, rep, err := kc.PutB([]byte("bar")) - c.Check(hash2, Equals, "") - c.Check(rep, Equals, 0) - c.Check(err, Equals, keepclient.InsufficientReplicasError) - log.Print("PutB") - } + hash2, rep, err := kc.PutB([]byte("bar")) + c.Check(hash2, Equals, "") + c.Check(rep, Equals, 0) + c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New(""))) - { - blocklen, _, err := kc.Ask(hash) - errNotFound, _ := err.(keepclient.ErrNotFound) - c.Check(errNotFound, NotNil) - c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true) - c.Check(blocklen, Equals, int64(0)) - log.Print("Ask 2") - } + blocklen, _, err := kc.Ask(hash) + c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{}) + c.Check(err, ErrorMatches, ".*not found.*") + c.Check(blocklen, Equals, int64(0)) - { - _, blocklen, _, err := kc.Get(hash) - errNotFound, _ := err.(keepclient.ErrNotFound) - c.Check(errNotFound, NotNil) - c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true) - c.Check(blocklen, Equals, int64(0)) - log.Print("Get") - } + _, blocklen, _, err = kc.Get(hash) + c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{}) + c.Check(err, ErrorMatches, ".*not found.*") + c.Check(blocklen, Equals, int64(0)) - log.Print("TestPutAskGetForbidden done") } func (s *ServerRequiredSuite) TestGetDisabled(c *C) { - log.Print("TestGetDisabled start") - - kc := runProxy(c, []string{"keepproxy", "-no-get"}, 29952, false) - waitForListener() + kc := runProxy(c, []string{"-no-get"}, false) defer closeListener() hash := fmt.Sprintf("%x", md5.Sum([]byte("baz"))) @@ -317,8 +358,8 @@ func (s *ServerRequiredSuite) TestGetDisabled(c *C) { _, _, err := kc.Ask(hash) errNotFound, _ := err.(keepclient.ErrNotFound) c.Check(errNotFound, NotNil) - c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true) - log.Print("Ask 1") + c.Assert(err, ErrorMatches, `.*HTTP 405.*`) + c.Log("Ask 1") } { @@ -326,65 +367,55 @@ func (s *ServerRequiredSuite) TestGetDisabled(c *C) { c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash)) c.Check(rep, Equals, 2) c.Check(err, Equals, nil) - log.Print("PutB") + c.Log("PutB") } { blocklen, _, err := kc.Ask(hash) errNotFound, _ := err.(keepclient.ErrNotFound) c.Check(errNotFound, NotNil) - c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true) + c.Assert(err, ErrorMatches, `.*HTTP 405.*`) c.Check(blocklen, Equals, int64(0)) - log.Print("Ask 2") + c.Log("Ask 2") } { _, blocklen, _, err := kc.Get(hash) errNotFound, _ := err.(keepclient.ErrNotFound) c.Check(errNotFound, NotNil) - c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true) + c.Assert(err, ErrorMatches, `.*HTTP 405.*`) c.Check(blocklen, Equals, int64(0)) - log.Print("Get") + c.Log("Get") } - - log.Print("TestGetDisabled done") } func (s *ServerRequiredSuite) TestPutDisabled(c *C) { - log.Print("TestPutDisabled start") - - kc := runProxy(c, []string{"keepproxy", "-no-put"}, 29953, false) - waitForListener() + kc := runProxy(c, []string{"-no-put"}, false) defer closeListener() - { - hash2, rep, err := kc.PutB([]byte("quux")) - c.Check(hash2, Equals, "") - c.Check(rep, Equals, 0) - c.Check(err, Equals, keepclient.InsufficientReplicasError) - log.Print("PutB") - } - - log.Print("TestPutDisabled done") + hash2, rep, err := kc.PutB([]byte("quux")) + c.Check(hash2, Equals, "") + c.Check(rep, Equals, 0) + c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New(""))) } func (s *ServerRequiredSuite) TestCorsHeaders(c *C) { - runProxy(c, []string{"keepproxy"}, 29954, false) - waitForListener() + runProxy(c, nil, false) defer closeListener() { client := http.Client{} req, err := http.NewRequest("OPTIONS", - fmt.Sprintf("http://localhost:29954/%x+3", - md5.Sum([]byte("foo"))), + fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))), nil) + c.Assert(err, IsNil) req.Header.Add("Access-Control-Request-Method", "PUT") req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas") resp, err := client.Do(req) c.Check(err, Equals, nil) c.Check(resp.StatusCode, Equals, 200) body, err := ioutil.ReadAll(resp.Body) + c.Check(err, IsNil) c.Check(string(body), Equals, "") c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS") c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*") @@ -392,8 +423,7 @@ func (s *ServerRequiredSuite) TestCorsHeaders(c *C) { { resp, err := http.Get( - fmt.Sprintf("http://localhost:29954/%x+3", - md5.Sum([]byte("foo")))) + fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo")))) c.Check(err, Equals, nil) c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas") c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*") @@ -401,16 +431,16 @@ func (s *ServerRequiredSuite) TestCorsHeaders(c *C) { } func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) { - runProxy(c, []string{"keepproxy"}, 29955, false) - waitForListener() + runProxy(c, nil, false) defer closeListener() { client := http.Client{} req, err := http.NewRequest("POST", - "http://localhost:29955/", + "http://"+listener.Addr().String()+"/", strings.NewReader("qux")) - req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") + c.Check(err, IsNil) + req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken) req.Header.Add("Content-Type", "application/octet-stream") resp, err := client.Do(req) c.Check(err, Equals, nil) @@ -444,8 +474,7 @@ func (s *ServerRequiredSuite) TestStripHint(c *C) { // With a valid but non-existing prefix (expect "\n") // With an invalid prefix (expect error) func (s *ServerRequiredSuite) TestGetIndex(c *C) { - kc := runProxy(c, []string{"keepproxy"}, 28852, false) - waitForListener() + kc := runProxy(c, nil, false) defer closeListener() // Put "index-data" blocks @@ -458,14 +487,17 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) { c.Check(err, Equals, nil) reader, blocklen, _, err := kc.Get(hash) - c.Assert(err, Equals, nil) + c.Assert(err, IsNil) c.Check(blocklen, Equals, int64(10)) all, err := ioutil.ReadAll(reader) + c.Assert(err, IsNil) c.Check(all, DeepEquals, data) // Put some more blocks - _, rep, err = kc.PutB([]byte("some-more-index-data")) - c.Check(err, Equals, nil) + _, _, err = kc.PutB([]byte("some-more-index-data")) + c.Check(err, IsNil) + + kc.Arvados.ApiToken = arvadostest.DataManagerToken // Invoke GetIndex for _, spec := range []struct { @@ -477,7 +509,7 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) { {hash[:3], true, false}, // with matching prefix {"abcdef", false, false}, // with no such prefix } { - indexReader, err := kc.GetIndex("proxy", spec.prefix) + indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix) c.Assert(err, Equals, nil) indexResp, err := ioutil.ReadAll(indexReader) c.Assert(err, Equals, nil) @@ -500,40 +532,57 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) { } // GetIndex with invalid prefix - _, err = kc.GetIndex("proxy", "xyz") + _, err = kc.GetIndex(TestProxyUUID, "xyz") c.Assert((err != nil), Equals, true) } +func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) { + kc := runProxy(c, nil, false) + defer closeListener() + hash, _, err := kc.PutB([]byte("shareddata")) + c.Check(err, IsNil) + kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken + rdr, _, _, err := kc.Get(hash) + c.Assert(err, IsNil) + data, err := ioutil.ReadAll(rdr) + c.Check(err, IsNil) + c.Check(data, DeepEquals, []byte("shareddata")) +} + func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) { - kc := runProxy(c, []string{"keepproxy"}, 28852, false) - waitForListener() + kc := runProxy(c, nil, false) defer closeListener() // Put a test block hash, rep, err := kc.PutB([]byte("foo")) - c.Check(err, Equals, nil) + c.Check(err, IsNil) c.Check(rep, Equals, 2) - for _, token := range []string{ + for _, badToken := range []string{ "nosuchtoken", "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired } { - // Change token to given bad token - kc.Arvados.ApiToken = token + kc.Arvados.ApiToken = badToken + + // Ask and Get will fail only if the upstream + // keepstore server checks for valid signatures. + // Without knowing the blob signing key, there is no + // way for keepproxy to know whether a given token is + // permitted to read a block. So these tests fail: + if false { + _, _, err = kc.Ask(hash) + c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{}) + c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false) + c.Check(err, ErrorMatches, ".*HTTP 403.*") + + _, _, _, err = kc.Get(hash) + c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{}) + c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false) + c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*") + } - // Ask should result in error - _, _, err = kc.Ask(hash) - c.Check(err, NotNil) - errNotFound, _ := err.(keepclient.ErrNotFound) - c.Check(errNotFound.Temporary(), Equals, false) - c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true) - - // Get should result in error - _, _, _, err = kc.Get(hash) - c.Check(err, NotNil) - errNotFound, _ = err.(keepclient.ErrNotFound) - c.Check(errNotFound.Temporary(), Equals, false) - c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true) + _, _, err = kc.PutB([]byte("foo")) + c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header") } } @@ -542,9 +591,9 @@ func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) { c.Assert(err, Equals, nil) // keepclient with no such keep server - kc := keepclient.New(&arv) + kc := keepclient.New(arv) locals := map[string]string{ - "proxy": "http://localhost:12345", + TestProxyUUID: "http://localhost:12345", } kc.SetServiceRoots(locals, nil, nil) @@ -554,33 +603,53 @@ func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) { c.Check(err, NotNil) errNotFound, _ := err.(*keepclient.ErrNotFound) c.Check(errNotFound.Temporary(), Equals, true) - c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true) + c.Assert(err, ErrorMatches, ".*connection refused.*") // Get should result in temporary connection refused error _, _, _, err = kc.Get(hash) c.Check(err, NotNil) errNotFound, _ = err.(*keepclient.ErrNotFound) c.Check(errNotFound.Temporary(), Equals, true) - c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true) + c.Assert(err, ErrorMatches, ".*connection refused.*") } func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) { - kc := runProxy(c, []string{"keepproxy"}, 29999, false) - waitForListener() + kc := runProxy(c, nil, false) defer closeListener() - // Ask should result in temporary connection refused error hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) - _, _, err := kc.Ask(hash) - c.Check(err, NotNil) - errNotFound, _ := err.(*keepclient.ErrNotFound) - c.Check(errNotFound.Temporary(), Equals, true) - c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true) + for _, f := range []func() error{ + func() error { + _, _, err := kc.Ask(hash) + return err + }, + func() error { + _, _, _, err := kc.Get(hash) + return err + }, + } { + err := f() + c.Assert(err, NotNil) + errNotFound, _ := err.(*keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, true) + c.Check(err, ErrorMatches, `.*HTTP 502.*`) + } +} - // Get should result in temporary connection refused error - _, _, _, err = kc.Get(hash) - c.Check(err, NotNil) - errNotFound, _ = err.(*keepclient.ErrNotFound) - c.Check(errNotFound.Temporary(), Equals, true) - c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true) +func (s *ServerRequiredSuite) TestPing(c *C) { + kc := runProxy(c, nil, false) + defer closeListener() + + rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken) + + req, err := http.NewRequest("GET", + "http://"+listener.Addr().String()+"/_health/ping", + nil) + c.Assert(err, IsNil) + req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken) + + resp := httptest.NewRecorder() + rtr.ServeHTTP(resp, req) + c.Check(resp.Code, Equals, 200) + c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`) }