X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/813d35123538b00ab70719e247b6bb0881269460..6613ec1e9c705fb5b950611fd160d4a2babed251:/services/keepproxy/keepproxy_test.go diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go index f350e0b657..6a349dae21 100644 --- a/services/keepproxy/keepproxy_test.go +++ b/services/keepproxy/keepproxy_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "crypto/md5" "fmt" "git.curoverse.com/arvados.git/sdk/go/arvadosclient" @@ -9,6 +10,7 @@ import ( "io/ioutil" "log" "net/http" + "net/http/httptest" "os" "strings" "testing" @@ -28,6 +30,12 @@ var _ = Suite(&ServerRequiredSuite{}) // Tests that require the Keep server running type ServerRequiredSuite struct{} +// Gocheck boilerplate +var _ = Suite(&NoKeepServerSuite{}) + +// Test with no keepserver to simulate errors +type NoKeepServerSuite struct{} + var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz" // Wait (up to 1 second) for keepproxy to listen on a port. This @@ -37,7 +45,7 @@ func waitForListener() { const ( ms = 5 ) - for i := 0; listener == nil && i < 1000; i += ms { + for i := 0; listener == nil && i < 10000; i += ms { time.Sleep(ms * time.Millisecond) } if listener == nil { @@ -53,7 +61,7 @@ func closeListener() { func (s *ServerRequiredSuite) SetUpSuite(c *C) { arvadostest.StartAPI() - arvadostest.StartKeep() + arvadostest.StartKeep(2, false) } func (s *ServerRequiredSuite) SetUpTest(c *C) { @@ -61,7 +69,23 @@ func (s *ServerRequiredSuite) SetUpTest(c *C) { } func (s *ServerRequiredSuite) TearDownSuite(c *C) { - arvadostest.StopKeep() + arvadostest.StopKeep(2) + arvadostest.StopAPI() +} + +func (s *NoKeepServerSuite) SetUpSuite(c *C) { + arvadostest.StartAPI() + // We need API to have some keep services listed, but the + // services themselves should be unresponsive. + arvadostest.StartKeep(2, false) + arvadostest.StopKeep(2) +} + +func (s *NoKeepServerSuite) SetUpTest(c *C) { + arvadostest.ResetEnv() +} + +func (s *NoKeepServerSuite) TearDownSuite(c *C) { arvadostest.StopAPI() } @@ -77,17 +101,72 @@ func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient if bogusClientToken { arv.ApiToken = "bogus-token" } - kc := keepclient.New(&arv) + kc := keepclient.New(arv) sr := map[string]string{ TestProxyUUID: "http://" + listener.Addr().String(), } kc.SetServiceRoots(sr, sr, sr) kc.Arvados.External = true - kc.Using_proxy = true return kc } +func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) { + kc := runProxy(c, nil, false) + defer closeListener() + + content := []byte("TestDesiredReplicas") + hash := fmt.Sprintf("%x", md5.Sum(content)) + + for _, kc.Want_replicas = range []int{0, 1, 2} { + locator, rep, err := kc.PutB(content) + c.Check(err, Equals, nil) + c.Check(rep, Equals, kc.Want_replicas) + if rep > 0 { + c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content))) + } + } +} + +func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) { + kc := runProxy(c, nil, false) + defer closeListener() + + content := []byte("TestPutWrongContentLength") + hash := fmt.Sprintf("%x", md5.Sum(content)) + + // If we use http.Client to send these requests to the network + // server we just started, the Go http library automatically + // fixes the invalid Content-Length header. In order to test + // our server behavior, we have to call the handler directly + // using an httptest.ResponseRecorder. + rtr := MakeRESTRouter(true, true, kc) + + type testcase struct { + sendLength string + expectStatus int + } + + for _, t := range []testcase{ + {"1", http.StatusBadRequest}, + {"", http.StatusLengthRequired}, + {"-1", http.StatusLengthRequired}, + {"abcdef", http.StatusLengthRequired}, + } { + req, err := http.NewRequest("PUT", + fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)), + bytes.NewReader(content)) + c.Assert(err, IsNil) + req.Header.Set("Content-Length", t.sendLength) + req.Header.Set("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") + req.Header.Set("Content-Type", "application/octet-stream") + + resp := httptest.NewRecorder() + rtr.ServeHTTP(resp, req) + c.Check(resp.Code, Equals, t.expectStatus) + } +} + func (s *ServerRequiredSuite) TestPutAskGet(c *C) { kc := runProxy(c, nil, false) defer closeListener() @@ -171,7 +250,9 @@ func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) { { _, _, err := kc.Ask(hash) - c.Check(err, Equals, keepclient.BlockNotFound) + errNotFound, _ := err.(keepclient.ErrNotFound) + c.Check(errNotFound, NotNil) + c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true) log.Print("Ask 1") } @@ -185,14 +266,18 @@ func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) { { blocklen, _, err := kc.Ask(hash) - c.Assert(err, Equals, keepclient.BlockNotFound) + errNotFound, _ := err.(keepclient.ErrNotFound) + c.Check(errNotFound, NotNil) + c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true) c.Check(blocklen, Equals, int64(0)) log.Print("Ask 2") } { _, blocklen, _, err := kc.Get(hash) - c.Assert(err, Equals, keepclient.BlockNotFound) + errNotFound, _ := err.(keepclient.ErrNotFound) + c.Check(errNotFound, NotNil) + c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true) c.Check(blocklen, Equals, int64(0)) log.Print("Get") } @@ -206,7 +291,9 @@ func (s *ServerRequiredSuite) TestGetDisabled(c *C) { { _, _, err := kc.Ask(hash) - c.Check(err, Equals, keepclient.BlockNotFound) + errNotFound, _ := err.(keepclient.ErrNotFound) + c.Check(errNotFound, NotNil) + c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true) log.Print("Ask 1") } @@ -220,14 +307,18 @@ func (s *ServerRequiredSuite) TestGetDisabled(c *C) { { blocklen, _, err := kc.Ask(hash) - c.Assert(err, Equals, keepclient.BlockNotFound) + errNotFound, _ := err.(keepclient.ErrNotFound) + c.Check(errNotFound, NotNil) + c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true) c.Check(blocklen, Equals, int64(0)) log.Print("Ask 2") } { _, blocklen, _, err := kc.Get(hash) - c.Assert(err, Equals, keepclient.BlockNotFound) + errNotFound, _ := err.(keepclient.ErrNotFound) + c.Check(errNotFound, NotNil) + c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true) c.Check(blocklen, Equals, int64(0)) log.Print("Get") } @@ -337,6 +428,8 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) { _, rep, err = kc.PutB([]byte("some-more-index-data")) c.Check(err, Equals, nil) + kc.Arvados.ApiToken = arvadostest.DataManagerToken + // Invoke GetIndex for _, spec := range []struct { prefix string @@ -373,3 +466,85 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) { _, err = kc.GetIndex(TestProxyUUID, "xyz") c.Assert((err != nil), Equals, true) } + +func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) { + kc := runProxy(c, nil, false) + defer closeListener() + + // Put a test block + hash, rep, err := kc.PutB([]byte("foo")) + c.Check(err, Equals, nil) + c.Check(rep, Equals, 2) + + for _, token := range []string{ + "nosuchtoken", + "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired + } { + // Change token to given bad token + kc.Arvados.ApiToken = token + + // Ask should result in error + _, _, err = kc.Ask(hash) + c.Check(err, NotNil) + errNotFound, _ := err.(keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, false) + c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true) + + // Get should result in error + _, _, _, err = kc.Get(hash) + c.Check(err, NotNil) + errNotFound, _ = err.(keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, false) + c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true) + } +} + +func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) { + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, Equals, nil) + + // keepclient with no such keep server + kc := keepclient.New(arv) + locals := map[string]string{ + TestProxyUUID: "http://localhost:12345", + } + kc.SetServiceRoots(locals, nil, nil) + + // Ask should result in temporary connection refused error + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + _, _, err = kc.Ask(hash) + c.Check(err, NotNil) + errNotFound, _ := err.(*keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, true) + c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true) + + // Get should result in temporary connection refused error + _, _, _, err = kc.Get(hash) + c.Check(err, NotNil) + errNotFound, _ = err.(*keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, true) + c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true) +} + +func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) { + kc := runProxy(c, nil, false) + defer closeListener() + + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + for _, f := range []func() error{ + func() error { + _, _, err := kc.Ask(hash) + return err + }, + func() error { + _, _, _, err := kc.Get(hash) + return err + }, + } { + err := f() + c.Assert(err, NotNil) + errNotFound, _ := err.(*keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, true) + c.Check(err, ErrorMatches, `.*HTTP 502.*`) + } +}