X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/dbebe700704e5f100eec34e531ae8e90cf9232c4..c5db21f4d720e931ed7bff0c5da53caf0cafd2f4:/sdk/go/keepclient/keepclient_test.go diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go index e4e459e83a..df4638619f 100644 --- a/sdk/go/keepclient/keepclient_test.go +++ b/sdk/go/keepclient/keepclient_test.go @@ -14,6 +14,7 @@ import ( "net" "net/http" "os" + "strings" "testing" ) @@ -45,14 +46,14 @@ func (s *ServerRequiredSuite) SetUpSuite(c *C) { return } arvadostest.StartAPI() - arvadostest.StartKeep() + arvadostest.StartKeep(2, false) } func (s *ServerRequiredSuite) TearDownSuite(c *C) { if *no_server { return } - arvadostest.StopKeep() + arvadostest.StopKeep(2) arvadostest.StopAPI() } @@ -69,6 +70,22 @@ func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) { } } +func (s *ServerRequiredSuite) TestDefaultReplications(c *C) { + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, Equals, nil) + + kc, err := MakeKeepClient(&arv) + c.Assert(kc.Want_replicas, Equals, 2) + + arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0 + kc, err = MakeKeepClient(&arv) + c.Assert(kc.Want_replicas, Equals, 3) + + arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0 + kc, err = MakeKeepClient(&arv) + c.Assert(kc.Want_replicas, Equals, 1) +} + type StubPutHandler struct { c *C expectPath string @@ -184,6 +201,31 @@ func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { fh.handled <- fmt.Sprintf("http://%s", req.Host) } +type FailThenSucceedHandler struct { + handled chan string + count int + successhandler StubGetHandler +} + +func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + if fh.count == 0 { + resp.WriteHeader(500) + fh.count += 1 + fh.handled <- fmt.Sprintf("http://%s", req.Host) + } else { + fh.successhandler.ServeHTTP(resp, req) + } +} + +type Error404Handler struct { + handled chan string +} + +func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + resp.WriteHeader(404) + fh.handled <- fmt.Sprintf("http://%s", req.Host) +} + func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) { log.Printf("TestFailedUploadToStubKeepServer") @@ -401,6 +443,7 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) { kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 2 + kc.Retries = 0 arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) @@ -464,7 +507,7 @@ func (s *StandaloneSuite) TestGet(c *C) { arv, err := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(&arv) arv.ApiToken = "abc123" - kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil) + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, n, url2, err := kc.Get(hash) defer r.Close() @@ -479,6 +522,26 @@ func (s *StandaloneSuite) TestGet(c *C) { log.Printf("TestGet done") } +func (s *StandaloneSuite) TestGet404(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := Error404Handler{make(chan string, 1)} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + r, n, url2, err := kc.Get(hash) + c.Check(err, Equals, BlockNotFound) + c.Check(n, Equals, int64(0)) + c.Check(url2, Equals, "") + c.Check(r, Equals, nil) +} + func (s *StandaloneSuite) TestGetFail(c *C) { hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) @@ -490,10 +553,62 @@ func (s *StandaloneSuite) TestGetFail(c *C) { arv, err := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(&arv) arv.ApiToken = "abc123" - kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil) + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + kc.Retries = 0 r, n, url2, err := kc.Get(hash) - c.Check(err, Equals, BlockNotFound) + errNotFound, _ := err.(*ErrNotFound) + c.Check(errNotFound, NotNil) + c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true) + c.Check(errNotFound.Temporary(), Equals, true) + c.Check(n, Equals, int64(0)) + c.Check(url2, Equals, "") + c.Check(r, Equals, nil) +} + +func (s *StandaloneSuite) TestGetFailRetry(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := &FailThenSucceedHandler{make(chan string, 1), 0, + StubGetHandler{ + c, + hash, + "abc123", + http.StatusOK, + []byte("foo")}} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + r, n, url2, err := kc.Get(hash) + defer r.Close() + c.Check(err, Equals, nil) + c.Check(n, Equals, int64(3)) + c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash)) + + content, err2 := ioutil.ReadAll(r) + c.Check(err2, Equals, nil) + c.Check(content, DeepEquals, []byte("foo")) +} + +func (s *StandaloneSuite) TestGetNetError(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil) + + r, n, url2, err := kc.Get(hash) + errNotFound, _ := err.(*ErrNotFound) + c.Check(errNotFound, NotNil) + c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true) + c.Check(errNotFound.Temporary(), Equals, true) c.Check(n, Equals, int64(0)) c.Check(url2, Equals, "") c.Check(r, Equals, nil) @@ -525,7 +640,7 @@ func (s *StandaloneSuite) TestGetWithServiceHint(c *C) { arv.ApiToken = "abc123" kc.SetServiceRoots( map[string]string{"x": ks0.url}, - map[string]string{"x": ks0.url}, + nil, map[string]string{uuid: ks.url}) r, n, uri, err := kc.Get(hash + "+K@" + uuid) @@ -572,11 +687,7 @@ func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) { "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url, "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url, uuid: ks.url}, - map[string]string{ - "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url, - "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url, - "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url, - uuid: ks.url}, + nil, map[string]string{ "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url, "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url, @@ -619,7 +730,7 @@ func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) { arv.ApiToken = "abc123" kc.SetServiceRoots( map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url}, - map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url}, + nil, map[string]string{uuid: ksGateway.url}) r, n, uri, err := kc.Get(hash + "+K@" + uuid) @@ -654,7 +765,7 @@ func (s *StandaloneSuite) TestChecksum(c *C) { arv, err := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(&arv) arv.ApiToken = "abc123" - kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil) + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, n, _, err := kc.Get(barhash) _, err = ioutil.ReadAll(r) @@ -675,7 +786,7 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) { content := []byte("waz") hash := fmt.Sprintf("%x", md5.Sum(content)) - fh := FailHandler{ + fh := Error404Handler{ make(chan string, 4)} st := StubGetHandler{ @@ -706,6 +817,7 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) { } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) + kc.Retries = 0 // This test works only if one of the failing services is // attempted before the succeeding service. Otherwise, @@ -948,3 +1060,189 @@ func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) { c.Check(err, Equals, InsufficientReplicasError) c.Check(replicas, Equals, 0) } + +type StubGetIndexHandler struct { + c *C + expectPath string + expectAPIToken string + httpStatus int + body []byte +} + +func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + h.c.Check(req.URL.Path, Equals, h.expectPath) + h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken)) + resp.WriteHeader(h.httpStatus) + resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body))) + resp.Write(h.body) +} + +func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := StubGetIndexHandler{ + c, + "/index", + "abc123", + http.StatusOK, + []byte(hash + "+3 1443559274\n\n")} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + r, err := kc.GetIndex("x", "") + c.Check(err, Equals, nil) + + content, err2 := ioutil.ReadAll(r) + c.Check(err2, Equals, nil) + c.Check(content, DeepEquals, st.body[0:len(st.body)-1]) +} + +func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := StubGetIndexHandler{ + c, + "/index/" + hash[0:3], + "abc123", + http.StatusOK, + []byte(hash + "+3 1443559274\n\n")} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + r, err := kc.GetIndex("x", hash[0:3]) + c.Check(err, Equals, nil) + + content, err2 := ioutil.ReadAll(r) + c.Check(err2, Equals, nil) + c.Check(content, DeepEquals, st.body[0:len(st.body)-1]) +} + +func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := StubGetIndexHandler{ + c, + "/index/" + hash[0:3], + "abc123", + http.StatusOK, + []byte(hash)} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + _, err = kc.GetIndex("x", hash[0:3]) + c.Check(err, Equals, ErrIncompleteIndex) +} + +func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := StubGetIndexHandler{ + c, + "/index/" + hash[0:3], + "abc123", + http.StatusOK, + []byte(hash)} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + _, err = kc.GetIndex("y", hash[0:3]) + c.Check(err, Equals, ErrNoSuchKeepServer) +} + +func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) { + st := StubGetIndexHandler{ + c, + "/index/abcd", + "abc123", + http.StatusOK, + []byte("\n")} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + r, err := kc.GetIndex("x", "abcd") + c.Check(err, Equals, nil) + + content, err2 := ioutil.ReadAll(r) + c.Check(err2, Equals, nil) + c.Check(content, DeepEquals, st.body[0:len(st.body)-1]) +} + +type FailThenSucceedPutHandler struct { + handled chan string + count int + successhandler StubPutHandler +} + +func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + if h.count == 0 { + resp.WriteHeader(500) + h.count += 1 + h.handled <- fmt.Sprintf("http://%s", req.Host) + } else { + h.successhandler.ServeHTTP(resp, req) + } +} + +func (s *StandaloneSuite) TestPutBRetry(c *C) { + st := &FailThenSucceedPutHandler{make(chan string, 1), 0, + StubPutHandler{ + c, + Md5String("foo"), + "abc123", + "foo", + make(chan string, 5)}} + + arv, _ := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + + kc.Want_replicas = 2 + arv.ApiToken = "abc123" + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) + + ks := RunSomeFakeKeepServers(st, 2) + + for i, k := range ks { + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + defer k.listener.Close() + } + + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) + + hash, replicas, err := kc.PutB([]byte("foo")) + + c.Check(err, Equals, nil) + c.Check(hash, Equals, "") + c.Check(replicas, Equals, 2) +}