X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1f37045c831a45c6899ef0d44f3cccd15476831a..7ed2f9d875d69b1494372c0cb790b18187dbacf2:/sdk/go/keepclient/keepclient_test.go diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go index b5bc5ced4b..87b9b1d1eb 100644 --- a/sdk/go/keepclient/keepclient_test.go +++ b/sdk/go/keepclient/keepclient_test.go @@ -14,7 +14,9 @@ import ( "net" "net/http" "os" + "strings" "testing" + "time" ) // Gocheck boilerplate @@ -45,14 +47,14 @@ func (s *ServerRequiredSuite) SetUpSuite(c *C) { return } arvadostest.StartAPI() - arvadostest.StartKeep() + arvadostest.StartKeep(2, false) } func (s *ServerRequiredSuite) TearDownSuite(c *C) { if *no_server { return } - arvadostest.StopKeep() + arvadostest.StopKeep(2) arvadostest.StopAPI() } @@ -142,10 +144,9 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) { make(chan string)} UploadToStubHelper(c, st, - func(kc *KeepClient, url string, reader io.ReadCloser, - writer io.WriteCloser, upload_status chan uploadStatus) { + func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) { - go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer") + go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0) writer.Write([]byte("foo")) writer.Close() @@ -177,7 +178,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) { br1 := tr.MakeStreamReader() - go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader") + go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0) writer.Write([]byte("foo")) writer.Close() @@ -237,7 +238,7 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) { func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) { - go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer") + go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0) writer.Write([]byte("foo")) writer.Close() @@ -442,6 +443,7 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) { kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 2 + kc.Retries = 0 arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) @@ -552,9 +554,13 @@ func (s *StandaloneSuite) TestGetFail(c *C) { kc, _ := MakeKeepClient(&arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + kc.Retries = 0 r, n, url2, err := kc.Get(hash) - c.Check(err, Equals, BlockNotFound) + errNotFound, _ := err.(*ErrNotFound) + c.Check(errNotFound, NotNil) + c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true) + c.Check(errNotFound.Temporary(), Equals, true) c.Check(n, Equals, int64(0)) c.Check(url2, Equals, "") c.Check(r, Equals, nil) @@ -599,7 +605,10 @@ func (s *StandaloneSuite) TestGetNetError(c *C) { kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil) r, n, url2, err := kc.Get(hash) - c.Check(err, Equals, BlockNotFound) + errNotFound, _ := err.(*ErrNotFound) + c.Check(errNotFound, NotNil) + c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true) + c.Check(errNotFound.Temporary(), Equals, true) c.Check(n, Equals, int64(0)) c.Check(url2, Equals, "") c.Check(r, Equals, nil) @@ -808,6 +817,7 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) { } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) + kc.Retries = 0 // This test works only if one of the failing services is // attempted before the succeeding service. Otherwise, @@ -885,7 +895,6 @@ func (s *StandaloneSuite) TestPutProxy(c *C) { kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 2 - kc.Using_proxy = true arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) @@ -918,7 +927,6 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) { kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 3 - kc.Using_proxy = true arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) @@ -1186,3 +1194,98 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) { c.Check(err2, Equals, nil) c.Check(content, DeepEquals, st.body[0:len(st.body)-1]) } + +type FailThenSucceedPutHandler struct { + handled chan string + count int + successhandler StubPutHandler +} + +func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + if h.count == 0 { + resp.WriteHeader(500) + h.count += 1 + h.handled <- fmt.Sprintf("http://%s", req.Host) + } else { + h.successhandler.ServeHTTP(resp, req) + } +} + +func (s *StandaloneSuite) TestPutBRetry(c *C) { + st := &FailThenSucceedPutHandler{make(chan string, 1), 0, + StubPutHandler{ + c, + Md5String("foo"), + "abc123", + "foo", + make(chan string, 5)}} + + arv, _ := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + + kc.Want_replicas = 2 + arv.ApiToken = "abc123" + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) + + ks := RunSomeFakeKeepServers(st, 2) + + for i, k := range ks { + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + defer k.listener.Close() + } + + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) + + hash, replicas, err := kc.PutB([]byte("foo")) + + c.Check(err, Equals, nil) + c.Check(hash, Equals, "") + c.Check(replicas, Equals, 2) +} + +func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) { + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, Equals, nil) + + // Add an additional "testblobstore" keepservice + blobKeepService := make(arvadosclient.Dict) + err = arv.Create("keep_services", + arvadosclient.Dict{"keep_service": arvadosclient.Dict{ + "service_host": "localhost", + "service_port": "21321", + "service_type": "testblobstore"}}, + &blobKeepService) + c.Assert(err, Equals, nil) + defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }() + + // Make a keepclient and ensure that the testblobstore is included + kc, err := MakeKeepClient(&arv) + c.Assert(err, Equals, nil) + + // verify kc.LocalRoots + c.Check(len(kc.LocalRoots()), Equals, 3) + for _, root := range kc.LocalRoots() { + c.Check(root, Matches, "http://localhost:\\d+") + } + c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") + + // verify kc.GatewayRoots + c.Check(len(kc.GatewayRoots()), Equals, 3) + for _, root := range kc.GatewayRoots() { + c.Check(root, Matches, "http://localhost:\\d+") + } + c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") + + // verify kc.WritableLocalRoots + c.Check(len(kc.WritableLocalRoots()), Equals, 3) + for _, root := range kc.WritableLocalRoots() { + c.Check(root, Matches, "http://localhost:\\d+") + } + c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") + + c.Assert(kc.replicasPerService, Equals, 0) + c.Assert(kc.foundNonDiskSvc, Equals, true) + c.Assert(kc.Client.Timeout, Equals, 300*time.Second) +}