X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/77b2a1d82d032f7e29f9c477d32127a4435ec13e..e31f6c9eb427302eb48150d431afcc962954b061:/sdk/go/keepclient/keepclient_test.go diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go index 4acaf70fe4..4ba1d7c245 100644 --- a/sdk/go/keepclient/keepclient_test.go +++ b/sdk/go/keepclient/keepclient_test.go @@ -16,6 +16,7 @@ import ( "os" "strings" "testing" + "time" ) // Gocheck boilerplate @@ -143,10 +144,9 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) { make(chan string)} UploadToStubHelper(c, st, - func(kc *KeepClient, url string, reader io.ReadCloser, - writer io.WriteCloser, upload_status chan uploadStatus) { + func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) { - go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer") + go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0) writer.Write([]byte("foo")) writer.Close() @@ -155,13 +155,9 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) { status := <-upload_status c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""}) }) - - log.Printf("TestUploadToStubKeepServer done") } func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) { - log.Printf("TestUploadToStubKeepServerBufferReader") - st := StubPutHandler{ c, "acbd18db4cc2f85cedef654fccc4a4d8", @@ -178,7 +174,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) { br1 := tr.MakeStreamReader() - go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader") + go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0) writer.Write([]byte("foo")) writer.Close() @@ -188,8 +184,6 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) { status := <-upload_status c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""}) }) - - log.Printf("TestUploadToStubKeepServerBufferReader done") } type FailHandler struct { @@ -227,8 +221,6 @@ func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) } func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) { - log.Printf("TestFailedUploadToStubKeepServer") - st := FailHandler{ make(chan string)} @@ -238,7 +230,7 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) { func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) { - go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer") + go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0) writer.Write([]byte("foo")) writer.Close() @@ -249,7 +241,6 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) { c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash)) c.Check(status.statusCode, Equals, 500) }) - log.Printf("TestFailedUploadToStubKeepServer done") } type KeepServer struct { @@ -268,8 +259,6 @@ func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) { } func (s *StandaloneSuite) TestPutB(c *C) { - log.Printf("TestPutB") - hash := Md5String("foo") st := StubPutHandler{ @@ -308,13 +297,9 @@ func (s *StandaloneSuite) TestPutB(c *C) { (s1 == shuff[1] && s2 == shuff[0]), Equals, true) - - log.Printf("TestPutB done") } func (s *StandaloneSuite) TestPutHR(c *C) { - log.Printf("TestPutHR") - hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) st := StubPutHandler{ @@ -352,7 +337,6 @@ func (s *StandaloneSuite) TestPutHR(c *C) { kc.PutHR(hash, reader, 3) shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots() - log.Print(shuff) s1 := <-st.handled s2 := <-st.handled @@ -361,13 +345,9 @@ func (s *StandaloneSuite) TestPutHR(c *C) { (s1 == shuff[1] && s2 == shuff[0]), Equals, true) - - log.Printf("TestPutHR done") } func (s *StandaloneSuite) TestPutWithFail(c *C) { - log.Printf("TestPutWithFail") - hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) st := StubPutHandler{ @@ -406,6 +386,7 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) { shuff := NewRootSorter( kc.LocalRoots(), Md5String("foo")).GetSortedRoots() + c.Logf("%+v", shuff) phash, replicas, err := kc.PutB([]byte("foo")) @@ -425,8 +406,6 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) { } func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) { - log.Printf("TestPutWithTooManyFail") - hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) st := StubPutHandler{ @@ -469,8 +448,6 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) { c.Check(err, Equals, InsufficientReplicasError) c.Check(replicas, Equals, 1) c.Check(<-st.handled, Equals, ks1[0].url) - - log.Printf("TestPutWithTooManyFail done") } type StubGetHandler struct { @@ -490,8 +467,6 @@ func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) } func (s *StandaloneSuite) TestGet(c *C) { - log.Printf("TestGet") - hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) st := StubGetHandler{ @@ -518,8 +493,6 @@ func (s *StandaloneSuite) TestGet(c *C) { content, err2 := ioutil.ReadAll(r) c.Check(err2, Equals, nil) c.Check(content, DeepEquals, []byte("foo")) - - log.Printf("TestGet done") } func (s *StandaloneSuite) TestGet404(c *C) { @@ -557,8 +530,10 @@ func (s *StandaloneSuite) TestGetFail(c *C) { kc.Retries = 0 r, n, url2, err := kc.Get(hash) - errNotFound, _ := err.(ErrNotFound) + errNotFound, _ := err.(*ErrNotFound) c.Check(errNotFound, NotNil) + c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true) + c.Check(errNotFound.Temporary(), Equals, true) c.Check(n, Equals, int64(0)) c.Check(url2, Equals, "") c.Check(r, Equals, nil) @@ -603,9 +578,10 @@ func (s *StandaloneSuite) TestGetNetError(c *C) { kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil) r, n, url2, err := kc.Get(hash) - errNotFound, _ := err.(ErrNotFound) + errNotFound, _ := err.(*ErrNotFound) c.Check(errNotFound, NotNil) - c.Check(strings.Contains(err.Error(), "connection refused"), Equals, true) + c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true) + c.Check(errNotFound.Temporary(), Equals, true) c.Check(n, Equals, int64(0)) c.Check(url2, Equals, "") c.Check(r, Equals, nil) @@ -884,15 +860,12 @@ func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reque } func (s *StandaloneSuite) TestPutProxy(c *C) { - log.Printf("TestPutProxy") - st := StubProxyHandler{make(chan string, 1)} arv, err := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 2 - kc.Using_proxy = true arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) @@ -912,20 +885,15 @@ func (s *StandaloneSuite) TestPutProxy(c *C) { c.Check(err, Equals, nil) c.Check(replicas, Equals, 2) - - log.Printf("TestPutProxy done") } func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) { - log.Printf("TestPutProxy") - st := StubProxyHandler{make(chan string, 1)} arv, err := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 3 - kc.Using_proxy = true arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) @@ -944,8 +912,6 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) { c.Check(err, Equals, InsufficientReplicasError) c.Check(replicas, Equals, 2) - - log.Printf("TestPutProxy done") } func (s *StandaloneSuite) TestMakeLocator(c *C) { @@ -1243,3 +1209,48 @@ func (s *StandaloneSuite) TestPutBRetry(c *C) { c.Check(hash, Equals, "") c.Check(replicas, Equals, 2) } + +func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) { + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, Equals, nil) + + // Add an additional "testblobstore" keepservice + blobKeepService := make(arvadosclient.Dict) + err = arv.Create("keep_services", + arvadosclient.Dict{"keep_service": arvadosclient.Dict{ + "service_host": "localhost", + "service_port": "21321", + "service_type": "testblobstore"}}, + &blobKeepService) + c.Assert(err, Equals, nil) + defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }() + + // Make a keepclient and ensure that the testblobstore is included + kc, err := MakeKeepClient(&arv) + c.Assert(err, Equals, nil) + + // verify kc.LocalRoots + c.Check(len(kc.LocalRoots()), Equals, 3) + for _, root := range kc.LocalRoots() { + c.Check(root, Matches, "http://localhost:\\d+") + } + c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") + + // verify kc.GatewayRoots + c.Check(len(kc.GatewayRoots()), Equals, 3) + for _, root := range kc.GatewayRoots() { + c.Check(root, Matches, "http://localhost:\\d+") + } + c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") + + // verify kc.WritableLocalRoots + c.Check(len(kc.WritableLocalRoots()), Equals, 3) + for _, root := range kc.WritableLocalRoots() { + c.Check(root, Matches, "http://localhost:\\d+") + } + c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") + + c.Assert(kc.replicasPerService, Equals, 0) + c.Assert(kc.foundNonDiskSvc, Equals, true) + c.Assert(kc.Client.Timeout, Equals, 300*time.Second) +}