7491: Manual merge with 7491-keepclient-bugs

[arvados.git] / sdk / go / keepclient / keepclient_test.go

diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 3724cdf8c578182aa6b9e54c6c1ce47e897802b8..95269bfef6e9b5382d29c0913ef79f61d30a5bfd 100644 (file)
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -69,6 +69,22 @@ func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
        }
 }
 
+func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
+       arv, err := arvadosclient.MakeArvadosClient()
+       c.Assert(err, Equals, nil)
+
+       kc, err := MakeKeepClient(&arv)
+       c.Assert(kc.Want_replicas, Equals, 2)
+
+       arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
+       kc, err = MakeKeepClient(&arv)
+       c.Assert(kc.Want_replicas, Equals, 3)
+
+       arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
+       kc, err = MakeKeepClient(&arv)
+       c.Assert(kc.Want_replicas, Equals, 1)
+}
+
 type StubPutHandler struct {
        c              *C
        expectPath     string
@@ -184,6 +200,31 @@ func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
        fh.handled <- fmt.Sprintf("http://%s", req.Host)
 }
 
+type FailThenSucceedHandler struct {
+       handled        chan string
+       count          int
+       successhandler StubGetHandler
+}
+
+func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+       if fh.count == 0 {
+               resp.WriteHeader(500)
+               fh.count += 1
+               fh.handled <- fmt.Sprintf("http://%s", req.Host)
+       } else {
+               fh.successhandler.ServeHTTP(resp, req)
+       }
+}
+
+type Error404Handler struct {
+       handled chan string
+}
+
+func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+       resp.WriteHeader(404)
+       fh.handled <- fmt.Sprintf("http://%s", req.Host)
+}
+
 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
        log.Printf("TestFailedUploadToStubKeepServer")
 
@@ -479,6 +520,26 @@ func (s *StandaloneSuite) TestGet(c *C) {
        log.Printf("TestGet done")
 }
 
+func (s *StandaloneSuite) TestGet404(c *C) {
+       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+       st := Error404Handler{make(chan string, 1)}
+
+       ks := RunFakeKeepServer(st)
+       defer ks.listener.Close()
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+       r, n, url2, err := kc.Get(hash)
+       c.Check(err, Equals, BlockNotFound)
+       c.Check(n, Equals, int64(0))
+       c.Check(url2, Equals, "")
+       c.Check(r, Equals, nil)
+}
+
 func (s *StandaloneSuite) TestGetFail(c *C) {
        hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
@@ -493,7 +554,52 @@ func (s *StandaloneSuite) TestGetFail(c *C) {
        kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
 
        r, n, url2, err := kc.Get(hash)
-       c.Check(err, Equals, BlockNotFound)
+       c.Check(err, Equals, KeepServerError)
+       c.Check(n, Equals, int64(0))
+       c.Check(url2, Equals, "")
+       c.Check(r, Equals, nil)
+}
+
+func (s *StandaloneSuite) TestGetFailRetry(c *C) {
+       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+       st := &FailThenSucceedHandler{make(chan string, 1), 0,
+               StubGetHandler{
+                       c,
+                       hash,
+                       "abc123",
+                       http.StatusOK,
+                       []byte("foo")}}
+
+       ks := RunFakeKeepServer(st)
+       defer ks.listener.Close()
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+       r, n, url2, err := kc.Get(hash)
+       defer r.Close()
+       c.Check(err, Equals, nil)
+       c.Check(n, Equals, int64(3))
+       c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
+
+       content, err2 := ioutil.ReadAll(r)
+       c.Check(err2, Equals, nil)
+       c.Check(content, DeepEquals, []byte("foo"))
+}
+
+func (s *StandaloneSuite) TestGetNetError(c *C) {
+       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, map[string]string{"http://localhost:62222": ""}, nil)
+
+       r, n, url2, err := kc.Get(hash)
+       c.Check(err, Equals, KeepServerError)
        c.Check(n, Equals, int64(0))
        c.Check(url2, Equals, "")
        c.Check(r, Equals, nil)
@@ -675,7 +781,7 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) {
        content := []byte("waz")
        hash := fmt.Sprintf("%x", md5.Sum(content))
 
-       fh := FailHandler{
+       fh := Error404Handler{
                make(chan string, 4)}
 
        st := StubGetHandler{
@@ -952,14 +1058,14 @@ func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
 type StubGetIndexHandler struct {
        c              *C
        expectPath     string
-       expectApiToken string
+       expectAPIToken string
        httpStatus     int
        body           []byte
 }
 
 func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
        h.c.Check(req.URL.Path, Equals, h.expectPath)
-       h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectApiToken))
+       h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
        resp.WriteHeader(h.httpStatus)
        resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
        resp.Write(h.body)
@@ -973,7 +1079,7 @@ func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
                "/index",
                "abc123",
                http.StatusOK,
-               []byte(string(hash) + "\n\n")}
+               []byte(hash + "+3 1443559274\n\n")}
 
        ks := RunFakeKeepServer(st)
        defer ks.listener.Close()
@@ -988,7 +1094,7 @@ func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
 
        content, err2 := ioutil.ReadAll(r)
        c.Check(err2, Equals, nil)
-       c.Check(content, DeepEquals, st.body)
+       c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
 }
 
 func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
@@ -999,7 +1105,7 @@ func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
                "/index/" + hash[0:3],
                "abc123",
                http.StatusOK,
-               []byte(string(hash) + "\n\n")}
+               []byte(hash + "+3 1443559274\n\n")}
 
        ks := RunFakeKeepServer(st)
        defer ks.listener.Close()
@@ -1014,7 +1120,7 @@ func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
 
        content, err2 := ioutil.ReadAll(r)
        c.Check(err2, Equals, nil)
-       c.Check(content, DeepEquals, st.body)
+       c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
 }
 
 func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
@@ -1025,7 +1131,7 @@ func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
                "/index/" + hash[0:3],
                "abc123",
                http.StatusOK,
-               []byte(string(hash))}
+               []byte(hash)}
 
        ks := RunFakeKeepServer(st)
        defer ks.listener.Close()
@@ -1036,7 +1142,7 @@ func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
        kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
 
        _, err = kc.GetIndex("x", hash[0:3])
-       c.Check(err, Equals, ErrorIncompleteIndex)
+       c.Check(err, Equals, ErrIncompleteIndex)
 }
 
 func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
@@ -1047,7 +1153,7 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
                "/index/" + hash[0:3],
                "abc123",
                http.StatusOK,
-               []byte(string(hash))}
+               []byte(hash)}
 
        ks := RunFakeKeepServer(st)
        defer ks.listener.Close()
@@ -1058,7 +1164,7 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
        kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
 
        _, err = kc.GetIndex("y", hash[0:3])
-       c.Check(err, Equals, ErrorNoSuchKeepServer)
+       c.Check(err, Equals, ErrNoSuchKeepServer)
 }
 
 func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
@@ -1082,5 +1188,5 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
 
        content, err2 := ioutil.ReadAll(r)
        c.Check(err2, Equals, nil)
-       c.Check(content, DeepEquals, st.body)
+       c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
 }