Merge branch 'master' into 7492-keepproxy-upstream-errors

[arvados.git] / sdk / go / keepclient / keepclient_test.go

diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index e22ffe06f0c0070ce9b457acecc3f3976190018a..df4638619f488bc86dd3626cf2579a60fda62bed 100644 (file)
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -14,6 +14,7 @@ import (
        "net"
        "net/http"
        "os"
+       "strings"
        "testing"
 )
 
@@ -45,14 +46,14 @@ func (s *ServerRequiredSuite) SetUpSuite(c *C) {
                return
        }
        arvadostest.StartAPI()
-       arvadostest.StartKeep()
+       arvadostest.StartKeep(2, false)
 }
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
        if *no_server {
                return
        }
-       arvadostest.StopKeep()
+       arvadostest.StopKeep(2)
        arvadostest.StopAPI()
 }
 
@@ -69,6 +70,22 @@ func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
        }
 }
 
+func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
+       arv, err := arvadosclient.MakeArvadosClient()
+       c.Assert(err, Equals, nil)
+
+       kc, err := MakeKeepClient(&arv)
+       c.Assert(kc.Want_replicas, Equals, 2)
+
+       arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
+       kc, err = MakeKeepClient(&arv)
+       c.Assert(kc.Want_replicas, Equals, 3)
+
+       arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
+       kc, err = MakeKeepClient(&arv)
+       c.Assert(kc.Want_replicas, Equals, 1)
+}
+
 type StubPutHandler struct {
        c              *C
        expectPath     string
@@ -184,6 +201,31 @@ func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
        fh.handled <- fmt.Sprintf("http://%s", req.Host)
 }
 
+type FailThenSucceedHandler struct {
+       handled        chan string
+       count          int
+       successhandler StubGetHandler
+}
+
+func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+       if fh.count == 0 {
+               resp.WriteHeader(500)
+               fh.count += 1
+               fh.handled <- fmt.Sprintf("http://%s", req.Host)
+       } else {
+               fh.successhandler.ServeHTTP(resp, req)
+       }
+}
+
+type Error404Handler struct {
+       handled chan string
+}
+
+func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+       resp.WriteHeader(404)
+       fh.handled <- fmt.Sprintf("http://%s", req.Host)
+}
+
 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
        log.Printf("TestFailedUploadToStubKeepServer")
 
@@ -243,17 +285,17 @@ func (s *StandaloneSuite) TestPutB(c *C) {
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
-       writableRoots := make(map[string]string)
+       writableLocalRoots := make(map[string]string)
 
        ks := RunSomeFakeKeepServers(st, 5)
 
        for i, k := range ks {
                localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
-               writableRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
 
-       kc.SetServiceRoots(localRoots, nil, writableRoots)
+       kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
        kc.PutB([]byte("foo"))
 
@@ -288,17 +330,17 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
-       writableRoots := make(map[string]string)
+       writableLocalRoots := make(map[string]string)
 
        ks := RunSomeFakeKeepServers(st, 5)
 
        for i, k := range ks {
                localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
-               writableRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
 
-       kc.SetServiceRoots(localRoots, nil, writableRoots)
+       kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
        reader, writer := io.Pipe()
 
@@ -344,23 +386,23 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
-       writableRoots := make(map[string]string)
+       writableLocalRoots := make(map[string]string)
 
        ks1 := RunSomeFakeKeepServers(st, 4)
        ks2 := RunSomeFakeKeepServers(fh, 1)
 
        for i, k := range ks1 {
                localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
-               writableRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
        for i, k := range ks2 {
                localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
-               writableRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
                defer k.listener.Close()
        }
 
-       kc.SetServiceRoots(localRoots, nil, writableRoots)
+       kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
        shuff := NewRootSorter(
                kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
@@ -401,25 +443,26 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
        kc, _ := MakeKeepClient(&arv)
 
        kc.Want_replicas = 2
+       kc.Retries = 0
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
-       writableRoots := make(map[string]string)
+       writableLocalRoots := make(map[string]string)
 
        ks1 := RunSomeFakeKeepServers(st, 1)
        ks2 := RunSomeFakeKeepServers(fh, 4)
 
        for i, k := range ks1 {
                localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
-               writableRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
        for i, k := range ks2 {
                localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
-               writableRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
                defer k.listener.Close()
        }
 
-       kc.SetServiceRoots(localRoots, nil, writableRoots)
+       kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
        _, replicas, err := kc.PutB([]byte("foo"))
 
@@ -464,7 +507,7 @@ func (s *StandaloneSuite) TestGet(c *C) {
        arv, err := arvadosclient.MakeArvadosClient()
        kc, _ := MakeKeepClient(&arv)
        arv.ApiToken = "abc123"
-       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, map[string]string{ks.url: ""})
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
        r, n, url2, err := kc.Get(hash)
        defer r.Close()
@@ -479,6 +522,26 @@ func (s *StandaloneSuite) TestGet(c *C) {
        log.Printf("TestGet done")
 }
 
+func (s *StandaloneSuite) TestGet404(c *C) {
+       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+       st := Error404Handler{make(chan string, 1)}
+
+       ks := RunFakeKeepServer(st)
+       defer ks.listener.Close()
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+       r, n, url2, err := kc.Get(hash)
+       c.Check(err, Equals, BlockNotFound)
+       c.Check(n, Equals, int64(0))
+       c.Check(url2, Equals, "")
+       c.Check(r, Equals, nil)
+}
+
 func (s *StandaloneSuite) TestGetFail(c *C) {
        hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
@@ -490,10 +553,62 @@ func (s *StandaloneSuite) TestGetFail(c *C) {
        arv, err := arvadosclient.MakeArvadosClient()
        kc, _ := MakeKeepClient(&arv)
        arv.ApiToken = "abc123"
-       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, map[string]string{ks.url: ""})
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+       kc.Retries = 0
 
        r, n, url2, err := kc.Get(hash)
-       c.Check(err, Equals, BlockNotFound)
+       errNotFound, _ := err.(*ErrNotFound)
+       c.Check(errNotFound, NotNil)
+       c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
+       c.Check(errNotFound.Temporary(), Equals, true)
+       c.Check(n, Equals, int64(0))
+       c.Check(url2, Equals, "")
+       c.Check(r, Equals, nil)
+}
+
+func (s *StandaloneSuite) TestGetFailRetry(c *C) {
+       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+       st := &FailThenSucceedHandler{make(chan string, 1), 0,
+               StubGetHandler{
+                       c,
+                       hash,
+                       "abc123",
+                       http.StatusOK,
+                       []byte("foo")}}
+
+       ks := RunFakeKeepServer(st)
+       defer ks.listener.Close()
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+       r, n, url2, err := kc.Get(hash)
+       defer r.Close()
+       c.Check(err, Equals, nil)
+       c.Check(n, Equals, int64(3))
+       c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
+
+       content, err2 := ioutil.ReadAll(r)
+       c.Check(err2, Equals, nil)
+       c.Check(content, DeepEquals, []byte("foo"))
+}
+
+func (s *StandaloneSuite) TestGetNetError(c *C) {
+       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
+
+       r, n, url2, err := kc.Get(hash)
+       errNotFound, _ := err.(*ErrNotFound)
+       c.Check(errNotFound, NotNil)
+       c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
+       c.Check(errNotFound.Temporary(), Equals, true)
        c.Check(n, Equals, int64(0))
        c.Check(url2, Equals, "")
        c.Check(r, Equals, nil)
@@ -525,8 +640,8 @@ func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(
                map[string]string{"x": ks0.url},
-               map[string]string{uuid: ks.url},
-               map[string]string{ks0.url: "", ks.url: ""})
+               nil,
+               map[string]string{uuid: ks.url})
 
        r, n, uri, err := kc.Get(hash + "+K@" + uuid)
        defer r.Close()
@@ -572,11 +687,7 @@ func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
                        "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
                        "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
                        uuid: ks.url},
-               map[string]string{
-                       "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
-                       "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
-                       "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
-                       uuid: ks.url},
+               nil,
                map[string]string{
                        "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
                        "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
@@ -619,8 +730,8 @@ func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(
                map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
-               map[string]string{uuid: ksGateway.url},
-               map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url})
+               nil,
+               map[string]string{uuid: ksGateway.url})
 
        r, n, uri, err := kc.Get(hash + "+K@" + uuid)
        c.Assert(err, Equals, nil)
@@ -654,7 +765,7 @@ func (s *StandaloneSuite) TestChecksum(c *C) {
        arv, err := arvadosclient.MakeArvadosClient()
        kc, _ := MakeKeepClient(&arv)
        arv.ApiToken = "abc123"
-       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, map[string]string{ks.url: ""})
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
        r, n, _, err := kc.Get(barhash)
        _, err = ioutil.ReadAll(r)
@@ -675,7 +786,7 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) {
        content := []byte("waz")
        hash := fmt.Sprintf("%x", md5.Sum(content))
 
-       fh := FailHandler{
+       fh := Error404Handler{
                make(chan string, 4)}
 
        st := StubGetHandler{
@@ -689,23 +800,24 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) {
        kc, _ := MakeKeepClient(&arv)
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
-       writableRoots := make(map[string]string)
+       writableLocalRoots := make(map[string]string)
 
        ks1 := RunSomeFakeKeepServers(st, 1)
        ks2 := RunSomeFakeKeepServers(fh, 4)
 
        for i, k := range ks1 {
                localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
-               writableRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
        for i, k := range ks2 {
                localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
-               writableRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
                defer k.listener.Close()
        }
 
-       kc.SetServiceRoots(localRoots, nil, writableRoots)
+       kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+       kc.Retries = 0
 
        // This test works only if one of the failing services is
        // attempted before the succeeding service. Otherwise,
@@ -743,7 +855,7 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
        }
        {
                hash2, replicas, err := kc.PutB(content)
-               c.Check(hash2, Equals, fmt.Sprintf("%s+%d", hash, len(content)))
+               c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content)))
                c.Check(replicas, Equals, 2)
                c.Check(err, Equals, nil)
        }
@@ -786,17 +898,17 @@ func (s *StandaloneSuite) TestPutProxy(c *C) {
        kc.Using_proxy = true
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
-       writableRoots := make(map[string]string)
+       writableLocalRoots := make(map[string]string)
 
        ks1 := RunSomeFakeKeepServers(st, 1)
 
        for i, k := range ks1 {
                localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
-               writableRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
 
-       kc.SetServiceRoots(localRoots, nil, writableRoots)
+       kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
        _, replicas, err := kc.PutB([]byte("foo"))
        <-st.handled
@@ -819,16 +931,16 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
        kc.Using_proxy = true
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
-       writableRoots := make(map[string]string)
+       writableLocalRoots := make(map[string]string)
 
        ks1 := RunSomeFakeKeepServers(st, 1)
 
        for i, k := range ks1 {
                localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
-               writableRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
-       kc.SetServiceRoots(localRoots, nil, writableRoots)
+       kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
        _, replicas, err := kc.PutB([]byte("foo"))
        <-st.handled
@@ -878,9 +990,7 @@ func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
        c.Check(err, Equals, InvalidLocatorError)
 }
 
-func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableRoots(c *C) {
-       log.Printf("TestPutWant2ReplicasWithOnlyOneWritableRoots")
-
+func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
        hash := Md5String("foo")
 
        st := StubPutHandler{
@@ -896,19 +1006,19 @@ func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableRoots(c *C) {
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
-       writableRoots := make(map[string]string)
+       writableLocalRoots := make(map[string]string)
 
        ks := RunSomeFakeKeepServers(st, 5)
 
        for i, k := range ks {
                localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                if i == 0 {
-                       writableRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+                       writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                }
                defer k.listener.Close()
        }
 
-       kc.SetServiceRoots(localRoots, nil, writableRoots)
+       kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
        _, replicas, err := kc.PutB([]byte("foo"))
 
@@ -916,13 +1026,9 @@ func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableRoots(c *C) {
        c.Check(replicas, Equals, 1)
 
        c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
-
-       log.Printf("TestPutWant2ReplicasWithOnlyOneWritableRoots done")
 }
 
-func (s *StandaloneSuite) TestPutBWithNoWritableRoots(c *C) {
-       log.Printf("TestPutBWithNoWritableRoots")
-
+func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
        hash := Md5String("foo")
 
        st := StubPutHandler{
@@ -938,7 +1044,7 @@ func (s *StandaloneSuite) TestPutBWithNoWritableRoots(c *C) {
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
-       writableRoots := make(map[string]string)
+       writableLocalRoots := make(map[string]string)
 
        ks := RunSomeFakeKeepServers(st, 5)
 
@@ -947,12 +1053,196 @@ func (s *StandaloneSuite) TestPutBWithNoWritableRoots(c *C) {
                defer k.listener.Close()
        }
 
-       kc.SetServiceRoots(localRoots, nil, writableRoots)
+       kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
        _, replicas, err := kc.PutB([]byte("foo"))
 
        c.Check(err, Equals, InsufficientReplicasError)
        c.Check(replicas, Equals, 0)
+}
+
+type StubGetIndexHandler struct {
+       c              *C
+       expectPath     string
+       expectAPIToken string
+       httpStatus     int
+       body           []byte
+}
+
+func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+       h.c.Check(req.URL.Path, Equals, h.expectPath)
+       h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
+       resp.WriteHeader(h.httpStatus)
+       resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
+       resp.Write(h.body)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
+       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+       st := StubGetIndexHandler{
+               c,
+               "/index",
+               "abc123",
+               http.StatusOK,
+               []byte(hash + "+3 1443559274\n\n")}
+
+       ks := RunFakeKeepServer(st)
+       defer ks.listener.Close()
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+       r, err := kc.GetIndex("x", "")
+       c.Check(err, Equals, nil)
+
+       content, err2 := ioutil.ReadAll(r)
+       c.Check(err2, Equals, nil)
+       c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
+}
+
+func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
+       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+       st := StubGetIndexHandler{
+               c,
+               "/index/" + hash[0:3],
+               "abc123",
+               http.StatusOK,
+               []byte(hash + "+3 1443559274\n\n")}
+
+       ks := RunFakeKeepServer(st)
+       defer ks.listener.Close()
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+       r, err := kc.GetIndex("x", hash[0:3])
+       c.Check(err, Equals, nil)
+
+       content, err2 := ioutil.ReadAll(r)
+       c.Check(err2, Equals, nil)
+       c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
+}
 
-       log.Printf("TestPutBWithNoWritableRoots done")
+func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
+       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+       st := StubGetIndexHandler{
+               c,
+               "/index/" + hash[0:3],
+               "abc123",
+               http.StatusOK,
+               []byte(hash)}
+
+       ks := RunFakeKeepServer(st)
+       defer ks.listener.Close()
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+       _, err = kc.GetIndex("x", hash[0:3])
+       c.Check(err, Equals, ErrIncompleteIndex)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
+       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+       st := StubGetIndexHandler{
+               c,
+               "/index/" + hash[0:3],
+               "abc123",
+               http.StatusOK,
+               []byte(hash)}
+
+       ks := RunFakeKeepServer(st)
+       defer ks.listener.Close()
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+       _, err = kc.GetIndex("y", hash[0:3])
+       c.Check(err, Equals, ErrNoSuchKeepServer)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
+       st := StubGetIndexHandler{
+               c,
+               "/index/abcd",
+               "abc123",
+               http.StatusOK,
+               []byte("\n")}
+
+       ks := RunFakeKeepServer(st)
+       defer ks.listener.Close()
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+       r, err := kc.GetIndex("x", "abcd")
+       c.Check(err, Equals, nil)
+
+       content, err2 := ioutil.ReadAll(r)
+       c.Check(err2, Equals, nil)
+       c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
+}
+
+type FailThenSucceedPutHandler struct {
+       handled        chan string
+       count          int
+       successhandler StubPutHandler
+}
+
+func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+       if h.count == 0 {
+               resp.WriteHeader(500)
+               h.count += 1
+               h.handled <- fmt.Sprintf("http://%s", req.Host)
+       } else {
+               h.successhandler.ServeHTTP(resp, req)
+       }
+}
+
+func (s *StandaloneSuite) TestPutBRetry(c *C) {
+       st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
+               StubPutHandler{
+                       c,
+                       Md5String("foo"),
+                       "abc123",
+                       "foo",
+                       make(chan string, 5)}}
+
+       arv, _ := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+
+       kc.Want_replicas = 2
+       arv.ApiToken = "abc123"
+       localRoots := make(map[string]string)
+       writableLocalRoots := make(map[string]string)
+
+       ks := RunSomeFakeKeepServers(st, 2)
+
+       for i, k := range ks {
+               localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+               defer k.listener.Close()
+       }
+
+       kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+
+       hash, replicas, err := kc.PutB([]byte("foo"))
+
+       c.Check(err, Equals, nil)
+       c.Check(hash, Equals, "")
+       c.Check(replicas, Equals, 2)
 }