X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/8653e6d34f5d895ee56fe3657c7fc3ca3cf615f4..3e1c43866e5b523c3f1d273c25942ad56dc66d3f:/sdk/go/keepclient/keepclient_test.go

diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index e2c4d9026c..392270909f 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -1,20 +1,27 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
 package keepclient
 
 import (
+	"bytes"
 	"crypto/md5"
-	"flag"
+	"errors"
 	"fmt"
-	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
-	"git.curoverse.com/arvados.git/sdk/go/streamer"
-	. "gopkg.in/check.v1"
 	"io"
 	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
 	"os"
+	"strings"
 	"testing"
+	"time"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+	. "gopkg.in/check.v1"
 )
 
 // Gocheck boilerplate
@@ -26,41 +33,40 @@ func Test(t *testing.T) {
 var _ = Suite(&ServerRequiredSuite{})
 var _ = Suite(&StandaloneSuite{})
 
-var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
-
 // Tests that require the Keep server running
 type ServerRequiredSuite struct{}
 
 // Standalone tests
 type StandaloneSuite struct{}
 
+func (s *StandaloneSuite) SetUpTest(c *C) {
+	RefreshServiceDiscovery()
+}
+
 func pythonDir() string {
 	cwd, _ := os.Getwd()
 	return fmt.Sprintf("%s/../../python/tests", cwd)
 }
 
 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
-	if *no_server {
-		c.Skip("Skipping tests that require server")
-		return
-	}
 	arvadostest.StartAPI()
-	arvadostest.StartKeep()
+	arvadostest.StartKeep(2, false)
 }
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
-	if *no_server {
-		return
-	}
-	arvadostest.StopKeep()
+	arvadostest.StopKeep(2)
 	arvadostest.StopAPI()
 }
 
+func (s *ServerRequiredSuite) SetUpTest(c *C) {
+	RefreshServiceDiscovery()
+}
+
 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
 	arv, err := arvadosclient.MakeArvadosClient()
 	c.Assert(err, Equals, nil)
 
-	kc, err := MakeKeepClient(&arv)
+	kc, err := MakeKeepClient(arv)
 
 	c.Assert(err, Equals, nil)
 	c.Check(len(kc.LocalRoots()), Equals, 2)
@@ -69,6 +75,23 @@ func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
 	}
 }
 
+func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
+	arv, err := arvadosclient.MakeArvadosClient()
+	c.Assert(err, Equals, nil)
+
+	kc, err := MakeKeepClient(arv)
+	c.Assert(kc.Want_replicas, Equals, 2)
+
+	arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
+	kc, err = MakeKeepClient(arv)
+	c.Assert(kc.Want_replicas, Equals, 3)
+
+	arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
+	kc, err = MakeKeepClient(arv)
+	c.Check(err, IsNil)
+	c.Assert(kc.Want_replicas, Equals, 1)
+}
+
 type StubPutHandler struct {
 	c              *C
 	expectPath     string
@@ -89,7 +112,9 @@ func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
 
 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
 	var err error
-	ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
+	// If we don't explicitly bind it to localhost, ks.listener.Addr() will
+	// bind to 0.0.0.0 or [::] which is not a valid address for Dial()
+	ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: 0})
 	if err != nil {
 		panic(fmt.Sprintf("Could not listen on any port"))
 	}
@@ -107,7 +132,7 @@ func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
 	arv, _ := arvadosclient.MakeArvadosClient()
 	arv.ApiToken = "abc123"
 
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	reader, writer := io.Pipe()
 	upload_status := make(chan uploadStatus)
@@ -126,10 +151,9 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 		make(chan string)}
 
 	UploadToStubHelper(c, st,
-		func(kc *KeepClient, url string, reader io.ReadCloser,
-			writer io.WriteCloser, upload_status chan uploadStatus) {
+		func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
 
-			go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer")
+			go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
 
 			writer.Write([]byte("foo"))
 			writer.Close()
@@ -138,13 +162,9 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 			status := <-upload_status
 			c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 		})
-
-	log.Printf("TestUploadToStubKeepServer done")
 }
 
 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
-	log.Printf("TestUploadToStubKeepServerBufferReader")
-
 	st := StubPutHandler{
 		c,
 		"acbd18db4cc2f85cedef654fccc4a4d8",
@@ -153,26 +173,14 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 		make(chan string)}
 
 	UploadToStubHelper(c, st,
-		func(kc *KeepClient, url string, reader io.ReadCloser,
-			writer io.WriteCloser, upload_status chan uploadStatus) {
-
-			tr := streamer.AsyncStreamFromReader(512, reader)
-			defer tr.Close()
-
-			br1 := tr.MakeStreamReader()
-
-			go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader")
-
-			writer.Write([]byte("foo"))
-			writer.Close()
+		func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) {
+			go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, 0)
 
 			<-st.handled
 
 			status := <-upload_status
 			c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 		})
-
-	log.Printf("TestUploadToStubKeepServerBufferReader done")
 }
 
 type FailHandler struct {
@@ -210,8 +218,6 @@ func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
 }
 
 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
-	log.Printf("TestFailedUploadToStubKeepServer")
-
 	st := FailHandler{
 		make(chan string)}
 
@@ -221,7 +227,7 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 		func(kc *KeepClient, url string, reader io.ReadCloser,
 			writer io.WriteCloser, upload_status chan uploadStatus) {
 
-			go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer")
+			go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
 
 			writer.Write([]byte("foo"))
 			writer.Close()
@@ -232,7 +238,6 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 			c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
 			c.Check(status.statusCode, Equals, 500)
 		})
-	log.Printf("TestFailedUploadToStubKeepServer done")
 }
 
 type KeepServer struct {
@@ -251,8 +256,6 @@ func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
 }
 
 func (s *StandaloneSuite) TestPutB(c *C) {
-	log.Printf("TestPutB")
-
 	hash := Md5String("foo")
 
 	st := StubPutHandler{
@@ -263,7 +266,7 @@ func (s *StandaloneSuite) TestPutB(c *C) {
 		make(chan string, 5)}
 
 	arv, _ := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
 	arv.ApiToken = "abc123"
@@ -291,13 +294,9 @@ func (s *StandaloneSuite) TestPutB(c *C) {
 		(s1 == shuff[1] && s2 == shuff[0]),
 		Equals,
 		true)
-
-	log.Printf("TestPutB done")
 }
 
 func (s *StandaloneSuite) TestPutHR(c *C) {
-	log.Printf("TestPutHR")
-
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
 	st := StubPutHandler{
@@ -308,7 +307,7 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
 		make(chan string, 5)}
 
 	arv, _ := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
 	arv.ApiToken = "abc123"
@@ -335,7 +334,6 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
 	kc.PutHR(hash, reader, 3)
 
 	shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
-	log.Print(shuff)
 
 	s1 := <-st.handled
 	s2 := <-st.handled
@@ -344,13 +342,9 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
 		(s1 == shuff[1] && s2 == shuff[0]),
 		Equals,
 		true)
-
-	log.Printf("TestPutHR done")
 }
 
 func (s *StandaloneSuite) TestPutWithFail(c *C) {
-	log.Printf("TestPutWithFail")
-
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
 	st := StubPutHandler{
@@ -364,7 +358,7 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
 		make(chan string, 1)}
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
 	arv.ApiToken = "abc123"
@@ -389,6 +383,7 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
 
 	shuff := NewRootSorter(
 		kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
+	c.Logf("%+v", shuff)
 
 	phash, replicas, err := kc.PutB([]byte("foo"))
 
@@ -408,8 +403,6 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
 }
 
 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
-	log.Printf("TestPutWithTooManyFail")
-
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
 	st := StubPutHandler{
@@ -423,9 +416,10 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 		make(chan string, 4)}
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
+	kc.Retries = 0
 	arv.ApiToken = "abc123"
 	localRoots := make(map[string]string)
 	writableLocalRoots := make(map[string]string)
@@ -448,11 +442,9 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 
 	_, replicas, err := kc.PutB([]byte("foo"))
 
-	c.Check(err, Equals, InsufficientReplicasError)
+	c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
 	c.Check(replicas, Equals, 1)
 	c.Check(<-st.handled, Equals, ks1[0].url)
-
-	log.Printf("TestPutWithTooManyFail done")
 }
 
 type StubGetHandler struct {
@@ -472,8 +464,6 @@ func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
 }
 
 func (s *StandaloneSuite) TestGet(c *C) {
-	log.Printf("TestGet")
-
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
 	st := StubGetHandler{
@@ -487,9 +477,9 @@ func (s *StandaloneSuite) TestGet(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
 	r, n, url2, err := kc.Get(hash)
 	defer r.Close()
@@ -500,8 +490,6 @@ func (s *StandaloneSuite) TestGet(c *C) {
 	content, err2 := ioutil.ReadAll(r)
 	c.Check(err2, Equals, nil)
 	c.Check(content, DeepEquals, []byte("foo"))
-
-	log.Printf("TestGet done")
 }
 
 func (s *StandaloneSuite) TestGet404(c *C) {
@@ -513,9 +501,9 @@ func (s *StandaloneSuite) TestGet404(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
 	r, n, url2, err := kc.Get(hash)
 	c.Check(err, Equals, BlockNotFound)
@@ -524,6 +512,27 @@ func (s *StandaloneSuite) TestGet404(c *C) {
 	c.Check(r, Equals, nil)
 }
 
+func (s *StandaloneSuite) TestGetEmptyBlock(c *C) {
+	st := Error404Handler{make(chan string, 1)}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+	r, n, url2, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e+0")
+	c.Check(err, IsNil)
+	c.Check(n, Equals, int64(0))
+	c.Check(url2, Equals, "")
+	c.Assert(r, NotNil)
+	buf, err := ioutil.ReadAll(r)
+	c.Check(err, IsNil)
+	c.Check(buf, DeepEquals, []byte{})
+}
+
 func (s *StandaloneSuite) TestGetFail(c *C) {
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
@@ -533,12 +542,16 @@ func (s *StandaloneSuite) TestGetFail(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+	kc.Retries = 0
 
 	r, n, url2, err := kc.Get(hash)
-	c.Check(err, Equals, KeepServerError)
+	errNotFound, _ := err.(*ErrNotFound)
+	c.Check(errNotFound, NotNil)
+	c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
+	c.Check(errNotFound.Temporary(), Equals, true)
 	c.Check(n, Equals, int64(0))
 	c.Check(url2, Equals, "")
 	c.Check(r, Equals, nil)
@@ -559,9 +572,9 @@ func (s *StandaloneSuite) TestGetFailRetry(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
 	r, n, url2, err := kc.Get(hash)
 	defer r.Close()
@@ -578,12 +591,15 @@ func (s *StandaloneSuite) TestGetNetError(c *C) {
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, map[string]string{"http://localhost:62222": ""}, nil)
+	kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
 
 	r, n, url2, err := kc.Get(hash)
-	c.Check(err, Equals, KeepServerError)
+	errNotFound, _ := err.(*ErrNotFound)
+	c.Check(errNotFound, NotNil)
+	c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
+	c.Check(errNotFound.Temporary(), Equals, true)
 	c.Check(n, Equals, int64(0))
 	c.Check(url2, Equals, "")
 	c.Check(r, Equals, nil)
@@ -611,11 +627,11 @@ func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
 	kc.SetServiceRoots(
 		map[string]string{"x": ks0.url},
-		map[string]string{"x": ks0.url},
+		nil,
 		map[string]string{uuid: ks.url})
 
 	r, n, uri, err := kc.Get(hash + "+K@" + uuid)
@@ -654,7 +670,7 @@ func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
 	kc.SetServiceRoots(
 		map[string]string{
@@ -662,11 +678,7 @@ func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
 			"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
 			"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
 			uuid: ks.url},
-		map[string]string{
-			"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
-			"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
-			"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
-			uuid: ks.url},
+		nil,
 		map[string]string{
 			"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
 			"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
@@ -705,11 +717,11 @@ func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
 	defer ksGateway.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
 	kc.SetServiceRoots(
 		map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
-		map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
+		nil,
 		map[string]string{uuid: ksGateway.url})
 
 	r, n, uri, err := kc.Get(hash + "+K@" + uuid)
@@ -742,9 +754,9 @@ func (s *StandaloneSuite) TestChecksum(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
 	r, n, _, err := kc.Get(barhash)
 	_, err = ioutil.ReadAll(r)
@@ -776,7 +788,7 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 		content}
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
 	localRoots := make(map[string]string)
 	writableLocalRoots := make(map[string]string)
@@ -796,6 +808,7 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 	}
 
 	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+	kc.Retries = 0
 
 	// This test works only if one of the failing services is
 	// attempted before the succeeding service. Otherwise,
@@ -821,7 +834,7 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
 	content := []byte("TestPutGetHead")
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, err := MakeKeepClient(&arv)
+	kc, err := MakeKeepClient(arv)
 	c.Assert(err, Equals, nil)
 
 	hash := fmt.Sprintf("%x", md5.Sum(content))
@@ -865,15 +878,12 @@ func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reque
 }
 
 func (s *StandaloneSuite) TestPutProxy(c *C) {
-	log.Printf("TestPutProxy")
-
 	st := StubProxyHandler{make(chan string, 1)}
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
-	kc.Using_proxy = true
 	arv.ApiToken = "abc123"
 	localRoots := make(map[string]string)
 	writableLocalRoots := make(map[string]string)
@@ -893,20 +903,15 @@ func (s *StandaloneSuite) TestPutProxy(c *C) {
 
 	c.Check(err, Equals, nil)
 	c.Check(replicas, Equals, 2)
-
-	log.Printf("TestPutProxy done")
 }
 
 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
-	log.Printf("TestPutProxy")
-
 	st := StubProxyHandler{make(chan string, 1)}
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 3
-	kc.Using_proxy = true
 	arv.ApiToken = "abc123"
 	localRoots := make(map[string]string)
 	writableLocalRoots := make(map[string]string)
@@ -923,10 +928,8 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
 	_, replicas, err := kc.PutB([]byte("foo"))
 	<-st.handled
 
-	c.Check(err, Equals, InsufficientReplicasError)
+	c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
 	c.Check(replicas, Equals, 2)
-
-	log.Printf("TestPutProxy done")
 }
 
 func (s *StandaloneSuite) TestMakeLocator(c *C) {
@@ -979,7 +982,7 @@ func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C
 		make(chan string, 5)}
 
 	arv, _ := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
 	arv.ApiToken = "abc123"
@@ -1000,7 +1003,7 @@ func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C
 
 	_, replicas, err := kc.PutB([]byte("foo"))
 
-	c.Check(err, Equals, InsufficientReplicasError)
+	c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
 	c.Check(replicas, Equals, 1)
 
 	c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
@@ -1017,7 +1020,7 @@ func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
 		make(chan string, 5)}
 
 	arv, _ := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
 	arv.ApiToken = "abc123"
@@ -1035,7 +1038,7 @@ func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
 
 	_, replicas, err := kc.PutB([]byte("foo"))
 
-	c.Check(err, Equals, InsufficientReplicasError)
+	c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
 	c.Check(replicas, Equals, 0)
 }
 
@@ -1069,12 +1072,14 @@ func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	c.Assert(err, IsNil)
+	kc, err := MakeKeepClient(arv)
+	c.Assert(err, IsNil)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
 	r, err := kc.GetIndex("x", "")
-	c.Check(err, Equals, nil)
+	c.Check(err, IsNil)
 
 	content, err2 := ioutil.ReadAll(r)
 	c.Check(err2, Equals, nil)
@@ -1095,12 +1100,12 @@ func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
 	r, err := kc.GetIndex("x", hash[0:3])
-	c.Check(err, Equals, nil)
+	c.Assert(err, Equals, nil)
 
 	content, err2 := ioutil.ReadAll(r)
 	c.Check(err2, Equals, nil)
@@ -1121,9 +1126,9 @@ func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
 	_, err = kc.GetIndex("x", hash[0:3])
 	c.Check(err, Equals, ErrIncompleteIndex)
@@ -1143,9 +1148,9 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
 	_, err = kc.GetIndex("y", hash[0:3])
 	c.Check(err, Equals, ErrNoSuchKeepServer)
@@ -1163,9 +1168,9 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
 	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
 	r, err := kc.GetIndex("x", "abcd")
 	c.Check(err, Equals, nil)
@@ -1174,3 +1179,99 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
 	c.Check(err2, Equals, nil)
 	c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
 }
+
+type FailThenSucceedPutHandler struct {
+	handled        chan string
+	count          int
+	successhandler StubPutHandler
+}
+
+func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	if h.count == 0 {
+		resp.WriteHeader(500)
+		h.count += 1
+		h.handled <- fmt.Sprintf("http://%s", req.Host)
+	} else {
+		h.successhandler.ServeHTTP(resp, req)
+	}
+}
+
+func (s *StandaloneSuite) TestPutBRetry(c *C) {
+	st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
+		StubPutHandler{
+			c,
+			Md5String("foo"),
+			"abc123",
+			"foo",
+			make(chan string, 5)}}
+
+	arv, _ := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+
+	kc.Want_replicas = 2
+	arv.ApiToken = "abc123"
+	localRoots := make(map[string]string)
+	writableLocalRoots := make(map[string]string)
+
+	ks := RunSomeFakeKeepServers(st, 2)
+
+	for i, k := range ks {
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		defer k.listener.Close()
+	}
+
+	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+
+	hash, replicas, err := kc.PutB([]byte("foo"))
+
+	c.Check(err, Equals, nil)
+	c.Check(hash, Equals, "")
+	c.Check(replicas, Equals, 2)
+}
+
+func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
+	arv, err := arvadosclient.MakeArvadosClient()
+	c.Assert(err, Equals, nil)
+
+	// Add an additional "testblobstore" keepservice
+	blobKeepService := make(arvadosclient.Dict)
+	err = arv.Create("keep_services",
+		arvadosclient.Dict{"keep_service": arvadosclient.Dict{
+			"service_host": "localhost",
+			"service_port": "21321",
+			"service_type": "testblobstore"}},
+		&blobKeepService)
+	c.Assert(err, Equals, nil)
+	defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
+	RefreshServiceDiscovery()
+
+	// Make a keepclient and ensure that the testblobstore is included
+	kc, err := MakeKeepClient(arv)
+	c.Assert(err, Equals, nil)
+
+	// verify kc.LocalRoots
+	c.Check(len(kc.LocalRoots()), Equals, 3)
+	for _, root := range kc.LocalRoots() {
+		c.Check(root, Matches, "http://localhost:\\d+")
+	}
+	c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
+
+	// verify kc.GatewayRoots
+	c.Check(len(kc.GatewayRoots()), Equals, 3)
+	for _, root := range kc.GatewayRoots() {
+		c.Check(root, Matches, "http://localhost:\\d+")
+	}
+	c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
+
+	// verify kc.WritableLocalRoots
+	c.Check(len(kc.WritableLocalRoots()), Equals, 3)
+	for _, root := range kc.WritableLocalRoots() {
+		c.Check(root, Matches, "http://localhost:\\d+")
+	}
+	c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
+
+	c.Assert(kc.replicasPerService, Equals, 0)
+	c.Assert(kc.foundNonDiskSvc, Equals, true)
+	c.Assert(kc.httpClient().(*http.Client).Timeout, Equals, 300*time.Second)
+}