X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/409fe707a4d05f1986d7bb0c29ec977d19aa63f5..c3d4f8a585202ec58df5506934b698039c200b68:/sdk/go/keepclient/keepclient_test.go

diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 04be03baa4..f0da600c24 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -1,11 +1,11 @@
 package keepclient
 
 import (
-	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-	"git.curoverse.com/arvados.git/sdk/go/streamer"
 	"crypto/md5"
-	"flag"
 	"fmt"
+	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+	"git.curoverse.com/arvados.git/sdk/go/streamer"
 	. "gopkg.in/check.v1"
 	"io"
 	"io/ioutil"
@@ -13,8 +13,9 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"os/exec"
+	"strings"
 	"testing"
+	"time"
 )
 
 // Gocheck boilerplate
@@ -26,8 +27,6 @@ func Test(t *testing.T) {
 var _ = Suite(&ServerRequiredSuite{})
 var _ = Suite(&StandaloneSuite{})
 
-var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
-
 // Tests that require the Keep server running
 type ServerRequiredSuite struct{}
 
@@ -40,68 +39,42 @@ func pythonDir() string {
 }
 
 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
-	if *no_server {
-		c.Skip("Skipping tests that require server")
-		return
-	}
-	os.Chdir(pythonDir())
-	{
-		cmd := exec.Command("python", "run_test_server.py", "start")
-		stderr, err := cmd.StderrPipe()
-		if err != nil {
-			log.Fatalf("Setting up stderr pipe: %s", err)
-		}
-		go io.Copy(os.Stderr, stderr)
-		if err := cmd.Run(); err != nil {
-			panic(fmt.Sprintf("'python run_test_server.py start' returned error %s", err))
-		}
-	}
-	{
-		cmd := exec.Command("python", "run_test_server.py", "start_keep")
-		stderr, err := cmd.StderrPipe()
-		if err != nil {
-			log.Fatalf("Setting up stderr pipe: %s", err)
-		}
-		go io.Copy(os.Stderr, stderr)
-		if err := cmd.Run(); err != nil {
-			panic(fmt.Sprintf("'python run_test_server.py start_keep' returned error %s", err))
-		}
-	}
+	arvadostest.StartAPI()
+	arvadostest.StartKeep(2, false)
 }
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
-	os.Chdir(pythonDir())
-	exec.Command("python", "run_test_server.py", "stop_keep").Run()
-	exec.Command("python", "run_test_server.py", "stop").Run()
+	arvadostest.StopKeep(2)
+	arvadostest.StopAPI()
 }
 
 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
-	os.Setenv("ARVADOS_API_HOST", "localhost:3000")
-	os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
-	os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
-
 	arv, err := arvadosclient.MakeArvadosClient()
 	c.Assert(err, Equals, nil)
 
-	kc, err := MakeKeepClient(&arv)
+	kc, err := MakeKeepClient(arv)
 
 	c.Assert(err, Equals, nil)
-	c.Check(len(kc.ServiceRoots()), Equals, 2)
-	c.Check(kc.ServiceRoots()[0], Equals, "http://localhost:25107")
-	c.Check(kc.ServiceRoots()[1], Equals, "http://localhost:25108")
+	c.Check(len(kc.LocalRoots()), Equals, 2)
+	for _, root := range kc.LocalRoots() {
+		c.Check(root, Matches, "http://localhost:\\d+")
+	}
 }
 
-func (s *StandaloneSuite) TestShuffleServiceRoots(c *C) {
-	kc := KeepClient{}
-	kc.SetServiceRoots([]string{"http://localhost:25107", "http://localhost:25108", "http://localhost:25109", "http://localhost:25110", "http://localhost:25111", "http://localhost:25112", "http://localhost:25113", "http://localhost:25114", "http://localhost:25115", "http://localhost:25116", "http://localhost:25117", "http://localhost:25118", "http://localhost:25119", "http://localhost:25120", "http://localhost:25121", "http://localhost:25122", "http://localhost:25123"})
+func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
+	arv, err := arvadosclient.MakeArvadosClient()
+	c.Assert(err, Equals, nil)
+
+	kc, err := MakeKeepClient(arv)
+	c.Assert(kc.Want_replicas, Equals, 2)
 
-	// "foo" acbd18db4cc2f85cedef654fccc4a4d8
-	foo_shuffle := []string{"http://localhost:25116", "http://localhost:25120", "http://localhost:25119", "http://localhost:25122", "http://localhost:25108", "http://localhost:25114", "http://localhost:25112", "http://localhost:25107", "http://localhost:25118", "http://localhost:25111", "http://localhost:25113", "http://localhost:25121", "http://localhost:25110", "http://localhost:25117", "http://localhost:25109", "http://localhost:25115", "http://localhost:25123"}
-	c.Check(kc.shuffledServiceRoots("acbd18db4cc2f85cedef654fccc4a4d8"), DeepEquals, foo_shuffle)
+	arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
+	kc, err = MakeKeepClient(arv)
+	c.Assert(kc.Want_replicas, Equals, 3)
 
-	// "bar" 37b51d194a7513e45b56f6524f2d51f2
-	bar_shuffle := []string{"http://localhost:25108", "http://localhost:25112", "http://localhost:25119", "http://localhost:25107", "http://localhost:25110", "http://localhost:25116", "http://localhost:25122", "http://localhost:25120", "http://localhost:25121", "http://localhost:25117", "http://localhost:25111", "http://localhost:25123", "http://localhost:25118", "http://localhost:25113", "http://localhost:25114", "http://localhost:25115", "http://localhost:25109"}
-	c.Check(kc.shuffledServiceRoots("37b51d194a7513e45b56f6524f2d51f2"), DeepEquals, bar_shuffle)
+	arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
+	kc, err = MakeKeepClient(arv)
+	c.Assert(kc.Want_replicas, Equals, 1)
 }
 
 type StubPutHandler struct {
@@ -112,44 +85,42 @@ type StubPutHandler struct {
 	handled        chan string
 }
 
-func (this StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
-	this.c.Check(req.URL.Path, Equals, "/"+this.expectPath)
-	this.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", this.expectApiToken))
+func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
+	sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
 	body, err := ioutil.ReadAll(req.Body)
-	this.c.Check(err, Equals, nil)
-	this.c.Check(body, DeepEquals, []byte(this.expectBody))
+	sph.c.Check(err, Equals, nil)
+	sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
 	resp.WriteHeader(200)
-	this.handled <- fmt.Sprintf("http://%s", req.Host)
+	sph.handled <- fmt.Sprintf("http://%s", req.Host)
 }
 
-func RunBogusKeepServer(st http.Handler, port int) (listener net.Listener, url string) {
+func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
 	var err error
-	listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: port})
+	ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
 	if err != nil {
-		panic(fmt.Sprintf("Could not listen on tcp port %v", port))
+		panic(fmt.Sprintf("Could not listen on any port"))
 	}
-
-	url = fmt.Sprintf("http://localhost:%d", port)
-
-	go http.Serve(listener, st)
-	return listener, url
+	ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
+	go http.Serve(ks.listener, st)
+	return
 }
 
-func UploadToStubHelper(c *C, st http.Handler, f func(KeepClient, string,
+func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
 	io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
 
-	listener, url := RunBogusKeepServer(st, 2990)
-	defer listener.Close()
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
 
 	arv, _ := arvadosclient.MakeArvadosClient()
 	arv.ApiToken = "abc123"
 
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	reader, writer := io.Pipe()
 	upload_status := make(chan uploadStatus)
 
-	f(kc, url, reader, writer, upload_status)
+	f(kc, ks.url, reader, writer, upload_status)
 }
 
 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
@@ -163,10 +134,9 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 		make(chan string)}
 
 	UploadToStubHelper(c, st,
-		func(kc KeepClient, url string, reader io.ReadCloser,
-			writer io.WriteCloser, upload_status chan uploadStatus) {
+		func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
 
-			go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")))
+			go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
 
 			writer.Write([]byte("foo"))
 			writer.Close()
@@ -175,13 +145,9 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 			status := <-upload_status
 			c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 		})
-
-	log.Printf("TestUploadToStubKeepServer done")
 }
 
 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
-	log.Printf("TestUploadToStubKeepServerBufferReader")
-
 	st := StubPutHandler{
 		c,
 		"acbd18db4cc2f85cedef654fccc4a4d8",
@@ -190,7 +156,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 		make(chan string)}
 
 	UploadToStubHelper(c, st,
-		func(kc KeepClient, url string, reader io.ReadCloser,
+		func(kc *KeepClient, url string, reader io.ReadCloser,
 			writer io.WriteCloser, upload_status chan uploadStatus) {
 
 			tr := streamer.AsyncStreamFromReader(512, reader)
@@ -198,7 +164,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 
 			br1 := tr.MakeStreamReader()
 
-			go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3)
+			go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0)
 
 			writer.Write([]byte("foo"))
 			writer.Close()
@@ -208,32 +174,53 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 			status := <-upload_status
 			c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 		})
-
-	log.Printf("TestUploadToStubKeepServerBufferReader done")
 }
 
 type FailHandler struct {
 	handled chan string
 }
 
-func (this FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 	resp.WriteHeader(500)
-	this.handled <- fmt.Sprintf("http://%s", req.Host)
+	fh.handled <- fmt.Sprintf("http://%s", req.Host)
 }
 
-func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
-	log.Printf("TestFailedUploadToStubKeepServer")
+type FailThenSucceedHandler struct {
+	handled        chan string
+	count          int
+	successhandler StubGetHandler
+}
+
+func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	if fh.count == 0 {
+		resp.WriteHeader(500)
+		fh.count += 1
+		fh.handled <- fmt.Sprintf("http://%s", req.Host)
+	} else {
+		fh.successhandler.ServeHTTP(resp, req)
+	}
+}
 
+type Error404Handler struct {
+	handled chan string
+}
+
+func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	resp.WriteHeader(404)
+	fh.handled <- fmt.Sprintf("http://%s", req.Host)
+}
+
+func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 	st := FailHandler{
 		make(chan string)}
 
 	hash := "acbd18db4cc2f85cedef654fccc4a4d8"
 
 	UploadToStubHelper(c, st,
-		func(kc KeepClient, url string, reader io.ReadCloser,
+		func(kc *KeepClient, url string, reader io.ReadCloser,
 			writer io.WriteCloser, upload_status chan uploadStatus) {
 
-			go kc.uploadToKeepServer(url, hash, reader, upload_status, 3)
+			go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
 
 			writer.Write([]byte("foo"))
 			writer.Close()
@@ -244,7 +231,6 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 			c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
 			c.Check(status.statusCode, Equals, 500)
 		})
-	log.Printf("TestFailedUploadToStubKeepServer done")
 }
 
 type KeepServer struct {
@@ -252,48 +238,48 @@ type KeepServer struct {
 	url      string
 }
 
-func RunSomeFakeKeepServers(st http.Handler, n int, port int) (ks []KeepServer) {
+func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
 	ks = make([]KeepServer, n)
 
 	for i := 0; i < n; i += 1 {
-		boguslistener, bogusurl := RunBogusKeepServer(st, port+i)
-		ks[i] = KeepServer{boguslistener, bogusurl}
+		ks[i] = RunFakeKeepServer(st)
 	}
 
 	return ks
 }
 
 func (s *StandaloneSuite) TestPutB(c *C) {
-	log.Printf("TestPutB")
-
-	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+	hash := Md5String("foo")
 
 	st := StubPutHandler{
 		c,
 		hash,
 		"abc123",
 		"foo",
-		make(chan string, 2)}
+		make(chan string, 5)}
 
 	arv, _ := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
 	arv.ApiToken = "abc123"
-	service_roots := make([]string, 5)
+	localRoots := make(map[string]string)
+	writableLocalRoots := make(map[string]string)
 
-	ks := RunSomeFakeKeepServers(st, 5, 2990)
+	ks := RunSomeFakeKeepServers(st, 5)
 
-	for i := 0; i < len(ks); i += 1 {
-		service_roots[i] = ks[i].url
-		defer ks[i].listener.Close()
+	for i, k := range ks {
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		defer k.listener.Close()
 	}
 
-	kc.SetServiceRoots(service_roots)
+	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
 	kc.PutB([]byte("foo"))
 
-	shuff := kc.shuffledServiceRoots(fmt.Sprintf("%x", md5.Sum([]byte("foo"))))
+	shuff := NewRootSorter(
+		kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 
 	s1 := <-st.handled
 	s2 := <-st.handled
@@ -301,13 +287,9 @@ func (s *StandaloneSuite) TestPutB(c *C) {
 		(s1 == shuff[1] && s2 == shuff[0]),
 		Equals,
 		true)
-
-	log.Printf("TestPutB done")
 }
 
 func (s *StandaloneSuite) TestPutHR(c *C) {
-	log.Printf("TestPutHR")
-
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
 	st := StubPutHandler{
@@ -315,23 +297,25 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
 		hash,
 		"abc123",
 		"foo",
-		make(chan string, 2)}
+		make(chan string, 5)}
 
 	arv, _ := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
 	arv.ApiToken = "abc123"
-	service_roots := make([]string, 5)
+	localRoots := make(map[string]string)
+	writableLocalRoots := make(map[string]string)
 
-	ks := RunSomeFakeKeepServers(st, 5, 2990)
+	ks := RunSomeFakeKeepServers(st, 5)
 
-	for i := 0; i < len(ks); i += 1 {
-		service_roots[i] = ks[i].url
-		defer ks[i].listener.Close()
+	for i, k := range ks {
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		defer k.listener.Close()
 	}
 
-	kc.SetServiceRoots(service_roots)
+	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
 	reader, writer := io.Pipe()
 
@@ -342,8 +326,7 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
 
 	kc.PutHR(hash, reader, 3)
 
-	shuff := kc.shuffledServiceRoots(hash)
-	log.Print(shuff)
+	shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
 
 	s1 := <-st.handled
 	s2 := <-st.handled
@@ -352,13 +335,9 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
 		(s1 == shuff[1] && s2 == shuff[0]),
 		Equals,
 		true)
-
-	log.Printf("TestPutHR done")
 }
 
 func (s *StandaloneSuite) TestPutWithFail(c *C) {
-	log.Printf("TestPutWithFail")
-
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
 	st := StubPutHandler{
@@ -366,33 +345,38 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
 		hash,
 		"abc123",
 		"foo",
-		make(chan string, 2)}
+		make(chan string, 4)}
 
 	fh := FailHandler{
 		make(chan string, 1)}
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
 	arv.ApiToken = "abc123"
-	service_roots := make([]string, 5)
+	localRoots := make(map[string]string)
+	writableLocalRoots := make(map[string]string)
 
-	ks1 := RunSomeFakeKeepServers(st, 4, 2990)
-	ks2 := RunSomeFakeKeepServers(fh, 1, 2995)
+	ks1 := RunSomeFakeKeepServers(st, 4)
+	ks2 := RunSomeFakeKeepServers(fh, 1)
 
 	for i, k := range ks1 {
-		service_roots[i] = k.url
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 		defer k.listener.Close()
 	}
 	for i, k := range ks2 {
-		service_roots[len(ks1)+i] = k.url
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 		defer k.listener.Close()
 	}
 
-	kc.SetServiceRoots(service_roots)
+	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
-	shuff := kc.shuffledServiceRoots(fmt.Sprintf("%x", md5.Sum([]byte("foo"))))
+	shuff := NewRootSorter(
+		kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
+	c.Logf("%+v", shuff)
 
 	phash, replicas, err := kc.PutB([]byte("foo"))
 
@@ -401,13 +385,17 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
 	c.Check(err, Equals, nil)
 	c.Check(phash, Equals, "")
 	c.Check(replicas, Equals, 2)
-	c.Check(<-st.handled, Equals, shuff[1])
-	c.Check(<-st.handled, Equals, shuff[2])
+
+	s1 := <-st.handled
+	s2 := <-st.handled
+
+	c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
+		(s1 == shuff[2] && s2 == shuff[1]),
+		Equals,
+		true)
 }
 
 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
-	log.Printf("TestPutWithTooManyFail")
-
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
 	st := StubPutHandler{
@@ -421,81 +409,121 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 		make(chan string, 4)}
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
+	kc.Retries = 0
 	arv.ApiToken = "abc123"
-	service_roots := make([]string, 5)
+	localRoots := make(map[string]string)
+	writableLocalRoots := make(map[string]string)
 
-	ks1 := RunSomeFakeKeepServers(st, 1, 2990)
-	ks2 := RunSomeFakeKeepServers(fh, 4, 2991)
+	ks1 := RunSomeFakeKeepServers(st, 1)
+	ks2 := RunSomeFakeKeepServers(fh, 4)
 
 	for i, k := range ks1 {
-		service_roots[i] = k.url
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 		defer k.listener.Close()
 	}
 	for i, k := range ks2 {
-		service_roots[len(ks1)+i] = k.url
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 		defer k.listener.Close()
 	}
 
-	kc.SetServiceRoots(service_roots)
-
-	shuff := kc.shuffledServiceRoots(fmt.Sprintf("%x", md5.Sum([]byte("foo"))))
+	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
 	_, replicas, err := kc.PutB([]byte("foo"))
 
 	c.Check(err, Equals, InsufficientReplicasError)
 	c.Check(replicas, Equals, 1)
-	c.Check(<-st.handled, Equals, shuff[1])
-
-	log.Printf("TestPutWithTooManyFail done")
+	c.Check(<-st.handled, Equals, ks1[0].url)
 }
 
 type StubGetHandler struct {
 	c              *C
 	expectPath     string
 	expectApiToken string
-	returnBody     []byte
+	httpStatus     int
+	body           []byte
 }
 
-func (this StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
-	this.c.Check(req.URL.Path, Equals, "/"+this.expectPath)
-	this.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", this.expectApiToken))
-	resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(this.returnBody)))
-	resp.Write(this.returnBody)
+func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
+	sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
+	resp.WriteHeader(sgh.httpStatus)
+	resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
+	resp.Write(sgh.body)
 }
 
 func (s *StandaloneSuite) TestGet(c *C) {
-	log.Printf("TestGet")
-
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
 	st := StubGetHandler{
 		c,
 		hash,
 		"abc123",
+		http.StatusOK,
 		[]byte("foo")}
 
-	listener, url := RunBogusKeepServer(st, 2990)
-	defer listener.Close()
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots([]string{url})
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
 	r, n, url2, err := kc.Get(hash)
 	defer r.Close()
 	c.Check(err, Equals, nil)
 	c.Check(n, Equals, int64(3))
-	c.Check(url2, Equals, fmt.Sprintf("%s/%s", url, hash))
+	c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 
 	content, err2 := ioutil.ReadAll(r)
 	c.Check(err2, Equals, nil)
 	c.Check(content, DeepEquals, []byte("foo"))
+}
+
+func (s *StandaloneSuite) TestGet404(c *C) {
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	st := Error404Handler{make(chan string, 1)}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
-	log.Printf("TestGet done")
+	r, n, url2, err := kc.Get(hash)
+	c.Check(err, Equals, BlockNotFound)
+	c.Check(n, Equals, int64(0))
+	c.Check(url2, Equals, "")
+	c.Check(r, Equals, nil)
+}
+
+func (s *StandaloneSuite) TestGetEmptyBlock(c *C) {
+	st := Error404Handler{make(chan string, 1)}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+	r, n, url2, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e+0")
+	c.Check(err, IsNil)
+	c.Check(n, Equals, int64(0))
+	c.Check(url2, Equals, "")
+	c.Assert(r, NotNil)
+	buf, err := ioutil.ReadAll(r)
+	c.Check(err, IsNil)
+	c.Check(buf, DeepEquals, []byte{})
 }
 
 func (s *StandaloneSuite) TestGetFail(c *C) {
@@ -503,21 +531,203 @@ func (s *StandaloneSuite) TestGetFail(c *C) {
 
 	st := FailHandler{make(chan string, 1)}
 
-	listener, url := RunBogusKeepServer(st, 2990)
-	defer listener.Close()
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots([]string{url})
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+	kc.Retries = 0
 
 	r, n, url2, err := kc.Get(hash)
-	c.Check(err, Equals, BlockNotFound)
+	errNotFound, _ := err.(*ErrNotFound)
+	c.Check(errNotFound, NotNil)
+	c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
+	c.Check(errNotFound.Temporary(), Equals, true)
+	c.Check(n, Equals, int64(0))
+	c.Check(url2, Equals, "")
+	c.Check(r, Equals, nil)
+}
+
+func (s *StandaloneSuite) TestGetFailRetry(c *C) {
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	st := &FailThenSucceedHandler{make(chan string, 1), 0,
+		StubGetHandler{
+			c,
+			hash,
+			"abc123",
+			http.StatusOK,
+			[]byte("foo")}}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+	r, n, url2, err := kc.Get(hash)
+	defer r.Close()
+	c.Check(err, Equals, nil)
+	c.Check(n, Equals, int64(3))
+	c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
+
+	content, err2 := ioutil.ReadAll(r)
+	c.Check(err2, Equals, nil)
+	c.Check(content, DeepEquals, []byte("foo"))
+}
+
+func (s *StandaloneSuite) TestGetNetError(c *C) {
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
+
+	r, n, url2, err := kc.Get(hash)
+	errNotFound, _ := err.(*ErrNotFound)
+	c.Check(errNotFound, NotNil)
+	c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
+	c.Check(errNotFound.Temporary(), Equals, true)
 	c.Check(n, Equals, int64(0))
 	c.Check(url2, Equals, "")
 	c.Check(r, Equals, nil)
 }
 
+func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
+	uuid := "zzzzz-bi6l4-123451234512345"
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	// This one shouldn't be used:
+	ks0 := RunFakeKeepServer(StubGetHandler{
+		c,
+		"error if used",
+		"abc123",
+		http.StatusOK,
+		[]byte("foo")})
+	defer ks0.listener.Close()
+	// This one should be used:
+	ks := RunFakeKeepServer(StubGetHandler{
+		c,
+		hash + "+K@" + uuid,
+		"abc123",
+		http.StatusOK,
+		[]byte("foo")})
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(
+		map[string]string{"x": ks0.url},
+		nil,
+		map[string]string{uuid: ks.url})
+
+	r, n, uri, err := kc.Get(hash + "+K@" + uuid)
+	defer r.Close()
+	c.Check(err, Equals, nil)
+	c.Check(n, Equals, int64(3))
+	c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
+
+	content, err := ioutil.ReadAll(r)
+	c.Check(err, Equals, nil)
+	c.Check(content, DeepEquals, []byte("foo"))
+}
+
+// Use a service hint to fetch from a local disk service, overriding
+// rendezvous probe order.
+func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
+	uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	// This one shouldn't be used, although it appears first in
+	// rendezvous probe order:
+	ks0 := RunFakeKeepServer(StubGetHandler{
+		c,
+		"error if used",
+		"abc123",
+		http.StatusOK,
+		[]byte("foo")})
+	defer ks0.listener.Close()
+	// This one should be used:
+	ks := RunFakeKeepServer(StubGetHandler{
+		c,
+		hash + "+K@" + uuid,
+		"abc123",
+		http.StatusOK,
+		[]byte("foo")})
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(
+		map[string]string{
+			"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
+			"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
+			"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
+			uuid: ks.url},
+		nil,
+		map[string]string{
+			"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
+			"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
+			"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
+			uuid: ks.url},
+	)
+
+	r, n, uri, err := kc.Get(hash + "+K@" + uuid)
+	defer r.Close()
+	c.Check(err, Equals, nil)
+	c.Check(n, Equals, int64(3))
+	c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
+
+	content, err := ioutil.ReadAll(r)
+	c.Check(err, Equals, nil)
+	c.Check(content, DeepEquals, []byte("foo"))
+}
+
+func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
+	uuid := "zzzzz-bi6l4-123451234512345"
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	ksLocal := RunFakeKeepServer(StubGetHandler{
+		c,
+		hash + "+K@" + uuid,
+		"abc123",
+		http.StatusOK,
+		[]byte("foo")})
+	defer ksLocal.listener.Close()
+	ksGateway := RunFakeKeepServer(StubGetHandler{
+		c,
+		hash + "+K@" + uuid,
+		"abc123",
+		http.StatusInternalServerError,
+		[]byte("Error")})
+	defer ksGateway.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(
+		map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
+		nil,
+		map[string]string{uuid: ksGateway.url})
+
+	r, n, uri, err := kc.Get(hash + "+K@" + uuid)
+	c.Assert(err, Equals, nil)
+	defer r.Close()
+	c.Check(n, Equals, int64(3))
+	c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
+
+	content, err := ioutil.ReadAll(r)
+	c.Check(err, Equals, nil)
+	c.Check(content, DeepEquals, []byte("foo"))
+}
+
 type BarHandler struct {
 	handled chan string
 }
@@ -533,13 +743,13 @@ func (s *StandaloneSuite) TestChecksum(c *C) {
 
 	st := BarHandler{make(chan string, 1)}
 
-	listener, url := RunBogusKeepServer(st, 2990)
-	defer listener.Close()
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots([]string{url})
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
 	r, n, _, err := kc.Get(barhash)
 	_, err = ioutil.ReadAll(r)
@@ -557,58 +767,70 @@ func (s *StandaloneSuite) TestChecksum(c *C) {
 }
 
 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
+	content := []byte("waz")
+	hash := fmt.Sprintf("%x", md5.Sum(content))
 
-	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
-
-	fh := FailHandler{
-		make(chan string, 1)}
+	fh := Error404Handler{
+		make(chan string, 4)}
 
 	st := StubGetHandler{
 		c,
 		hash,
 		"abc123",
-		[]byte("foo")}
+		http.StatusOK,
+		content}
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 	arv.ApiToken = "abc123"
-	service_roots := make([]string, 5)
+	localRoots := make(map[string]string)
+	writableLocalRoots := make(map[string]string)
 
-	ks1 := RunSomeFakeKeepServers(st, 1, 2990)
-	ks2 := RunSomeFakeKeepServers(fh, 4, 2991)
+	ks1 := RunSomeFakeKeepServers(st, 1)
+	ks2 := RunSomeFakeKeepServers(fh, 4)
 
 	for i, k := range ks1 {
-		service_roots[i] = k.url
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 		defer k.listener.Close()
 	}
 	for i, k := range ks2 {
-		service_roots[len(ks1)+i] = k.url
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 		defer k.listener.Close()
 	}
 
-	kc.SetServiceRoots(service_roots)
+	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+	kc.Retries = 0
+
+	// This test works only if one of the failing services is
+	// attempted before the succeeding service. Otherwise,
+	// <-fh.handled below will just hang! (Probe order depends on
+	// the choice of block content "waz" and the UUIDs of the fake
+	// servers, so we just tried different strings until we found
+	// an example that passes this Assert.)
+	c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
 
 	r, n, url2, err := kc.Get(hash)
+
 	<-fh.handled
 	c.Check(err, Equals, nil)
 	c.Check(n, Equals, int64(3))
 	c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
 
-	content, err2 := ioutil.ReadAll(r)
+	read_content, err2 := ioutil.ReadAll(r)
 	c.Check(err2, Equals, nil)
-	c.Check(content, DeepEquals, []byte("foo"))
+	c.Check(read_content, DeepEquals, content)
 }
 
 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
-	os.Setenv("ARVADOS_API_HOST", "localhost:3000")
-	os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
-	os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
+	content := []byte("TestPutGetHead")
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, err := MakeKeepClient(&arv)
+	kc, err := MakeKeepClient(arv)
 	c.Assert(err, Equals, nil)
 
-	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+	hash := fmt.Sprintf("%x", md5.Sum(content))
 
 	{
 		n, _, err := kc.Ask(hash)
@@ -616,26 +838,26 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
 		c.Check(n, Equals, int64(0))
 	}
 	{
-		hash2, replicas, err := kc.PutB([]byte("foo"))
-		c.Check(hash2, Equals, fmt.Sprintf("%s+%v", hash, 3))
+		hash2, replicas, err := kc.PutB(content)
+		c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content)))
 		c.Check(replicas, Equals, 2)
 		c.Check(err, Equals, nil)
 	}
 	{
 		r, n, url2, err := kc.Get(hash)
 		c.Check(err, Equals, nil)
-		c.Check(n, Equals, int64(3))
-		c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash))
+		c.Check(n, Equals, int64(len(content)))
+		c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 
-		content, err2 := ioutil.ReadAll(r)
+		read_content, err2 := ioutil.ReadAll(r)
 		c.Check(err2, Equals, nil)
-		c.Check(content, DeepEquals, []byte("foo"))
+		c.Check(read_content, DeepEquals, content)
 	}
 	{
 		n, url2, err := kc.Ask(hash)
 		c.Check(err, Equals, nil)
-		c.Check(n, Equals, int64(3))
-		c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash))
+		c.Check(n, Equals, int64(len(content)))
+		c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 	}
 }
 
@@ -649,71 +871,397 @@ func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reque
 }
 
 func (s *StandaloneSuite) TestPutProxy(c *C) {
-	log.Printf("TestPutProxy")
-
 	st := StubProxyHandler{make(chan string, 1)}
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 2
-	kc.Using_proxy = true
 	arv.ApiToken = "abc123"
-	service_roots := make([]string, 1)
+	localRoots := make(map[string]string)
+	writableLocalRoots := make(map[string]string)
 
-	ks1 := RunSomeFakeKeepServers(st, 1, 2990)
+	ks1 := RunSomeFakeKeepServers(st, 1)
 
 	for i, k := range ks1 {
-		service_roots[i] = k.url
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 		defer k.listener.Close()
 	}
 
-	kc.SetServiceRoots(service_roots)
+	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
 	_, replicas, err := kc.PutB([]byte("foo"))
 	<-st.handled
 
 	c.Check(err, Equals, nil)
 	c.Check(replicas, Equals, 2)
-
-	log.Printf("TestPutProxy done")
 }
 
 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
-	log.Printf("TestPutProxy")
-
 	st := StubProxyHandler{make(chan string, 1)}
 
 	arv, err := arvadosclient.MakeArvadosClient()
-	kc, _ := MakeKeepClient(&arv)
+	kc, _ := MakeKeepClient(arv)
 
 	kc.Want_replicas = 3
-	kc.Using_proxy = true
 	arv.ApiToken = "abc123"
-	service_roots := make([]string, 1)
+	localRoots := make(map[string]string)
+	writableLocalRoots := make(map[string]string)
 
-	ks1 := RunSomeFakeKeepServers(st, 1, 2990)
+	ks1 := RunSomeFakeKeepServers(st, 1)
 
 	for i, k := range ks1 {
-		service_roots[i] = k.url
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 		defer k.listener.Close()
 	}
-	kc.SetServiceRoots(service_roots)
+	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 
 	_, replicas, err := kc.PutB([]byte("foo"))
 	<-st.handled
 
 	c.Check(err, Equals, InsufficientReplicasError)
 	c.Check(replicas, Equals, 2)
-
-	log.Printf("TestPutProxy done")
 }
 
 func (s *StandaloneSuite) TestMakeLocator(c *C) {
-	l := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
+	l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
+	c.Check(err, Equals, nil)
+	c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
+	c.Check(l.Size, Equals, 3)
+	c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
+}
+
+func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
+	l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
+	c.Check(err, Equals, nil)
+	c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
+	c.Check(l.Size, Equals, -1)
+	c.Check(l.Hints, DeepEquals, []string{})
+}
 
+func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
+	l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
+	c.Check(err, Equals, nil)
+	c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
+	c.Check(l.Size, Equals, -1)
+	c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
+}
+
+func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
+	str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
+	l, err := MakeLocator(str)
+	c.Check(err, Equals, nil)
 	c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 	c.Check(l.Size, Equals, 3)
-	c.Check(l.Signature, Equals, "abcde")
-	c.Check(l.Timestamp, Equals, "12345678")
+	c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
+	c.Check(l.String(), Equals, str)
+}
+
+func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
+	_, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
+	c.Check(err, Equals, InvalidLocatorError)
+}
+
+func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
+	hash := Md5String("foo")
+
+	st := StubPutHandler{
+		c,
+		hash,
+		"abc123",
+		"foo",
+		make(chan string, 5)}
+
+	arv, _ := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+
+	kc.Want_replicas = 2
+	arv.ApiToken = "abc123"
+	localRoots := make(map[string]string)
+	writableLocalRoots := make(map[string]string)
+
+	ks := RunSomeFakeKeepServers(st, 5)
+
+	for i, k := range ks {
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		if i == 0 {
+			writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		}
+		defer k.listener.Close()
+	}
+
+	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+
+	_, replicas, err := kc.PutB([]byte("foo"))
+
+	c.Check(err, Equals, InsufficientReplicasError)
+	c.Check(replicas, Equals, 1)
+
+	c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
+}
+
+func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
+	hash := Md5String("foo")
+
+	st := StubPutHandler{
+		c,
+		hash,
+		"abc123",
+		"foo",
+		make(chan string, 5)}
+
+	arv, _ := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+
+	kc.Want_replicas = 2
+	arv.ApiToken = "abc123"
+	localRoots := make(map[string]string)
+	writableLocalRoots := make(map[string]string)
+
+	ks := RunSomeFakeKeepServers(st, 5)
+
+	for i, k := range ks {
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		defer k.listener.Close()
+	}
+
+	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+
+	_, replicas, err := kc.PutB([]byte("foo"))
+
+	c.Check(err, Equals, InsufficientReplicasError)
+	c.Check(replicas, Equals, 0)
+}
+
+type StubGetIndexHandler struct {
+	c              *C
+	expectPath     string
+	expectAPIToken string
+	httpStatus     int
+	body           []byte
+}
+
+func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	h.c.Check(req.URL.Path, Equals, h.expectPath)
+	h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
+	resp.WriteHeader(h.httpStatus)
+	resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
+	resp.Write(h.body)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	st := StubGetIndexHandler{
+		c,
+		"/index",
+		"abc123",
+		http.StatusOK,
+		[]byte(hash + "+3 1443559274\n\n")}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+	r, err := kc.GetIndex("x", "")
+	c.Check(err, Equals, nil)
+
+	content, err2 := ioutil.ReadAll(r)
+	c.Check(err2, Equals, nil)
+	c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
+}
+
+func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	st := StubGetIndexHandler{
+		c,
+		"/index/" + hash[0:3],
+		"abc123",
+		http.StatusOK,
+		[]byte(hash + "+3 1443559274\n\n")}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+	r, err := kc.GetIndex("x", hash[0:3])
+	c.Check(err, Equals, nil)
+
+	content, err2 := ioutil.ReadAll(r)
+	c.Check(err2, Equals, nil)
+	c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
+}
+
+func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	st := StubGetIndexHandler{
+		c,
+		"/index/" + hash[0:3],
+		"abc123",
+		http.StatusOK,
+		[]byte(hash)}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+	_, err = kc.GetIndex("x", hash[0:3])
+	c.Check(err, Equals, ErrIncompleteIndex)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	st := StubGetIndexHandler{
+		c,
+		"/index/" + hash[0:3],
+		"abc123",
+		http.StatusOK,
+		[]byte(hash)}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+	_, err = kc.GetIndex("y", hash[0:3])
+	c.Check(err, Equals, ErrNoSuchKeepServer)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
+	st := StubGetIndexHandler{
+		c,
+		"/index/abcd",
+		"abc123",
+		http.StatusOK,
+		[]byte("\n")}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+	r, err := kc.GetIndex("x", "abcd")
+	c.Check(err, Equals, nil)
+
+	content, err2 := ioutil.ReadAll(r)
+	c.Check(err2, Equals, nil)
+	c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
+}
+
+type FailThenSucceedPutHandler struct {
+	handled        chan string
+	count          int
+	successhandler StubPutHandler
+}
+
+func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	if h.count == 0 {
+		resp.WriteHeader(500)
+		h.count += 1
+		h.handled <- fmt.Sprintf("http://%s", req.Host)
+	} else {
+		h.successhandler.ServeHTTP(resp, req)
+	}
+}
+
+func (s *StandaloneSuite) TestPutBRetry(c *C) {
+	st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
+		StubPutHandler{
+			c,
+			Md5String("foo"),
+			"abc123",
+			"foo",
+			make(chan string, 5)}}
+
+	arv, _ := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(arv)
+
+	kc.Want_replicas = 2
+	arv.ApiToken = "abc123"
+	localRoots := make(map[string]string)
+	writableLocalRoots := make(map[string]string)
+
+	ks := RunSomeFakeKeepServers(st, 2)
+
+	for i, k := range ks {
+		localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		defer k.listener.Close()
+	}
+
+	kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+
+	hash, replicas, err := kc.PutB([]byte("foo"))
+
+	c.Check(err, Equals, nil)
+	c.Check(hash, Equals, "")
+	c.Check(replicas, Equals, 2)
+}
+
+func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
+	arv, err := arvadosclient.MakeArvadosClient()
+	c.Assert(err, Equals, nil)
+
+	// Add an additional "testblobstore" keepservice
+	blobKeepService := make(arvadosclient.Dict)
+	err = arv.Create("keep_services",
+		arvadosclient.Dict{"keep_service": arvadosclient.Dict{
+			"service_host": "localhost",
+			"service_port": "21321",
+			"service_type": "testblobstore"}},
+		&blobKeepService)
+	c.Assert(err, Equals, nil)
+	defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
+
+	// Make a keepclient and ensure that the testblobstore is included
+	kc, err := MakeKeepClient(arv)
+	c.Assert(err, Equals, nil)
+
+	// verify kc.LocalRoots
+	c.Check(len(kc.LocalRoots()), Equals, 3)
+	for _, root := range kc.LocalRoots() {
+		c.Check(root, Matches, "http://localhost:\\d+")
+	}
+	c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
+
+	// verify kc.GatewayRoots
+	c.Check(len(kc.GatewayRoots()), Equals, 3)
+	for _, root := range kc.GatewayRoots() {
+		c.Check(root, Matches, "http://localhost:\\d+")
+	}
+	c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
+
+	// verify kc.WritableLocalRoots
+	c.Check(len(kc.WritableLocalRoots()), Equals, 3)
+	for _, root := range kc.WritableLocalRoots() {
+		c.Check(root, Matches, "http://localhost:\\d+")
+	}
+	c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
+
+	c.Assert(kc.replicasPerService, Equals, 0)
+	c.Assert(kc.foundNonDiskSvc, Equals, true)
+	c.Assert(kc.Client.Timeout, Equals, 300*time.Second)
 }