Merge branch 'master' into 13822-nm-delayed-daemon

[arvados.git] / sdk / go / keepclient / keepclient_test.go

diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 4ba1d7c245c272ae5a017e513eca4d8adcd292ee..dc80ad7e1d6378ad09da968db62cf038002d0b9c 100644 (file)
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -1,13 +1,14 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
 package keepclient
 
 import (
+       "bytes"
        "crypto/md5"
-       "flag"
+       "errors"
        "fmt"
-       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-       "git.curoverse.com/arvados.git/sdk/go/arvadostest"
-       "git.curoverse.com/arvados.git/sdk/go/streamer"
-       . "gopkg.in/check.v1"
        "io"
        "io/ioutil"
        "log"
@@ -17,6 +18,10 @@ import (
        "strings"
        "testing"
        "time"
+
+       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+       "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+       . "gopkg.in/check.v1"
 )
 
 // Gocheck boilerplate
@@ -28,41 +33,40 @@ func Test(t *testing.T) {
 var _ = Suite(&ServerRequiredSuite{})
 var _ = Suite(&StandaloneSuite{})
 
-var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
-
 // Tests that require the Keep server running
 type ServerRequiredSuite struct{}
 
 // Standalone tests
 type StandaloneSuite struct{}
 
+func (s *StandaloneSuite) SetUpTest(c *C) {
+       RefreshServiceDiscovery()
+}
+
 func pythonDir() string {
        cwd, _ := os.Getwd()
        return fmt.Sprintf("%s/../../python/tests", cwd)
 }
 
 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
-       if *no_server {
-               c.Skip("Skipping tests that require server")
-               return
-       }
        arvadostest.StartAPI()
        arvadostest.StartKeep(2, false)
 }
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
-       if *no_server {
-               return
-       }
        arvadostest.StopKeep(2)
        arvadostest.StopAPI()
 }
 
+func (s *ServerRequiredSuite) SetUpTest(c *C) {
+       RefreshServiceDiscovery()
+}
+
 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
        arv, err := arvadosclient.MakeArvadosClient()
        c.Assert(err, Equals, nil)
 
-       kc, err := MakeKeepClient(&arv)
+       kc, err := MakeKeepClient(arv)
 
        c.Assert(err, Equals, nil)
        c.Check(len(kc.LocalRoots()), Equals, 2)
@@ -75,29 +79,32 @@ func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
        arv, err := arvadosclient.MakeArvadosClient()
        c.Assert(err, Equals, nil)
 
-       kc, err := MakeKeepClient(&arv)
+       kc, err := MakeKeepClient(arv)
        c.Assert(kc.Want_replicas, Equals, 2)
 
        arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
-       kc, err = MakeKeepClient(&arv)
+       kc, err = MakeKeepClient(arv)
        c.Assert(kc.Want_replicas, Equals, 3)
 
        arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
-       kc, err = MakeKeepClient(&arv)
+       kc, err = MakeKeepClient(arv)
+       c.Check(err, IsNil)
        c.Assert(kc.Want_replicas, Equals, 1)
 }
 
 type StubPutHandler struct {
-       c              *C
-       expectPath     string
-       expectApiToken string
-       expectBody     string
-       handled        chan string
+       c                  *C
+       expectPath         string
+       expectApiToken     string
+       expectBody         string
+       expectStorageClass string
+       handled            chan string
 }
 
 func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
        sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
        sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
+       sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
        body, err := ioutil.ReadAll(req.Body)
        sph.c.Check(err, Equals, nil)
        sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
@@ -107,7 +114,9 @@ func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
 
 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
        var err error
-       ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
+       // If we don't explicitly bind it to localhost, ks.listener.Addr() will
+       // bind to 0.0.0.0 or [::] which is not a valid address for Dial()
+       ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: 0})
        if err != nil {
                panic(fmt.Sprintf("Could not listen on any port"))
        }
@@ -125,7 +134,7 @@ func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
        arv, _ := arvadosclient.MakeArvadosClient()
        arv.ApiToken = "abc123"
 
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
 
        reader, writer := io.Pipe()
        upload_status := make(chan uploadStatus)
@@ -141,12 +150,13 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
                "acbd18db4cc2f85cedef654fccc4a4d8",
                "abc123",
                "foo",
+               "hot",
                make(chan string)}
 
        UploadToStubHelper(c, st,
                func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
-
-                       go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
+                       kc.StorageClasses = []string{"hot"}
+                       go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), kc.getRequestID())
 
                        writer.Write([]byte("foo"))
                        writer.Close()
@@ -163,21 +173,12 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
                "acbd18db4cc2f85cedef654fccc4a4d8",
                "abc123",
                "foo",
+               "",
                make(chan string)}
 
        UploadToStubHelper(c, st,
-               func(kc *KeepClient, url string, reader io.ReadCloser,
-                       writer io.WriteCloser, upload_status chan uploadStatus) {
-
-                       tr := streamer.AsyncStreamFromReader(512, reader)
-                       defer tr.Close()
-
-                       br1 := tr.MakeStreamReader()
-
-                       go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0)
-
-                       writer.Write([]byte("foo"))
-                       writer.Close()
+               func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) {
+                       go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, kc.getRequestID())
 
                        <-st.handled
 
@@ -198,10 +199,12 @@ func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 type FailThenSucceedHandler struct {
        handled        chan string
        count          int
-       successhandler StubGetHandler
+       successhandler http.Handler
+       reqIDs         []string
 }
 
 func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+       fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id"))
        if fh.count == 0 {
                resp.WriteHeader(500)
                fh.count += 1
@@ -230,7 +233,7 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
                func(kc *KeepClient, url string, reader io.ReadCloser,
                        writer io.WriteCloser, upload_status chan uploadStatus) {
 
-                       go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
+                       go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, kc.getRequestID())
 
                        writer.Write([]byte("foo"))
                        writer.Close()
@@ -266,10 +269,11 @@ func (s *StandaloneSuite) TestPutB(c *C) {
                hash,
                "abc123",
                "foo",
+               "",
                make(chan string, 5)}
 
        arv, _ := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
 
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
@@ -307,10 +311,11 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
                hash,
                "abc123",
                "foo",
+               "",
                make(chan string, 5)}
 
        arv, _ := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
 
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
@@ -355,13 +360,14 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
                hash,
                "abc123",
                "foo",
+               "",
                make(chan string, 4)}
 
        fh := FailHandler{
                make(chan string, 1)}
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
 
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
@@ -413,13 +419,14 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
                hash,
                "abc123",
                "foo",
+               "",
                make(chan string, 1)}
 
        fh := FailHandler{
                make(chan string, 4)}
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
 
        kc.Want_replicas = 2
        kc.Retries = 0
@@ -445,7 +452,7 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 
        _, replicas, err := kc.PutB([]byte("foo"))
 
-       c.Check(err, Equals, InsufficientReplicasError)
+       c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
        c.Check(replicas, Equals, 1)
        c.Check(<-st.handled, Equals, ks1[0].url)
 }
@@ -480,7 +487,7 @@ func (s *StandaloneSuite) TestGet(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
@@ -504,7 +511,7 @@ func (s *StandaloneSuite) TestGet404(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
@@ -515,6 +522,27 @@ func (s *StandaloneSuite) TestGet404(c *C) {
        c.Check(r, Equals, nil)
 }
 
+func (s *StandaloneSuite) TestGetEmptyBlock(c *C) {
+       st := Error404Handler{make(chan string, 1)}
+
+       ks := RunFakeKeepServer(st)
+       defer ks.listener.Close()
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(arv)
+       arv.ApiToken = "abc123"
+       kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+       r, n, url2, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e+0")
+       c.Check(err, IsNil)
+       c.Check(n, Equals, int64(0))
+       c.Check(url2, Equals, "")
+       c.Assert(r, NotNil)
+       buf, err := ioutil.ReadAll(r)
+       c.Check(err, IsNil)
+       c.Check(buf, DeepEquals, []byte{})
+}
+
 func (s *StandaloneSuite) TestGetFail(c *C) {
        hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
@@ -524,7 +552,7 @@ func (s *StandaloneSuite) TestGetFail(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
        kc.Retries = 0
@@ -542,8 +570,9 @@ func (s *StandaloneSuite) TestGetFail(c *C) {
 func (s *StandaloneSuite) TestGetFailRetry(c *C) {
        hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
-       st := &FailThenSucceedHandler{make(chan string, 1), 0,
-               StubGetHandler{
+       st := &FailThenSucceedHandler{
+               handled: make(chan string, 1),
+               successhandler: StubGetHandler{
                        c,
                        hash,
                        "abc123",
@@ -554,7 +583,7 @@ func (s *StandaloneSuite) TestGetFailRetry(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
@@ -567,13 +596,20 @@ func (s *StandaloneSuite) TestGetFailRetry(c *C) {
        content, err2 := ioutil.ReadAll(r)
        c.Check(err2, Equals, nil)
        c.Check(content, DeepEquals, []byte("foo"))
+
+       c.Logf("%q", st.reqIDs)
+       c.Assert(len(st.reqIDs) > 1, Equals, true)
+       for _, reqid := range st.reqIDs {
+               c.Check(reqid, Not(Equals), "")
+               c.Check(reqid, Equals, st.reqIDs[0])
+       }
 }
 
 func (s *StandaloneSuite) TestGetNetError(c *C) {
        hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
 
@@ -609,7 +645,7 @@ func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(
                map[string]string{"x": ks0.url},
@@ -652,7 +688,7 @@ func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(
                map[string]string{
@@ -699,7 +735,7 @@ func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
        defer ksGateway.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(
                map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
@@ -736,7 +772,7 @@ func (s *StandaloneSuite) TestChecksum(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
@@ -770,7 +806,7 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) {
                content}
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
        writableLocalRoots := make(map[string]string)
@@ -816,7 +852,7 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
        content := []byte("TestPutGetHead")
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, err := MakeKeepClient(&arv)
+       kc, err := MakeKeepClient(arv)
        c.Assert(err, Equals, nil)
 
        hash := fmt.Sprintf("%x", md5.Sum(content))
@@ -863,7 +899,7 @@ func (s *StandaloneSuite) TestPutProxy(c *C) {
        st := StubProxyHandler{make(chan string, 1)}
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
 
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
@@ -891,7 +927,7 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
        st := StubProxyHandler{make(chan string, 1)}
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
 
        kc.Want_replicas = 3
        arv.ApiToken = "abc123"
@@ -910,7 +946,7 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
        _, replicas, err := kc.PutB([]byte("foo"))
        <-st.handled
 
-       c.Check(err, Equals, InsufficientReplicasError)
+       c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
        c.Check(replicas, Equals, 2)
 }
 
@@ -961,10 +997,11 @@ func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C
                hash,
                "abc123",
                "foo",
+               "",
                make(chan string, 5)}
 
        arv, _ := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
 
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
@@ -985,7 +1022,7 @@ func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C
 
        _, replicas, err := kc.PutB([]byte("foo"))
 
-       c.Check(err, Equals, InsufficientReplicasError)
+       c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
        c.Check(replicas, Equals, 1)
 
        c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
@@ -999,10 +1036,11 @@ func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
                hash,
                "abc123",
                "foo",
+               "",
                make(chan string, 5)}
 
        arv, _ := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
 
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
@@ -1020,7 +1058,7 @@ func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
 
        _, replicas, err := kc.PutB([]byte("foo"))
 
-       c.Check(err, Equals, InsufficientReplicasError)
+       c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
        c.Check(replicas, Equals, 0)
 }
 
@@ -1054,12 +1092,14 @@ func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       c.Assert(err, IsNil)
+       kc, err := MakeKeepClient(arv)
+       c.Assert(err, IsNil)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
        r, err := kc.GetIndex("x", "")
-       c.Check(err, Equals, nil)
+       c.Check(err, IsNil)
 
        content, err2 := ioutil.ReadAll(r)
        c.Check(err2, Equals, nil)
@@ -1080,12 +1120,12 @@ func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
        r, err := kc.GetIndex("x", hash[0:3])
-       c.Check(err, Equals, nil)
+       c.Assert(err, Equals, nil)
 
        content, err2 := ioutil.ReadAll(r)
        c.Check(err2, Equals, nil)
@@ -1106,7 +1146,7 @@ func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
@@ -1128,7 +1168,7 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
@@ -1148,7 +1188,7 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
        defer ks.listener.Close()
 
        arv, err := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 
@@ -1160,33 +1200,19 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
        c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
 }
 
-type FailThenSucceedPutHandler struct {
-       handled        chan string
-       count          int
-       successhandler StubPutHandler
-}
-
-func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
-       if h.count == 0 {
-               resp.WriteHeader(500)
-               h.count += 1
-               h.handled <- fmt.Sprintf("http://%s", req.Host)
-       } else {
-               h.successhandler.ServeHTTP(resp, req)
-       }
-}
-
 func (s *StandaloneSuite) TestPutBRetry(c *C) {
-       st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
-               StubPutHandler{
+       st := &FailThenSucceedHandler{
+               handled: make(chan string, 1),
+               successhandler: StubPutHandler{
                        c,
                        Md5String("foo"),
                        "abc123",
                        "foo",
+                       "",
                        make(chan string, 5)}}
 
        arv, _ := arvadosclient.MakeArvadosClient()
-       kc, _ := MakeKeepClient(&arv)
+       kc, _ := MakeKeepClient(arv)
 
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
@@ -1224,9 +1250,10 @@ func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
                &blobKeepService)
        c.Assert(err, Equals, nil)
        defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
+       RefreshServiceDiscovery()
 
        // Make a keepclient and ensure that the testblobstore is included
-       kc, err := MakeKeepClient(&arv)
+       kc, err := MakeKeepClient(arv)
        c.Assert(err, Equals, nil)
 
        // verify kc.LocalRoots
@@ -1252,5 +1279,5 @@ func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
 
        c.Assert(kc.replicasPerService, Equals, 0)
        c.Assert(kc.foundNonDiskSvc, Equals, true)
-       c.Assert(kc.Client.Timeout, Equals, 300*time.Second)
+       c.Assert(kc.httpClient().(*http.Client).Timeout, Equals, 300*time.Second)
 }