Merge branch 'master' into 7492-keepproxy-upstream-errors

diff --combined sdk/go/keepclient/keepclient_test.go
index 3714a544fcec4d952f35cab58c91b2ab8ea71f39,c03ba90736868d15cdba09d0638e160e60d57998..ddfb20ec75a4485e3753779851ae91bcd93a0150
--- 1/sdk/go/keepclient/keepclient_test.go
--- 2/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@@ -14,7 -14,6 +14,7 @@@ import 
        "net"
        "net/http"
        "os"
 +      "strings"
        "testing"
  )
  
@@@ -443,6 -442,7 +443,7 @@@ func (s *StandaloneSuite) TestPutWithTo
        kc, _ := MakeKeepClient(&arv)
  
        kc.Want_replicas = 2
+       kc.Retries = 0
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
        writableLocalRoots := make(map[string]string)
@@@ -553,11 -553,10 +554,12 @@@ func (s *StandaloneSuite) TestGetFail(
        kc, _ := MakeKeepClient(&arv)
        arv.ApiToken = "abc123"
        kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+       kc.Retries = 0
  
        r, n, url2, err := kc.Get(hash)
 -      c.Check(err, Equals, BlockNotFound)
 +      errNotFound, _ := err.(ErrNotFound)
 +      c.Check(errNotFound, NotNil)
 +      c.Check(strings.Contains(err.Error(), "use of closed network connection"), Equals, true)
        c.Check(n, Equals, int64(0))
        c.Check(url2, Equals, "")
        c.Check(r, Equals, nil)
@@@ -602,9 -601,7 +604,9 @@@ func (s *StandaloneSuite) TestGetNetErr
        kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
  
        r, n, url2, err := kc.Get(hash)
 -      c.Check(err, Equals, BlockNotFound)
 +      errNotFound, _ := err.(ErrNotFound)
 +      c.Check(errNotFound, NotNil)
 +      c.Check(strings.Contains(err.Error(), "connection refused"), Equals, true)
        c.Check(n, Equals, int64(0))
        c.Check(url2, Equals, "")
        c.Check(r, Equals, nil)
@@@ -813,6 -810,7 +815,7 @@@ func (s *StandaloneSuite) TestGetWithFa
        }
  
        kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+       kc.Retries = 0
  
        // This test works only if one of the failing services is
        // attempted before the succeeding service. Otherwise,
@@@ -1191,3 -1189,53 +1194,53 @@@ func (s *StandaloneSuite) TestGetIndexW
        c.Check(err2, Equals, nil)
        c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
  }
+ 
+ type FailThenSucceedPutHandler struct {
+       handled        chan string
+       count          int
+       successhandler StubPutHandler
+ }
+ 
+ func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+       if h.count == 0 {
+               resp.WriteHeader(500)
+               h.count += 1
+               h.handled <- fmt.Sprintf("http://%s", req.Host)
+       } else {
+               h.successhandler.ServeHTTP(resp, req)
+       }
+ }
+ 
+ func (s *StandaloneSuite) TestPutBRetry(c *C) {
+       st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
+               StubPutHandler{
+                       c,
+                       Md5String("foo"),
+                       "abc123",
+                       "foo",
+                       make(chan string, 5)}}
+ 
+       arv, _ := arvadosclient.MakeArvadosClient()
+       kc, _ := MakeKeepClient(&arv)
+ 
+       kc.Want_replicas = 2
+       arv.ApiToken = "abc123"
+       localRoots := make(map[string]string)
+       writableLocalRoots := make(map[string]string)
+ 
+       ks := RunSomeFakeKeepServers(st, 2)
+ 
+       for i, k := range ks {
+               localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+               writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+               defer k.listener.Close()
+       }
+ 
+       kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+ 
+       hash, replicas, err := kc.PutB([]byte("foo"))
+ 
+       c.Check(err, Equals, nil)
+       c.Check(hash, Equals, "")
+       c.Check(replicas, Equals, 2)
+ }