services/keepproxy/keepproxy_test.go

   1 package main
   2
   3 import (
   4         "bytes"
   5         "crypto/md5"
   6         "errors"
   7         "fmt"
   8         "io/ioutil"
   9         "log"
  10         "net/http"
  11         "net/http/httptest"
  12         "os"
  13         "strings"
  14         "testing"
  15         "time"
  16
  17         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  18         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  19         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  20
  21         . "gopkg.in/check.v1"
  22 )
  23
  24 // Gocheck boilerplate
  25 func Test(t *testing.T) {
  26         TestingT(t)
  27 }
  28
  29 // Gocheck boilerplate
  30 var _ = Suite(&ServerRequiredSuite{})
  31
  32 // Tests that require the Keep server running
  33 type ServerRequiredSuite struct{}
  34
  35 // Gocheck boilerplate
  36 var _ = Suite(&NoKeepServerSuite{})
  37
  38 // Test with no keepserver to simulate errors
  39 type NoKeepServerSuite struct{}
  40
  41 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  42
  43 // Wait (up to 1 second) for keepproxy to listen on a port. This
  44 // avoids a race condition where we hit a "connection refused" error
  45 // because we start testing the proxy too soon.
  46 func waitForListener() {
  47         const (
  48                 ms = 5
  49         )
  50         for i := 0; listener == nil && i < 10000; i += ms {
  51                 time.Sleep(ms * time.Millisecond)
  52         }
  53         if listener == nil {
  54                 log.Fatalf("Timed out waiting for listener to start")
  55         }
  56 }
  57
  58 func closeListener() {
  59         if listener != nil {
  60                 listener.Close()
  61         }
  62 }
  63
  64 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  65         arvadostest.StartAPI()
  66         arvadostest.StartKeep(2, false)
  67 }
  68
  69 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  70         arvadostest.ResetEnv()
  71 }
  72
  73 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  74         arvadostest.StopKeep(2)
  75         arvadostest.StopAPI()
  76 }
  77
  78 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  79         arvadostest.StartAPI()
  80         // We need API to have some keep services listed, but the
  81         // services themselves should be unresponsive.
  82         arvadostest.StartKeep(2, false)
  83         arvadostest.StopKeep(2)
  84 }
  85
  86 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  87         arvadostest.ResetEnv()
  88 }
  89
  90 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  91         arvadostest.StopAPI()
  92 }
  93
  94 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
  95         args = append([]string{"keepproxy"}, args...)
  96         os.Args = append(args, "-listen=:0")
  97         listener = nil
  98         go main()
  99         waitForListener()
 100
 101         arv, err := arvadosclient.MakeArvadosClient()
 102         c.Assert(err, Equals, nil)
 103         if bogusClientToken {
 104                 arv.ApiToken = "bogus-token"
 105         }
 106         kc := keepclient.New(arv)
 107         sr := map[string]string{
 108                 TestProxyUUID: "http://" + listener.Addr().String(),
 109         }
 110         kc.SetServiceRoots(sr, sr, sr)
 111         kc.Arvados.External = true
 112
 113         return kc
 114 }
 115
 116 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 117         runProxy(c, nil, false)
 118         defer closeListener()
 119
 120         req, err := http.NewRequest("POST",
 121                 "http://"+listener.Addr().String()+"/",
 122                 strings.NewReader("TestViaHeader"))
 123         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 124         resp, err := (&http.Client{}).Do(req)
 125         c.Assert(err, Equals, nil)
 126         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 127         locator, err := ioutil.ReadAll(resp.Body)
 128         c.Assert(err, Equals, nil)
 129         resp.Body.Close()
 130
 131         req, err = http.NewRequest("GET",
 132                 "http://"+listener.Addr().String()+"/"+string(locator),
 133                 nil)
 134         c.Assert(err, Equals, nil)
 135         resp, err = (&http.Client{}).Do(req)
 136         c.Assert(err, Equals, nil)
 137         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 138         resp.Body.Close()
 139 }
 140
 141 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 142         kc := runProxy(c, nil, false)
 143         defer closeListener()
 144
 145         sr := map[string]string{
 146                 TestProxyUUID: "http://" + listener.Addr().String(),
 147         }
 148         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 149
 150         content := []byte("TestLoopDetection")
 151         _, _, err := kc.PutB(content)
 152         c.Check(err, ErrorMatches, `.*loop detected.*`)
 153
 154         hash := fmt.Sprintf("%x", md5.Sum(content))
 155         _, _, _, err = kc.Get(hash)
 156         c.Check(err, ErrorMatches, `.*loop detected.*`)
 157 }
 158
 159 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 160         kc := runProxy(c, nil, false)
 161         defer closeListener()
 162
 163         content := []byte("TestDesiredReplicas")
 164         hash := fmt.Sprintf("%x", md5.Sum(content))
 165
 166         for _, kc.Want_replicas = range []int{0, 1, 2} {
 167                 locator, rep, err := kc.PutB(content)
 168                 c.Check(err, Equals, nil)
 169                 c.Check(rep, Equals, kc.Want_replicas)
 170                 if rep > 0 {
 171                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 172                 }
 173         }
 174 }
 175
 176 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 177         kc := runProxy(c, nil, false)
 178         defer closeListener()
 179
 180         content := []byte("TestPutWrongContentLength")
 181         hash := fmt.Sprintf("%x", md5.Sum(content))
 182
 183         // If we use http.Client to send these requests to the network
 184         // server we just started, the Go http library automatically
 185         // fixes the invalid Content-Length header. In order to test
 186         // our server behavior, we have to call the handler directly
 187         // using an httptest.ResponseRecorder.
 188         rtr := MakeRESTRouter(true, true, kc, 10*time.Second)
 189
 190         type testcase struct {
 191                 sendLength   string
 192                 expectStatus int
 193         }
 194
 195         for _, t := range []testcase{
 196                 {"1", http.StatusBadRequest},
 197                 {"", http.StatusLengthRequired},
 198                 {"-1", http.StatusLengthRequired},
 199                 {"abcdef", http.StatusLengthRequired},
 200         } {
 201                 req, err := http.NewRequest("PUT",
 202                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 203                         bytes.NewReader(content))
 204                 c.Assert(err, IsNil)
 205                 req.Header.Set("Content-Length", t.sendLength)
 206                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 207                 req.Header.Set("Content-Type", "application/octet-stream")
 208
 209                 resp := httptest.NewRecorder()
 210                 rtr.ServeHTTP(resp, req)
 211                 c.Check(resp.Code, Equals, t.expectStatus)
 212         }
 213 }
 214
 215 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 216         kc := runProxy(c, nil, false)
 217         defer closeListener()
 218
 219         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 220         var hash2 string
 221
 222         {
 223                 _, _, err := kc.Ask(hash)
 224                 c.Check(err, Equals, keepclient.BlockNotFound)
 225                 log.Print("Finished Ask (expected BlockNotFound)")
 226         }
 227
 228         {
 229                 reader, _, _, err := kc.Get(hash)
 230                 c.Check(reader, Equals, nil)
 231                 c.Check(err, Equals, keepclient.BlockNotFound)
 232                 log.Print("Finished Get (expected BlockNotFound)")
 233         }
 234
 235         // Note in bug #5309 among other errors keepproxy would set
 236         // Content-Length incorrectly on the 404 BlockNotFound response, this
 237         // would result in a protocol violation that would prevent reuse of the
 238         // connection, which would manifest by the next attempt to use the
 239         // connection (in this case the PutB below) failing.  So to test for
 240         // that bug it's necessary to trigger an error response (such as
 241         // BlockNotFound) and then do something else with the same httpClient
 242         // connection.
 243
 244         {
 245                 var rep int
 246                 var err error
 247                 hash2, rep, err = kc.PutB([]byte("foo"))
 248                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 249                 c.Check(rep, Equals, 2)
 250                 c.Check(err, Equals, nil)
 251                 log.Print("Finished PutB (expected success)")
 252         }
 253
 254         {
 255                 blocklen, _, err := kc.Ask(hash2)
 256                 c.Assert(err, Equals, nil)
 257                 c.Check(blocklen, Equals, int64(3))
 258                 log.Print("Finished Ask (expected success)")
 259         }
 260
 261         {
 262                 reader, blocklen, _, err := kc.Get(hash2)
 263                 c.Assert(err, Equals, nil)
 264                 all, err := ioutil.ReadAll(reader)
 265                 c.Check(all, DeepEquals, []byte("foo"))
 266                 c.Check(blocklen, Equals, int64(3))
 267                 log.Print("Finished Get (expected success)")
 268         }
 269
 270         {
 271                 var rep int
 272                 var err error
 273                 hash2, rep, err = kc.PutB([]byte(""))
 274                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 275                 c.Check(rep, Equals, 2)
 276                 c.Check(err, Equals, nil)
 277                 log.Print("Finished PutB zero block")
 278         }
 279
 280         {
 281                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 282                 c.Assert(err, Equals, nil)
 283                 all, err := ioutil.ReadAll(reader)
 284                 c.Check(all, DeepEquals, []byte(""))
 285                 c.Check(blocklen, Equals, int64(0))
 286                 log.Print("Finished Get zero block")
 287         }
 288 }
 289
 290 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 291         kc := runProxy(c, nil, true)
 292         defer closeListener()
 293
 294         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 295
 296         {
 297                 _, _, err := kc.Ask(hash)
 298                 errNotFound, _ := err.(keepclient.ErrNotFound)
 299                 c.Check(errNotFound, NotNil)
 300                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 301                 log.Print("Ask 1")
 302         }
 303
 304         {
 305                 hash2, rep, err := kc.PutB([]byte("bar"))
 306                 c.Check(hash2, Equals, "")
 307                 c.Check(rep, Equals, 0)
 308                 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 309                 log.Print("PutB")
 310         }
 311
 312         {
 313                 blocklen, _, err := kc.Ask(hash)
 314                 errNotFound, _ := err.(keepclient.ErrNotFound)
 315                 c.Check(errNotFound, NotNil)
 316                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 317                 c.Check(blocklen, Equals, int64(0))
 318                 log.Print("Ask 2")
 319         }
 320
 321         {
 322                 _, blocklen, _, err := kc.Get(hash)
 323                 errNotFound, _ := err.(keepclient.ErrNotFound)
 324                 c.Check(errNotFound, NotNil)
 325                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 326                 c.Check(blocklen, Equals, int64(0))
 327                 log.Print("Get")
 328         }
 329 }
 330
 331 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 332         kc := runProxy(c, []string{"-no-get"}, false)
 333         defer closeListener()
 334
 335         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 336
 337         {
 338                 _, _, err := kc.Ask(hash)
 339                 errNotFound, _ := err.(keepclient.ErrNotFound)
 340                 c.Check(errNotFound, NotNil)
 341                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 342                 log.Print("Ask 1")
 343         }
 344
 345         {
 346                 hash2, rep, err := kc.PutB([]byte("baz"))
 347                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 348                 c.Check(rep, Equals, 2)
 349                 c.Check(err, Equals, nil)
 350                 log.Print("PutB")
 351         }
 352
 353         {
 354                 blocklen, _, err := kc.Ask(hash)
 355                 errNotFound, _ := err.(keepclient.ErrNotFound)
 356                 c.Check(errNotFound, NotNil)
 357                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 358                 c.Check(blocklen, Equals, int64(0))
 359                 log.Print("Ask 2")
 360         }
 361
 362         {
 363                 _, blocklen, _, err := kc.Get(hash)
 364                 errNotFound, _ := err.(keepclient.ErrNotFound)
 365                 c.Check(errNotFound, NotNil)
 366                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 367                 c.Check(blocklen, Equals, int64(0))
 368                 log.Print("Get")
 369         }
 370 }
 371
 372 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 373         kc := runProxy(c, []string{"-no-put"}, false)
 374         defer closeListener()
 375
 376         hash2, rep, err := kc.PutB([]byte("quux"))
 377         c.Check(hash2, Equals, "")
 378         c.Check(rep, Equals, 0)
 379         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 380 }
 381
 382 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 383         runProxy(c, nil, false)
 384         defer closeListener()
 385
 386         {
 387                 client := http.Client{}
 388                 req, err := http.NewRequest("OPTIONS",
 389                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 390                         nil)
 391                 req.Header.Add("Access-Control-Request-Method", "PUT")
 392                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 393                 resp, err := client.Do(req)
 394                 c.Check(err, Equals, nil)
 395                 c.Check(resp.StatusCode, Equals, 200)
 396                 body, err := ioutil.ReadAll(resp.Body)
 397                 c.Check(string(body), Equals, "")
 398                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 399                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 400         }
 401
 402         {
 403                 resp, err := http.Get(
 404                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 405                 c.Check(err, Equals, nil)
 406                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 407                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 408         }
 409 }
 410
 411 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 412         runProxy(c, nil, false)
 413         defer closeListener()
 414
 415         {
 416                 client := http.Client{}
 417                 req, err := http.NewRequest("POST",
 418                         "http://"+listener.Addr().String()+"/",
 419                         strings.NewReader("qux"))
 420                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 421                 req.Header.Add("Content-Type", "application/octet-stream")
 422                 resp, err := client.Do(req)
 423                 c.Check(err, Equals, nil)
 424                 body, err := ioutil.ReadAll(resp.Body)
 425                 c.Check(err, Equals, nil)
 426                 c.Check(string(body), Matches,
 427                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 428         }
 429 }
 430
 431 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 432         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 433                 Equals,
 434                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 435         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 436                 Equals,
 437                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 438         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 439                 Equals,
 440                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 441         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 442                 Equals,
 443                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 444
 445 }
 446
 447 // Test GetIndex
 448 //   Put one block, with 2 replicas
 449 //   With no prefix (expect the block locator, twice)
 450 //   With an existing prefix (expect the block locator, twice)
 451 //   With a valid but non-existing prefix (expect "\n")
 452 //   With an invalid prefix (expect error)
 453 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 454         kc := runProxy(c, nil, false)
 455         defer closeListener()
 456
 457         // Put "index-data" blocks
 458         data := []byte("index-data")
 459         hash := fmt.Sprintf("%x", md5.Sum(data))
 460
 461         hash2, rep, err := kc.PutB(data)
 462         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 463         c.Check(rep, Equals, 2)
 464         c.Check(err, Equals, nil)
 465
 466         reader, blocklen, _, err := kc.Get(hash)
 467         c.Assert(err, Equals, nil)
 468         c.Check(blocklen, Equals, int64(10))
 469         all, err := ioutil.ReadAll(reader)
 470         c.Check(all, DeepEquals, data)
 471
 472         // Put some more blocks
 473         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 474         c.Check(err, Equals, nil)
 475
 476         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 477
 478         // Invoke GetIndex
 479         for _, spec := range []struct {
 480                 prefix         string
 481                 expectTestHash bool
 482                 expectOther    bool
 483         }{
 484                 {"", true, true},         // with no prefix
 485                 {hash[:3], true, false},  // with matching prefix
 486                 {"abcdef", false, false}, // with no such prefix
 487         } {
 488                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 489                 c.Assert(err, Equals, nil)
 490                 indexResp, err := ioutil.ReadAll(indexReader)
 491                 c.Assert(err, Equals, nil)
 492                 locators := strings.Split(string(indexResp), "\n")
 493                 gotTestHash := 0
 494                 gotOther := 0
 495                 for _, locator := range locators {
 496                         if locator == "" {
 497                                 continue
 498                         }
 499                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 500                         if locator[:32] == hash {
 501                                 gotTestHash++
 502                         } else {
 503                                 gotOther++
 504                         }
 505                 }
 506                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 507                 c.Check(gotOther > 0, Equals, spec.expectOther)
 508         }
 509
 510         // GetIndex with invalid prefix
 511         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 512         c.Assert((err != nil), Equals, true)
 513 }
 514
 515 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 516         kc := runProxy(c, nil, false)
 517         defer closeListener()
 518
 519         // Put a test block
 520         hash, rep, err := kc.PutB([]byte("foo"))
 521         c.Check(err, Equals, nil)
 522         c.Check(rep, Equals, 2)
 523
 524         for _, token := range []string{
 525                 "nosuchtoken",
 526                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 527         } {
 528                 // Change token to given bad token
 529                 kc.Arvados.ApiToken = token
 530
 531                 // Ask should result in error
 532                 _, _, err = kc.Ask(hash)
 533                 c.Check(err, NotNil)
 534                 errNotFound, _ := err.(keepclient.ErrNotFound)
 535                 c.Check(errNotFound.Temporary(), Equals, false)
 536                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 537
 538                 // Get should result in error
 539                 _, _, _, err = kc.Get(hash)
 540                 c.Check(err, NotNil)
 541                 errNotFound, _ = err.(keepclient.ErrNotFound)
 542                 c.Check(errNotFound.Temporary(), Equals, false)
 543                 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
 544         }
 545 }
 546
 547 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 548         arv, err := arvadosclient.MakeArvadosClient()
 549         c.Assert(err, Equals, nil)
 550
 551         // keepclient with no such keep server
 552         kc := keepclient.New(arv)
 553         locals := map[string]string{
 554                 TestProxyUUID: "http://localhost:12345",
 555         }
 556         kc.SetServiceRoots(locals, nil, nil)
 557
 558         // Ask should result in temporary connection refused error
 559         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 560         _, _, err = kc.Ask(hash)
 561         c.Check(err, NotNil)
 562         errNotFound, _ := err.(*keepclient.ErrNotFound)
 563         c.Check(errNotFound.Temporary(), Equals, true)
 564         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 565
 566         // Get should result in temporary connection refused error
 567         _, _, _, err = kc.Get(hash)
 568         c.Check(err, NotNil)
 569         errNotFound, _ = err.(*keepclient.ErrNotFound)
 570         c.Check(errNotFound.Temporary(), Equals, true)
 571         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 572 }
 573
 574 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 575         kc := runProxy(c, nil, false)
 576         defer closeListener()
 577
 578         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 579         for _, f := range []func() error{
 580                 func() error {
 581                         _, _, err := kc.Ask(hash)
 582                         return err
 583                 },
 584                 func() error {
 585                         _, _, _, err := kc.Get(hash)
 586                         return err
 587                 },
 588         } {
 589                 err := f()
 590                 c.Assert(err, NotNil)
 591                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 592                 c.Check(errNotFound.Temporary(), Equals, true)
 593                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 594         }
 595 }