services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "log"
  14         "net/http"
  15         "net/http/httptest"
  16         "os"
  17         "strings"
  18         "testing"
  19         "time"
  20
  21         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  22         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  23         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  24
  25         . "gopkg.in/check.v1"
  26 )
  27
  28 // Gocheck boilerplate
  29 func Test(t *testing.T) {
  30         TestingT(t)
  31 }
  32
  33 // Gocheck boilerplate
  34 var _ = Suite(&ServerRequiredSuite{})
  35
  36 // Tests that require the Keep server running
  37 type ServerRequiredSuite struct{}
  38
  39 // Gocheck boilerplate
  40 var _ = Suite(&NoKeepServerSuite{})
  41
  42 // Test with no keepserver to simulate errors
  43 type NoKeepServerSuite struct{}
  44
  45 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  46
  47 // Wait (up to 1 second) for keepproxy to listen on a port. This
  48 // avoids a race condition where we hit a "connection refused" error
  49 // because we start testing the proxy too soon.
  50 func waitForListener() {
  51         const (
  52                 ms = 5
  53         )
  54         for i := 0; listener == nil && i < 10000; i += ms {
  55                 time.Sleep(ms * time.Millisecond)
  56         }
  57         if listener == nil {
  58                 log.Fatalf("Timed out waiting for listener to start")
  59         }
  60 }
  61
  62 func closeListener() {
  63         if listener != nil {
  64                 listener.Close()
  65         }
  66 }
  67
  68 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  69         arvadostest.StartAPI()
  70         arvadostest.StartKeep(2, false)
  71 }
  72
  73 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  74         arvadostest.ResetEnv()
  75 }
  76
  77 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  78         arvadostest.StopKeep(2)
  79         arvadostest.StopAPI()
  80 }
  81
  82 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  83         arvadostest.StartAPI()
  84         // We need API to have some keep services listed, but the
  85         // services themselves should be unresponsive.
  86         arvadostest.StartKeep(2, false)
  87         arvadostest.StopKeep(2)
  88 }
  89
  90 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  91         arvadostest.ResetEnv()
  92 }
  93
  94 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  95         arvadostest.StopAPI()
  96 }
  97
  98 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
  99         args = append([]string{"keepproxy"}, args...)
 100         os.Args = append(args, "-listen=:0")
 101         listener = nil
 102         go main()
 103         waitForListener()
 104
 105         arv, err := arvadosclient.MakeArvadosClient()
 106         c.Assert(err, Equals, nil)
 107         if bogusClientToken {
 108                 arv.ApiToken = "bogus-token"
 109         }
 110         kc := keepclient.New(arv)
 111         sr := map[string]string{
 112                 TestProxyUUID: "http://" + listener.Addr().String(),
 113         }
 114         kc.SetServiceRoots(sr, sr, sr)
 115         kc.Arvados.External = true
 116
 117         return kc
 118 }
 119
 120 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 121         runProxy(c, nil, false)
 122         defer closeListener()
 123
 124         req, err := http.NewRequest("POST",
 125                 "http://"+listener.Addr().String()+"/",
 126                 strings.NewReader("TestViaHeader"))
 127         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 128         resp, err := (&http.Client{}).Do(req)
 129         c.Assert(err, Equals, nil)
 130         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 131         locator, err := ioutil.ReadAll(resp.Body)
 132         c.Assert(err, Equals, nil)
 133         resp.Body.Close()
 134
 135         req, err = http.NewRequest("GET",
 136                 "http://"+listener.Addr().String()+"/"+string(locator),
 137                 nil)
 138         c.Assert(err, Equals, nil)
 139         resp, err = (&http.Client{}).Do(req)
 140         c.Assert(err, Equals, nil)
 141         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 142         resp.Body.Close()
 143 }
 144
 145 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 146         kc := runProxy(c, nil, false)
 147         defer closeListener()
 148
 149         sr := map[string]string{
 150                 TestProxyUUID: "http://" + listener.Addr().String(),
 151         }
 152         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 153
 154         content := []byte("TestLoopDetection")
 155         _, _, err := kc.PutB(content)
 156         c.Check(err, ErrorMatches, `.*loop detected.*`)
 157
 158         hash := fmt.Sprintf("%x", md5.Sum(content))
 159         _, _, _, err = kc.Get(hash)
 160         c.Check(err, ErrorMatches, `.*loop detected.*`)
 161 }
 162
 163 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 164         kc := runProxy(c, nil, false)
 165         defer closeListener()
 166
 167         content := []byte("TestDesiredReplicas")
 168         hash := fmt.Sprintf("%x", md5.Sum(content))
 169
 170         for _, kc.Want_replicas = range []int{0, 1, 2} {
 171                 locator, rep, err := kc.PutB(content)
 172                 c.Check(err, Equals, nil)
 173                 c.Check(rep, Equals, kc.Want_replicas)
 174                 if rep > 0 {
 175                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 176                 }
 177         }
 178 }
 179
 180 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 181         kc := runProxy(c, nil, false)
 182         defer closeListener()
 183
 184         content := []byte("TestPutWrongContentLength")
 185         hash := fmt.Sprintf("%x", md5.Sum(content))
 186
 187         // If we use http.Client to send these requests to the network
 188         // server we just started, the Go http library automatically
 189         // fixes the invalid Content-Length header. In order to test
 190         // our server behavior, we have to call the handler directly
 191         // using an httptest.ResponseRecorder.
 192         rtr := MakeRESTRouter(true, true, kc, 10*time.Second)
 193
 194         type testcase struct {
 195                 sendLength   string
 196                 expectStatus int
 197         }
 198
 199         for _, t := range []testcase{
 200                 {"1", http.StatusBadRequest},
 201                 {"", http.StatusLengthRequired},
 202                 {"-1", http.StatusLengthRequired},
 203                 {"abcdef", http.StatusLengthRequired},
 204         } {
 205                 req, err := http.NewRequest("PUT",
 206                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 207                         bytes.NewReader(content))
 208                 c.Assert(err, IsNil)
 209                 req.Header.Set("Content-Length", t.sendLength)
 210                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 211                 req.Header.Set("Content-Type", "application/octet-stream")
 212
 213                 resp := httptest.NewRecorder()
 214                 rtr.ServeHTTP(resp, req)
 215                 c.Check(resp.Code, Equals, t.expectStatus)
 216         }
 217 }
 218
 219 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 220         kc := runProxy(c, nil, false)
 221         defer closeListener()
 222
 223         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 224         var hash2 string
 225
 226         {
 227                 _, _, err := kc.Ask(hash)
 228                 c.Check(err, Equals, keepclient.BlockNotFound)
 229                 log.Print("Finished Ask (expected BlockNotFound)")
 230         }
 231
 232         {
 233                 reader, _, _, err := kc.Get(hash)
 234                 c.Check(reader, Equals, nil)
 235                 c.Check(err, Equals, keepclient.BlockNotFound)
 236                 log.Print("Finished Get (expected BlockNotFound)")
 237         }
 238
 239         // Note in bug #5309 among other errors keepproxy would set
 240         // Content-Length incorrectly on the 404 BlockNotFound response, this
 241         // would result in a protocol violation that would prevent reuse of the
 242         // connection, which would manifest by the next attempt to use the
 243         // connection (in this case the PutB below) failing.  So to test for
 244         // that bug it's necessary to trigger an error response (such as
 245         // BlockNotFound) and then do something else with the same httpClient
 246         // connection.
 247
 248         {
 249                 var rep int
 250                 var err error
 251                 hash2, rep, err = kc.PutB([]byte("foo"))
 252                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 253                 c.Check(rep, Equals, 2)
 254                 c.Check(err, Equals, nil)
 255                 log.Print("Finished PutB (expected success)")
 256         }
 257
 258         {
 259                 blocklen, _, err := kc.Ask(hash2)
 260                 c.Assert(err, Equals, nil)
 261                 c.Check(blocklen, Equals, int64(3))
 262                 log.Print("Finished Ask (expected success)")
 263         }
 264
 265         {
 266                 reader, blocklen, _, err := kc.Get(hash2)
 267                 c.Assert(err, Equals, nil)
 268                 all, err := ioutil.ReadAll(reader)
 269                 c.Check(all, DeepEquals, []byte("foo"))
 270                 c.Check(blocklen, Equals, int64(3))
 271                 log.Print("Finished Get (expected success)")
 272         }
 273
 274         {
 275                 var rep int
 276                 var err error
 277                 hash2, rep, err = kc.PutB([]byte(""))
 278                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 279                 c.Check(rep, Equals, 2)
 280                 c.Check(err, Equals, nil)
 281                 log.Print("Finished PutB zero block")
 282         }
 283
 284         {
 285                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 286                 c.Assert(err, Equals, nil)
 287                 all, err := ioutil.ReadAll(reader)
 288                 c.Check(all, DeepEquals, []byte(""))
 289                 c.Check(blocklen, Equals, int64(0))
 290                 log.Print("Finished Get zero block")
 291         }
 292 }
 293
 294 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 295         kc := runProxy(c, nil, true)
 296         defer closeListener()
 297
 298         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 299
 300         {
 301                 _, _, err := kc.Ask(hash)
 302                 errNotFound, _ := err.(keepclient.ErrNotFound)
 303                 c.Check(errNotFound, NotNil)
 304                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 305                 log.Print("Ask 1")
 306         }
 307
 308         {
 309                 hash2, rep, err := kc.PutB([]byte("bar"))
 310                 c.Check(hash2, Equals, "")
 311                 c.Check(rep, Equals, 0)
 312                 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 313                 log.Print("PutB")
 314         }
 315
 316         {
 317                 blocklen, _, err := kc.Ask(hash)
 318                 errNotFound, _ := err.(keepclient.ErrNotFound)
 319                 c.Check(errNotFound, NotNil)
 320                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 321                 c.Check(blocklen, Equals, int64(0))
 322                 log.Print("Ask 2")
 323         }
 324
 325         {
 326                 _, blocklen, _, err := kc.Get(hash)
 327                 errNotFound, _ := err.(keepclient.ErrNotFound)
 328                 c.Check(errNotFound, NotNil)
 329                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 330                 c.Check(blocklen, Equals, int64(0))
 331                 log.Print("Get")
 332         }
 333 }
 334
 335 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 336         kc := runProxy(c, []string{"-no-get"}, false)
 337         defer closeListener()
 338
 339         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 340
 341         {
 342                 _, _, err := kc.Ask(hash)
 343                 errNotFound, _ := err.(keepclient.ErrNotFound)
 344                 c.Check(errNotFound, NotNil)
 345                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 346                 log.Print("Ask 1")
 347         }
 348
 349         {
 350                 hash2, rep, err := kc.PutB([]byte("baz"))
 351                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 352                 c.Check(rep, Equals, 2)
 353                 c.Check(err, Equals, nil)
 354                 log.Print("PutB")
 355         }
 356
 357         {
 358                 blocklen, _, err := kc.Ask(hash)
 359                 errNotFound, _ := err.(keepclient.ErrNotFound)
 360                 c.Check(errNotFound, NotNil)
 361                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 362                 c.Check(blocklen, Equals, int64(0))
 363                 log.Print("Ask 2")
 364         }
 365
 366         {
 367                 _, blocklen, _, err := kc.Get(hash)
 368                 errNotFound, _ := err.(keepclient.ErrNotFound)
 369                 c.Check(errNotFound, NotNil)
 370                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 371                 c.Check(blocklen, Equals, int64(0))
 372                 log.Print("Get")
 373         }
 374 }
 375
 376 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 377         kc := runProxy(c, []string{"-no-put"}, false)
 378         defer closeListener()
 379
 380         hash2, rep, err := kc.PutB([]byte("quux"))
 381         c.Check(hash2, Equals, "")
 382         c.Check(rep, Equals, 0)
 383         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 384 }
 385
 386 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 387         runProxy(c, nil, false)
 388         defer closeListener()
 389
 390         {
 391                 client := http.Client{}
 392                 req, err := http.NewRequest("OPTIONS",
 393                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 394                         nil)
 395                 req.Header.Add("Access-Control-Request-Method", "PUT")
 396                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 397                 resp, err := client.Do(req)
 398                 c.Check(err, Equals, nil)
 399                 c.Check(resp.StatusCode, Equals, 200)
 400                 body, err := ioutil.ReadAll(resp.Body)
 401                 c.Check(string(body), Equals, "")
 402                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 403                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 404         }
 405
 406         {
 407                 resp, err := http.Get(
 408                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 409                 c.Check(err, Equals, nil)
 410                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 411                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 412         }
 413 }
 414
 415 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 416         runProxy(c, nil, false)
 417         defer closeListener()
 418
 419         {
 420                 client := http.Client{}
 421                 req, err := http.NewRequest("POST",
 422                         "http://"+listener.Addr().String()+"/",
 423                         strings.NewReader("qux"))
 424                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 425                 req.Header.Add("Content-Type", "application/octet-stream")
 426                 resp, err := client.Do(req)
 427                 c.Check(err, Equals, nil)
 428                 body, err := ioutil.ReadAll(resp.Body)
 429                 c.Check(err, Equals, nil)
 430                 c.Check(string(body), Matches,
 431                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 432         }
 433 }
 434
 435 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 436         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 437                 Equals,
 438                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 439         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 440                 Equals,
 441                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 442         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 443                 Equals,
 444                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 445         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 446                 Equals,
 447                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 448
 449 }
 450
 451 // Test GetIndex
 452 //   Put one block, with 2 replicas
 453 //   With no prefix (expect the block locator, twice)
 454 //   With an existing prefix (expect the block locator, twice)
 455 //   With a valid but non-existing prefix (expect "\n")
 456 //   With an invalid prefix (expect error)
 457 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 458         kc := runProxy(c, nil, false)
 459         defer closeListener()
 460
 461         // Put "index-data" blocks
 462         data := []byte("index-data")
 463         hash := fmt.Sprintf("%x", md5.Sum(data))
 464
 465         hash2, rep, err := kc.PutB(data)
 466         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 467         c.Check(rep, Equals, 2)
 468         c.Check(err, Equals, nil)
 469
 470         reader, blocklen, _, err := kc.Get(hash)
 471         c.Assert(err, Equals, nil)
 472         c.Check(blocklen, Equals, int64(10))
 473         all, err := ioutil.ReadAll(reader)
 474         c.Check(all, DeepEquals, data)
 475
 476         // Put some more blocks
 477         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 478         c.Check(err, Equals, nil)
 479
 480         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 481
 482         // Invoke GetIndex
 483         for _, spec := range []struct {
 484                 prefix         string
 485                 expectTestHash bool
 486                 expectOther    bool
 487         }{
 488                 {"", true, true},         // with no prefix
 489                 {hash[:3], true, false},  // with matching prefix
 490                 {"abcdef", false, false}, // with no such prefix
 491         } {
 492                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 493                 c.Assert(err, Equals, nil)
 494                 indexResp, err := ioutil.ReadAll(indexReader)
 495                 c.Assert(err, Equals, nil)
 496                 locators := strings.Split(string(indexResp), "\n")
 497                 gotTestHash := 0
 498                 gotOther := 0
 499                 for _, locator := range locators {
 500                         if locator == "" {
 501                                 continue
 502                         }
 503                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 504                         if locator[:32] == hash {
 505                                 gotTestHash++
 506                         } else {
 507                                 gotOther++
 508                         }
 509                 }
 510                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 511                 c.Check(gotOther > 0, Equals, spec.expectOther)
 512         }
 513
 514         // GetIndex with invalid prefix
 515         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 516         c.Assert((err != nil), Equals, true)
 517 }
 518
 519 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 520         kc := runProxy(c, nil, false)
 521         defer closeListener()
 522
 523         // Put a test block
 524         hash, rep, err := kc.PutB([]byte("foo"))
 525         c.Check(err, Equals, nil)
 526         c.Check(rep, Equals, 2)
 527
 528         for _, token := range []string{
 529                 "nosuchtoken",
 530                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 531         } {
 532                 // Change token to given bad token
 533                 kc.Arvados.ApiToken = token
 534
 535                 // Ask should result in error
 536                 _, _, err = kc.Ask(hash)
 537                 c.Check(err, NotNil)
 538                 errNotFound, _ := err.(keepclient.ErrNotFound)
 539                 c.Check(errNotFound.Temporary(), Equals, false)
 540                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 541
 542                 // Get should result in error
 543                 _, _, _, err = kc.Get(hash)
 544                 c.Check(err, NotNil)
 545                 errNotFound, _ = err.(keepclient.ErrNotFound)
 546                 c.Check(errNotFound.Temporary(), Equals, false)
 547                 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
 548         }
 549 }
 550
 551 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 552         arv, err := arvadosclient.MakeArvadosClient()
 553         c.Assert(err, Equals, nil)
 554
 555         // keepclient with no such keep server
 556         kc := keepclient.New(arv)
 557         locals := map[string]string{
 558                 TestProxyUUID: "http://localhost:12345",
 559         }
 560         kc.SetServiceRoots(locals, nil, nil)
 561
 562         // Ask should result in temporary connection refused error
 563         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 564         _, _, err = kc.Ask(hash)
 565         c.Check(err, NotNil)
 566         errNotFound, _ := err.(*keepclient.ErrNotFound)
 567         c.Check(errNotFound.Temporary(), Equals, true)
 568         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 569
 570         // Get should result in temporary connection refused error
 571         _, _, _, err = kc.Get(hash)
 572         c.Check(err, NotNil)
 573         errNotFound, _ = err.(*keepclient.ErrNotFound)
 574         c.Check(errNotFound.Temporary(), Equals, true)
 575         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 576 }
 577
 578 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 579         kc := runProxy(c, nil, false)
 580         defer closeListener()
 581
 582         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 583         for _, f := range []func() error{
 584                 func() error {
 585                         _, _, err := kc.Ask(hash)
 586                         return err
 587                 },
 588                 func() error {
 589                         _, _, _, err := kc.Get(hash)
 590                         return err
 591                 },
 592         } {
 593                 err := f()
 594                 c.Assert(err, NotNil)
 595                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 596                 c.Check(errNotFound.Temporary(), Equals, true)
 597                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 598         }
 599 }