services/keepproxy/keepproxy_test.go

   1 package main
   2
   3 import (
   4         "crypto/md5"
   5         "crypto/tls"
   6         "fmt"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   8         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   9         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  10         . "gopkg.in/check.v1"
  11         "io"
  12         "io/ioutil"
  13         "log"
  14         "net/http"
  15         "net/url"
  16         "os"
  17         "strings"
  18         "testing"
  19         "time"
  20 )
  21
  22 // Gocheck boilerplate
  23 func Test(t *testing.T) {
  24         TestingT(t)
  25 }
  26
  27 // Gocheck boilerplate
  28 var _ = Suite(&ServerRequiredSuite{})
  29
  30 // Tests that require the Keep server running
  31 type ServerRequiredSuite struct{}
  32
  33 // Gocheck boilerplate
  34 var _ = Suite(&NoKeepServerSuite{})
  35
  36 // Test with no keepserver to simulate errors
  37 type NoKeepServerSuite struct{}
  38
  39 // Wait (up to 1 second) for keepproxy to listen on a port. This
  40 // avoids a race condition where we hit a "connection refused" error
  41 // because we start testing the proxy too soon.
  42 func waitForListener() {
  43         const (
  44                 ms = 5
  45         )
  46         for i := 0; listener == nil && i < 1000; i += ms {
  47                 time.Sleep(ms * time.Millisecond)
  48         }
  49         if listener == nil {
  50                 log.Fatalf("Timed out waiting for listener to start")
  51         }
  52 }
  53
  54 func closeListener() {
  55         if listener != nil {
  56                 listener.Close()
  57         }
  58 }
  59
  60 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  61         arvadostest.StartAPI()
  62         arvadostest.StartKeep(2, false)
  63 }
  64
  65 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  66         arvadostest.ResetEnv()
  67 }
  68
  69 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  70         arvadostest.StopKeep(2)
  71         arvadostest.StopAPI()
  72 }
  73
  74 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  75         arvadostest.StartAPI()
  76 }
  77
  78 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  79         arvadostest.ResetEnv()
  80 }
  81
  82 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  83         arvadostest.StopAPI()
  84 }
  85
  86 func setupProxyService() {
  87
  88         client := &http.Client{Transport: &http.Transport{
  89                 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
  90
  91         var req *http.Request
  92         var err error
  93         if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil {
  94                 panic(err.Error())
  95         }
  96         req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN")))
  97
  98         reader, writer := io.Pipe()
  99
 100         req.Body = reader
 101
 102         go func() {
 103                 data := url.Values{}
 104                 data.Set("keep_service", `{
 105   "service_host": "localhost",
 106   "service_port": 29950,
 107   "service_ssl_flag": false,
 108   "service_type": "proxy"
 109 }`)
 110
 111                 writer.Write([]byte(data.Encode()))
 112                 writer.Close()
 113         }()
 114
 115         var resp *http.Response
 116         if resp, err = client.Do(req); err != nil {
 117                 panic(err.Error())
 118         }
 119         if resp.StatusCode != 200 {
 120                 panic(resp.Status)
 121         }
 122 }
 123
 124 func runProxy(c *C, args []string, port int, bogusClientToken bool) *keepclient.KeepClient {
 125         if bogusClientToken {
 126                 os.Setenv("ARVADOS_API_TOKEN", "bogus-token")
 127         }
 128         arv, err := arvadosclient.MakeArvadosClient()
 129         c.Assert(err, Equals, nil)
 130         kc := keepclient.KeepClient{
 131                 Arvados:       &arv,
 132                 Want_replicas: 2,
 133                 Using_proxy:   true,
 134                 Client:        &http.Client{},
 135         }
 136         locals := map[string]string{
 137                 "proxy": fmt.Sprintf("http://localhost:%v", port),
 138         }
 139         writableLocals := map[string]string{
 140                 "proxy": fmt.Sprintf("http://localhost:%v", port),
 141         }
 142         kc.SetServiceRoots(locals, writableLocals, nil)
 143         c.Check(kc.Using_proxy, Equals, true)
 144         c.Check(len(kc.LocalRoots()), Equals, 1)
 145         for _, root := range kc.LocalRoots() {
 146                 c.Check(root, Equals, fmt.Sprintf("http://localhost:%v", port))
 147         }
 148         log.Print("keepclient created")
 149         if bogusClientToken {
 150                 arvadostest.ResetEnv()
 151         }
 152
 153         {
 154                 os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
 155                 listener = nil
 156                 go main()
 157         }
 158
 159         return &kc
 160 }
 161
 162 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 163         log.Print("TestPutAndGet start")
 164
 165         os.Args = []string{"keepproxy", "-listen=:29950"}
 166         listener = nil
 167         go main()
 168         time.Sleep(100 * time.Millisecond)
 169
 170         setupProxyService()
 171
 172         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
 173         arv, err := arvadosclient.MakeArvadosClient()
 174         c.Assert(err, Equals, nil)
 175         kc, err := keepclient.MakeKeepClient(&arv)
 176         c.Assert(err, Equals, nil)
 177         c.Check(kc.Arvados.External, Equals, true)
 178         c.Check(kc.Using_proxy, Equals, true)
 179         c.Check(len(kc.LocalRoots()), Equals, 1)
 180         for _, root := range kc.LocalRoots() {
 181                 c.Check(root, Equals, "http://localhost:29950")
 182         }
 183         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
 184
 185         waitForListener()
 186         defer closeListener()
 187
 188         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 189         var hash2 string
 190
 191         {
 192                 _, _, err := kc.Ask(hash)
 193                 c.Check(err, Equals, keepclient.BlockNotFound)
 194                 log.Print("Finished Ask (expected BlockNotFound)")
 195         }
 196
 197         {
 198                 reader, _, _, err := kc.Get(hash)
 199                 c.Check(reader, Equals, nil)
 200                 c.Check(err, Equals, keepclient.BlockNotFound)
 201                 log.Print("Finished Get (expected BlockNotFound)")
 202         }
 203
 204         // Note in bug #5309 among other errors keepproxy would set
 205         // Content-Length incorrectly on the 404 BlockNotFound response, this
 206         // would result in a protocol violation that would prevent reuse of the
 207         // connection, which would manifest by the next attempt to use the
 208         // connection (in this case the PutB below) failing.  So to test for
 209         // that bug it's necessary to trigger an error response (such as
 210         // BlockNotFound) and then do something else with the same httpClient
 211         // connection.
 212
 213         {
 214                 var rep int
 215                 var err error
 216                 hash2, rep, err = kc.PutB([]byte("foo"))
 217                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 218                 c.Check(rep, Equals, 2)
 219                 c.Check(err, Equals, nil)
 220                 log.Print("Finished PutB (expected success)")
 221         }
 222
 223         {
 224                 blocklen, _, err := kc.Ask(hash2)
 225                 c.Assert(err, Equals, nil)
 226                 c.Check(blocklen, Equals, int64(3))
 227                 log.Print("Finished Ask (expected success)")
 228         }
 229
 230         {
 231                 reader, blocklen, _, err := kc.Get(hash2)
 232                 c.Assert(err, Equals, nil)
 233                 all, err := ioutil.ReadAll(reader)
 234                 c.Check(all, DeepEquals, []byte("foo"))
 235                 c.Check(blocklen, Equals, int64(3))
 236                 log.Print("Finished Get (expected success)")
 237         }
 238
 239         {
 240                 var rep int
 241                 var err error
 242                 hash2, rep, err = kc.PutB([]byte(""))
 243                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 244                 c.Check(rep, Equals, 2)
 245                 c.Check(err, Equals, nil)
 246                 log.Print("Finished PutB zero block")
 247         }
 248
 249         {
 250                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 251                 c.Assert(err, Equals, nil)
 252                 all, err := ioutil.ReadAll(reader)
 253                 c.Check(all, DeepEquals, []byte(""))
 254                 c.Check(blocklen, Equals, int64(0))
 255                 log.Print("Finished Get zero block")
 256         }
 257
 258         log.Print("TestPutAndGet done")
 259 }
 260
 261 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 262         log.Print("TestPutAskGetForbidden start")
 263
 264         kc := runProxy(c, []string{"keepproxy"}, 29951, true)
 265         waitForListener()
 266         defer closeListener()
 267
 268         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 269
 270         {
 271                 _, _, err := kc.Ask(hash)
 272                 errNotFound, _ := err.(keepclient.ErrNotFound)
 273                 c.Check(errNotFound, NotNil)
 274                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 275                 log.Print("Ask 1")
 276         }
 277
 278         {
 279                 hash2, rep, err := kc.PutB([]byte("bar"))
 280                 c.Check(hash2, Equals, "")
 281                 c.Check(rep, Equals, 0)
 282                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 283                 log.Print("PutB")
 284         }
 285
 286         {
 287                 blocklen, _, err := kc.Ask(hash)
 288                 errNotFound, _ := err.(keepclient.ErrNotFound)
 289                 c.Check(errNotFound, NotNil)
 290                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 291                 c.Check(blocklen, Equals, int64(0))
 292                 log.Print("Ask 2")
 293         }
 294
 295         {
 296                 _, blocklen, _, err := kc.Get(hash)
 297                 errNotFound, _ := err.(keepclient.ErrNotFound)
 298                 c.Check(errNotFound, NotNil)
 299                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 300                 c.Check(blocklen, Equals, int64(0))
 301                 log.Print("Get")
 302         }
 303
 304         log.Print("TestPutAskGetForbidden done")
 305 }
 306
 307 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 308         log.Print("TestGetDisabled start")
 309
 310         kc := runProxy(c, []string{"keepproxy", "-no-get"}, 29952, false)
 311         waitForListener()
 312         defer closeListener()
 313
 314         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 315
 316         {
 317                 _, _, err := kc.Ask(hash)
 318                 errNotFound, _ := err.(keepclient.ErrNotFound)
 319                 c.Check(errNotFound, NotNil)
 320                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 321                 log.Print("Ask 1")
 322         }
 323
 324         {
 325                 hash2, rep, err := kc.PutB([]byte("baz"))
 326                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 327                 c.Check(rep, Equals, 2)
 328                 c.Check(err, Equals, nil)
 329                 log.Print("PutB")
 330         }
 331
 332         {
 333                 blocklen, _, err := kc.Ask(hash)
 334                 errNotFound, _ := err.(keepclient.ErrNotFound)
 335                 c.Check(errNotFound, NotNil)
 336                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 337                 c.Check(blocklen, Equals, int64(0))
 338                 log.Print("Ask 2")
 339         }
 340
 341         {
 342                 _, blocklen, _, err := kc.Get(hash)
 343                 errNotFound, _ := err.(keepclient.ErrNotFound)
 344                 c.Check(errNotFound, NotNil)
 345                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 346                 c.Check(blocklen, Equals, int64(0))
 347                 log.Print("Get")
 348         }
 349
 350         log.Print("TestGetDisabled done")
 351 }
 352
 353 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 354         log.Print("TestPutDisabled start")
 355
 356         kc := runProxy(c, []string{"keepproxy", "-no-put"}, 29953, false)
 357         waitForListener()
 358         defer closeListener()
 359
 360         {
 361                 hash2, rep, err := kc.PutB([]byte("quux"))
 362                 c.Check(hash2, Equals, "")
 363                 c.Check(rep, Equals, 0)
 364                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 365                 log.Print("PutB")
 366         }
 367
 368         log.Print("TestPutDisabled done")
 369 }
 370
 371 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 372         runProxy(c, []string{"keepproxy"}, 29954, false)
 373         waitForListener()
 374         defer closeListener()
 375
 376         {
 377                 client := http.Client{}
 378                 req, err := http.NewRequest("OPTIONS",
 379                         fmt.Sprintf("http://localhost:29954/%x+3",
 380                                 md5.Sum([]byte("foo"))),
 381                         nil)
 382                 req.Header.Add("Access-Control-Request-Method", "PUT")
 383                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 384                 resp, err := client.Do(req)
 385                 c.Check(err, Equals, nil)
 386                 c.Check(resp.StatusCode, Equals, 200)
 387                 body, err := ioutil.ReadAll(resp.Body)
 388                 c.Check(string(body), Equals, "")
 389                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 390                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 391         }
 392
 393         {
 394                 resp, err := http.Get(
 395                         fmt.Sprintf("http://localhost:29954/%x+3",
 396                                 md5.Sum([]byte("foo"))))
 397                 c.Check(err, Equals, nil)
 398                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 399                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 400         }
 401 }
 402
 403 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 404         runProxy(c, []string{"keepproxy"}, 29955, false)
 405         waitForListener()
 406         defer closeListener()
 407
 408         {
 409                 client := http.Client{}
 410                 req, err := http.NewRequest("POST",
 411                         "http://localhost:29955/",
 412                         strings.NewReader("qux"))
 413                 req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 414                 req.Header.Add("Content-Type", "application/octet-stream")
 415                 resp, err := client.Do(req)
 416                 c.Check(err, Equals, nil)
 417                 body, err := ioutil.ReadAll(resp.Body)
 418                 c.Check(err, Equals, nil)
 419                 c.Check(string(body), Matches,
 420                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 421         }
 422 }
 423
 424 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 425         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 426                 Equals,
 427                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 428         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 429                 Equals,
 430                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 431         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 432                 Equals,
 433                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 434         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 435                 Equals,
 436                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 437
 438 }
 439
 440 // Test GetIndex
 441 //   Put one block, with 2 replicas
 442 //   With no prefix (expect the block locator, twice)
 443 //   With an existing prefix (expect the block locator, twice)
 444 //   With a valid but non-existing prefix (expect "\n")
 445 //   With an invalid prefix (expect error)
 446 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 447         kc := runProxy(c, []string{"keepproxy"}, 28852, false)
 448         waitForListener()
 449         defer closeListener()
 450
 451         // Put "index-data" blocks
 452         data := []byte("index-data")
 453         hash := fmt.Sprintf("%x", md5.Sum(data))
 454
 455         hash2, rep, err := kc.PutB(data)
 456         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 457         c.Check(rep, Equals, 2)
 458         c.Check(err, Equals, nil)
 459
 460         reader, blocklen, _, err := kc.Get(hash)
 461         c.Assert(err, Equals, nil)
 462         c.Check(blocklen, Equals, int64(10))
 463         all, err := ioutil.ReadAll(reader)
 464         c.Check(all, DeepEquals, data)
 465
 466         // Put some more blocks
 467         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 468         c.Check(err, Equals, nil)
 469
 470         // Invoke GetIndex
 471         for _, spec := range []struct {
 472                 prefix         string
 473                 expectTestHash bool
 474                 expectOther    bool
 475         }{
 476                 {"", true, true},         // with no prefix
 477                 {hash[:3], true, false},  // with matching prefix
 478                 {"abcdef", false, false}, // with no such prefix
 479         } {
 480                 indexReader, err := kc.GetIndex("proxy", spec.prefix)
 481                 c.Assert(err, Equals, nil)
 482                 indexResp, err := ioutil.ReadAll(indexReader)
 483                 c.Assert(err, Equals, nil)
 484                 locators := strings.Split(string(indexResp), "\n")
 485                 gotTestHash := 0
 486                 gotOther := 0
 487                 for _, locator := range locators {
 488                         if locator == "" {
 489                                 continue
 490                         }
 491                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 492                         if locator[:32] == hash {
 493                                 gotTestHash++
 494                         } else {
 495                                 gotOther++
 496                         }
 497                 }
 498                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 499                 c.Check(gotOther > 0, Equals, spec.expectOther)
 500         }
 501
 502         // GetIndex with invalid prefix
 503         _, err = kc.GetIndex("proxy", "xyz")
 504         c.Assert((err != nil), Equals, true)
 505 }
 506
 507 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 508         kc := runProxy(c, []string{"keepproxy"}, 28852, false)
 509         waitForListener()
 510         defer closeListener()
 511
 512         // Put a test block
 513         hash, rep, err := kc.PutB([]byte("foo"))
 514         c.Check(err, Equals, nil)
 515         c.Check(rep, Equals, 2)
 516
 517         for _, token := range []string{
 518                 "nosuchtoken",
 519                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 520         } {
 521                 // Change token to given bad token
 522                 kc.Arvados.ApiToken = token
 523
 524                 // Ask should result in error
 525                 _, _, err = kc.Ask(hash)
 526                 c.Check(err, NotNil)
 527                 errNotFound, _ := err.(keepclient.ErrNotFound)
 528                 c.Check(errNotFound.Temporary(), Equals, false)
 529                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 530
 531                 // Get should result in error
 532                 _, _, _, err = kc.Get(hash)
 533                 c.Check(err, NotNil)
 534                 errNotFound, _ = err.(keepclient.ErrNotFound)
 535                 c.Check(errNotFound.Temporary(), Equals, false)
 536                 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
 537         }
 538 }
 539
 540 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 541         arv, err := arvadosclient.MakeArvadosClient()
 542         c.Assert(err, Equals, nil)
 543
 544         // keepclient with no such keep server
 545         kc := keepclient.New(&arv)
 546         locals := map[string]string{
 547                 "proxy": "http://localhost:12345",
 548         }
 549         kc.SetServiceRoots(locals, nil, nil)
 550
 551         // Ask should result in temporary connection refused error
 552         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 553         _, _, err = kc.Ask(hash)
 554         c.Check(err, NotNil)
 555         errNotFound, _ := err.(*keepclient.ErrNotFound)
 556         c.Check(errNotFound.Temporary(), Equals, true)
 557         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 558
 559         // Get should result in temporary connection refused error
 560         _, _, _, err = kc.Get(hash)
 561         c.Check(err, NotNil)
 562         errNotFound, _ = err.(*keepclient.ErrNotFound)
 563         c.Check(errNotFound.Temporary(), Equals, true)
 564         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 565 }
 566
 567 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 568         kc := runProxy(c, []string{"keepproxy"}, 29999, false)
 569         waitForListener()
 570         defer closeListener()
 571
 572         // Ask should result in temporary connection refused error
 573         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 574         _, _, err := kc.Ask(hash)
 575         c.Check(err, NotNil)
 576         errNotFound, _ := err.(*keepclient.ErrNotFound)
 577         c.Check(errNotFound.Temporary(), Equals, true)
 578         c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true)
 579
 580         // Get should result in temporary connection refused error
 581         _, _, _, err = kc.Get(hash)
 582         c.Check(err, NotNil)
 583         errNotFound, _ = err.(*keepclient.ErrNotFound)
 584         c.Check(errNotFound.Temporary(), Equals, true)
 585         c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true)
 586 }