services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "os"
  16         "strings"
  17         "testing"
  18         "time"
  19
  20         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  21         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  22         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  23
  24         . "gopkg.in/check.v1"
  25 )
  26
  27 // Gocheck boilerplate
  28 func Test(t *testing.T) {
  29         TestingT(t)
  30 }
  31
  32 // Gocheck boilerplate
  33 var _ = Suite(&ServerRequiredSuite{})
  34
  35 // Tests that require the Keep server running
  36 type ServerRequiredSuite struct{}
  37
  38 // Gocheck boilerplate
  39 var _ = Suite(&NoKeepServerSuite{})
  40
  41 // Test with no keepserver to simulate errors
  42 type NoKeepServerSuite struct{}
  43
  44 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  45
  46 // Wait (up to 1 second) for keepproxy to listen on a port. This
  47 // avoids a race condition where we hit a "connection refused" error
  48 // because we start testing the proxy too soon.
  49 func waitForListener() {
  50         const (
  51                 ms = 5
  52         )
  53         for i := 0; listener == nil && i < 10000; i += ms {
  54                 time.Sleep(ms * time.Millisecond)
  55         }
  56         if listener == nil {
  57                 panic("Timed out waiting for listener to start")
  58         }
  59 }
  60
  61 func closeListener() {
  62         if listener != nil {
  63                 listener.Close()
  64         }
  65 }
  66
  67 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  68         arvadostest.StartAPI()
  69         arvadostest.StartKeep(2, false)
  70 }
  71
  72 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  73         arvadostest.ResetEnv()
  74 }
  75
  76 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  77         arvadostest.StopKeep(2)
  78         arvadostest.StopAPI()
  79 }
  80
  81 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  82         arvadostest.StartAPI()
  83         // We need API to have some keep services listed, but the
  84         // services themselves should be unresponsive.
  85         arvadostest.StartKeep(2, false)
  86         arvadostest.StopKeep(2)
  87 }
  88
  89 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  90         arvadostest.ResetEnv()
  91 }
  92
  93 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  94         arvadostest.StopAPI()
  95 }
  96
  97 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
  98         args = append([]string{"keepproxy"}, args...)
  99         os.Args = append(args, "-listen=:0")
 100         listener = nil
 101         go main()
 102         waitForListener()
 103
 104         arv, err := arvadosclient.MakeArvadosClient()
 105         c.Assert(err, Equals, nil)
 106         if bogusClientToken {
 107                 arv.ApiToken = "bogus-token"
 108         }
 109         kc := keepclient.New(arv)
 110         sr := map[string]string{
 111                 TestProxyUUID: "http://" + listener.Addr().String(),
 112         }
 113         kc.SetServiceRoots(sr, sr, sr)
 114         kc.Arvados.External = true
 115
 116         return kc
 117 }
 118
 119 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 120         runProxy(c, nil, false)
 121         defer closeListener()
 122
 123         req, err := http.NewRequest("POST",
 124                 "http://"+listener.Addr().String()+"/",
 125                 strings.NewReader("TestViaHeader"))
 126         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 127         resp, err := (&http.Client{}).Do(req)
 128         c.Assert(err, Equals, nil)
 129         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 130         locator, err := ioutil.ReadAll(resp.Body)
 131         c.Assert(err, Equals, nil)
 132         resp.Body.Close()
 133
 134         req, err = http.NewRequest("GET",
 135                 "http://"+listener.Addr().String()+"/"+string(locator),
 136                 nil)
 137         c.Assert(err, Equals, nil)
 138         resp, err = (&http.Client{}).Do(req)
 139         c.Assert(err, Equals, nil)
 140         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 141         resp.Body.Close()
 142 }
 143
 144 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 145         kc := runProxy(c, nil, false)
 146         defer closeListener()
 147
 148         sr := map[string]string{
 149                 TestProxyUUID: "http://" + listener.Addr().String(),
 150         }
 151         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 152
 153         content := []byte("TestLoopDetection")
 154         _, _, err := kc.PutB(content)
 155         c.Check(err, ErrorMatches, `.*loop detected.*`)
 156
 157         hash := fmt.Sprintf("%x", md5.Sum(content))
 158         _, _, _, err = kc.Get(hash)
 159         c.Check(err, ErrorMatches, `.*loop detected.*`)
 160 }
 161
 162 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 163         kc := runProxy(c, nil, false)
 164         defer closeListener()
 165
 166         content := []byte("TestDesiredReplicas")
 167         hash := fmt.Sprintf("%x", md5.Sum(content))
 168
 169         for _, kc.Want_replicas = range []int{0, 1, 2} {
 170                 locator, rep, err := kc.PutB(content)
 171                 c.Check(err, Equals, nil)
 172                 c.Check(rep, Equals, kc.Want_replicas)
 173                 if rep > 0 {
 174                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 175                 }
 176         }
 177 }
 178
 179 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 180         kc := runProxy(c, nil, false)
 181         defer closeListener()
 182
 183         content := []byte("TestPutWrongContentLength")
 184         hash := fmt.Sprintf("%x", md5.Sum(content))
 185
 186         // If we use http.Client to send these requests to the network
 187         // server we just started, the Go http library automatically
 188         // fixes the invalid Content-Length header. In order to test
 189         // our server behavior, we have to call the handler directly
 190         // using an httptest.ResponseRecorder.
 191         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
 192
 193         type testcase struct {
 194                 sendLength   string
 195                 expectStatus int
 196         }
 197
 198         for _, t := range []testcase{
 199                 {"1", http.StatusBadRequest},
 200                 {"", http.StatusLengthRequired},
 201                 {"-1", http.StatusLengthRequired},
 202                 {"abcdef", http.StatusLengthRequired},
 203         } {
 204                 req, err := http.NewRequest("PUT",
 205                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 206                         bytes.NewReader(content))
 207                 c.Assert(err, IsNil)
 208                 req.Header.Set("Content-Length", t.sendLength)
 209                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 210                 req.Header.Set("Content-Type", "application/octet-stream")
 211
 212                 resp := httptest.NewRecorder()
 213                 rtr.ServeHTTP(resp, req)
 214                 c.Check(resp.Code, Equals, t.expectStatus)
 215         }
 216 }
 217
 218 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 219         kc := runProxy(c, nil, false)
 220         defer closeListener()
 221
 222         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 223         var hash2 string
 224
 225         {
 226                 _, _, err := kc.Ask(hash)
 227                 c.Check(err, Equals, keepclient.BlockNotFound)
 228                 c.Log("Finished Ask (expected BlockNotFound)")
 229         }
 230
 231         {
 232                 reader, _, _, err := kc.Get(hash)
 233                 c.Check(reader, Equals, nil)
 234                 c.Check(err, Equals, keepclient.BlockNotFound)
 235                 c.Log("Finished Get (expected BlockNotFound)")
 236         }
 237
 238         // Note in bug #5309 among other errors keepproxy would set
 239         // Content-Length incorrectly on the 404 BlockNotFound response, this
 240         // would result in a protocol violation that would prevent reuse of the
 241         // connection, which would manifest by the next attempt to use the
 242         // connection (in this case the PutB below) failing.  So to test for
 243         // that bug it's necessary to trigger an error response (such as
 244         // BlockNotFound) and then do something else with the same httpClient
 245         // connection.
 246
 247         {
 248                 var rep int
 249                 var err error
 250                 hash2, rep, err = kc.PutB([]byte("foo"))
 251                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 252                 c.Check(rep, Equals, 2)
 253                 c.Check(err, Equals, nil)
 254                 c.Log("Finished PutB (expected success)")
 255         }
 256
 257         {
 258                 blocklen, _, err := kc.Ask(hash2)
 259                 c.Assert(err, Equals, nil)
 260                 c.Check(blocklen, Equals, int64(3))
 261                 c.Log("Finished Ask (expected success)")
 262         }
 263
 264         {
 265                 reader, blocklen, _, err := kc.Get(hash2)
 266                 c.Assert(err, Equals, nil)
 267                 all, err := ioutil.ReadAll(reader)
 268                 c.Check(all, DeepEquals, []byte("foo"))
 269                 c.Check(blocklen, Equals, int64(3))
 270                 c.Log("Finished Get (expected success)")
 271         }
 272
 273         {
 274                 var rep int
 275                 var err error
 276                 hash2, rep, err = kc.PutB([]byte(""))
 277                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 278                 c.Check(rep, Equals, 2)
 279                 c.Check(err, Equals, nil)
 280                 c.Log("Finished PutB zero block")
 281         }
 282
 283         {
 284                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 285                 c.Assert(err, Equals, nil)
 286                 all, err := ioutil.ReadAll(reader)
 287                 c.Check(all, DeepEquals, []byte(""))
 288                 c.Check(blocklen, Equals, int64(0))
 289                 c.Log("Finished Get zero block")
 290         }
 291 }
 292
 293 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 294         kc := runProxy(c, nil, true)
 295         defer closeListener()
 296
 297         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 298
 299         {
 300                 _, _, err := kc.Ask(hash)
 301                 errNotFound, _ := err.(keepclient.ErrNotFound)
 302                 c.Check(errNotFound, NotNil)
 303                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 304                 c.Log("Ask 1")
 305         }
 306
 307         {
 308                 hash2, rep, err := kc.PutB([]byte("bar"))
 309                 c.Check(hash2, Equals, "")
 310                 c.Check(rep, Equals, 0)
 311                 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 312                 c.Log("PutB")
 313         }
 314
 315         {
 316                 blocklen, _, err := kc.Ask(hash)
 317                 errNotFound, _ := err.(keepclient.ErrNotFound)
 318                 c.Check(errNotFound, NotNil)
 319                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 320                 c.Check(blocklen, Equals, int64(0))
 321                 c.Log("Ask 2")
 322         }
 323
 324         {
 325                 _, blocklen, _, err := kc.Get(hash)
 326                 errNotFound, _ := err.(keepclient.ErrNotFound)
 327                 c.Check(errNotFound, NotNil)
 328                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 329                 c.Check(blocklen, Equals, int64(0))
 330                 c.Log("Get")
 331         }
 332 }
 333
 334 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 335         kc := runProxy(c, []string{"-no-get"}, false)
 336         defer closeListener()
 337
 338         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 339
 340         {
 341                 _, _, err := kc.Ask(hash)
 342                 errNotFound, _ := err.(keepclient.ErrNotFound)
 343                 c.Check(errNotFound, NotNil)
 344                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 345                 c.Log("Ask 1")
 346         }
 347
 348         {
 349                 hash2, rep, err := kc.PutB([]byte("baz"))
 350                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 351                 c.Check(rep, Equals, 2)
 352                 c.Check(err, Equals, nil)
 353                 c.Log("PutB")
 354         }
 355
 356         {
 357                 blocklen, _, err := kc.Ask(hash)
 358                 errNotFound, _ := err.(keepclient.ErrNotFound)
 359                 c.Check(errNotFound, NotNil)
 360                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 361                 c.Check(blocklen, Equals, int64(0))
 362                 c.Log("Ask 2")
 363         }
 364
 365         {
 366                 _, blocklen, _, err := kc.Get(hash)
 367                 errNotFound, _ := err.(keepclient.ErrNotFound)
 368                 c.Check(errNotFound, NotNil)
 369                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 370                 c.Check(blocklen, Equals, int64(0))
 371                 c.Log("Get")
 372         }
 373 }
 374
 375 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 376         kc := runProxy(c, []string{"-no-put"}, false)
 377         defer closeListener()
 378
 379         hash2, rep, err := kc.PutB([]byte("quux"))
 380         c.Check(hash2, Equals, "")
 381         c.Check(rep, Equals, 0)
 382         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 383 }
 384
 385 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 386         runProxy(c, nil, false)
 387         defer closeListener()
 388
 389         {
 390                 client := http.Client{}
 391                 req, err := http.NewRequest("OPTIONS",
 392                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 393                         nil)
 394                 req.Header.Add("Access-Control-Request-Method", "PUT")
 395                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 396                 resp, err := client.Do(req)
 397                 c.Check(err, Equals, nil)
 398                 c.Check(resp.StatusCode, Equals, 200)
 399                 body, err := ioutil.ReadAll(resp.Body)
 400                 c.Check(string(body), Equals, "")
 401                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 402                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 403         }
 404
 405         {
 406                 resp, err := http.Get(
 407                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 408                 c.Check(err, Equals, nil)
 409                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 410                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 411         }
 412 }
 413
 414 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 415         runProxy(c, nil, false)
 416         defer closeListener()
 417
 418         {
 419                 client := http.Client{}
 420                 req, err := http.NewRequest("POST",
 421                         "http://"+listener.Addr().String()+"/",
 422                         strings.NewReader("qux"))
 423                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 424                 req.Header.Add("Content-Type", "application/octet-stream")
 425                 resp, err := client.Do(req)
 426                 c.Check(err, Equals, nil)
 427                 body, err := ioutil.ReadAll(resp.Body)
 428                 c.Check(err, Equals, nil)
 429                 c.Check(string(body), Matches,
 430                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 431         }
 432 }
 433
 434 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 435         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 436                 Equals,
 437                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 438         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 439                 Equals,
 440                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 441         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 442                 Equals,
 443                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 444         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 445                 Equals,
 446                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 447
 448 }
 449
 450 // Test GetIndex
 451 //   Put one block, with 2 replicas
 452 //   With no prefix (expect the block locator, twice)
 453 //   With an existing prefix (expect the block locator, twice)
 454 //   With a valid but non-existing prefix (expect "\n")
 455 //   With an invalid prefix (expect error)
 456 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 457         kc := runProxy(c, nil, false)
 458         defer closeListener()
 459
 460         // Put "index-data" blocks
 461         data := []byte("index-data")
 462         hash := fmt.Sprintf("%x", md5.Sum(data))
 463
 464         hash2, rep, err := kc.PutB(data)
 465         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 466         c.Check(rep, Equals, 2)
 467         c.Check(err, Equals, nil)
 468
 469         reader, blocklen, _, err := kc.Get(hash)
 470         c.Assert(err, Equals, nil)
 471         c.Check(blocklen, Equals, int64(10))
 472         all, err := ioutil.ReadAll(reader)
 473         c.Check(all, DeepEquals, data)
 474
 475         // Put some more blocks
 476         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 477         c.Check(err, Equals, nil)
 478
 479         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 480
 481         // Invoke GetIndex
 482         for _, spec := range []struct {
 483                 prefix         string
 484                 expectTestHash bool
 485                 expectOther    bool
 486         }{
 487                 {"", true, true},         // with no prefix
 488                 {hash[:3], true, false},  // with matching prefix
 489                 {"abcdef", false, false}, // with no such prefix
 490         } {
 491                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 492                 c.Assert(err, Equals, nil)
 493                 indexResp, err := ioutil.ReadAll(indexReader)
 494                 c.Assert(err, Equals, nil)
 495                 locators := strings.Split(string(indexResp), "\n")
 496                 gotTestHash := 0
 497                 gotOther := 0
 498                 for _, locator := range locators {
 499                         if locator == "" {
 500                                 continue
 501                         }
 502                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 503                         if locator[:32] == hash {
 504                                 gotTestHash++
 505                         } else {
 506                                 gotOther++
 507                         }
 508                 }
 509                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 510                 c.Check(gotOther > 0, Equals, spec.expectOther)
 511         }
 512
 513         // GetIndex with invalid prefix
 514         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 515         c.Assert((err != nil), Equals, true)
 516 }
 517
 518 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 519         kc := runProxy(c, nil, false)
 520         defer closeListener()
 521
 522         // Put a test block
 523         hash, rep, err := kc.PutB([]byte("foo"))
 524         c.Check(err, Equals, nil)
 525         c.Check(rep, Equals, 2)
 526
 527         for _, token := range []string{
 528                 "nosuchtoken",
 529                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 530         } {
 531                 // Change token to given bad token
 532                 kc.Arvados.ApiToken = token
 533
 534                 // Ask should result in error
 535                 _, _, err = kc.Ask(hash)
 536                 c.Check(err, NotNil)
 537                 errNotFound, _ := err.(keepclient.ErrNotFound)
 538                 c.Check(errNotFound.Temporary(), Equals, false)
 539                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 540
 541                 // Get should result in error
 542                 _, _, _, err = kc.Get(hash)
 543                 c.Check(err, NotNil)
 544                 errNotFound, _ = err.(keepclient.ErrNotFound)
 545                 c.Check(errNotFound.Temporary(), Equals, false)
 546                 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
 547         }
 548 }
 549
 550 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 551         arv, err := arvadosclient.MakeArvadosClient()
 552         c.Assert(err, Equals, nil)
 553
 554         // keepclient with no such keep server
 555         kc := keepclient.New(arv)
 556         locals := map[string]string{
 557                 TestProxyUUID: "http://localhost:12345",
 558         }
 559         kc.SetServiceRoots(locals, nil, nil)
 560
 561         // Ask should result in temporary connection refused error
 562         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 563         _, _, err = kc.Ask(hash)
 564         c.Check(err, NotNil)
 565         errNotFound, _ := err.(*keepclient.ErrNotFound)
 566         c.Check(errNotFound.Temporary(), Equals, true)
 567         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 568
 569         // Get should result in temporary connection refused error
 570         _, _, _, err = kc.Get(hash)
 571         c.Check(err, NotNil)
 572         errNotFound, _ = err.(*keepclient.ErrNotFound)
 573         c.Check(errNotFound.Temporary(), Equals, true)
 574         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 575 }
 576
 577 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 578         kc := runProxy(c, nil, false)
 579         defer closeListener()
 580
 581         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 582         for _, f := range []func() error{
 583                 func() error {
 584                         _, _, err := kc.Ask(hash)
 585                         return err
 586                 },
 587                 func() error {
 588                         _, _, _, err := kc.Get(hash)
 589                         return err
 590                 },
 591         } {
 592                 err := f()
 593                 c.Assert(err, NotNil)
 594                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 595                 c.Check(errNotFound.Temporary(), Equals, true)
 596                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 597         }
 598 }
 599
 600 func (s *ServerRequiredSuite) TestPing(c *C) {
 601         kc := runProxy(c, nil, false)
 602         defer closeListener()
 603
 604         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
 605
 606         req, err := http.NewRequest("GET",
 607                 "http://"+listener.Addr().String()+"/_health/ping",
 608                 nil)
 609         c.Assert(err, IsNil)
 610         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 611
 612         resp := httptest.NewRecorder()
 613         rtr.ServeHTTP(resp, req)
 614         c.Check(resp.Code, Equals, 200)
 615         c.Assert(strings.Contains(resp.Body.String(), `{"health":"OK"}`), Equals, true)
 616 }