services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "os"
  17         "strings"
  18         "sync"
  19         "testing"
  20         "time"
  21
  22         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25
  26         . "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 func Test(t *testing.T) {
  31         TestingT(t)
  32 }
  33
  34 // Gocheck boilerplate
  35 var _ = Suite(&ServerRequiredSuite{})
  36
  37 // Tests that require the Keep server running
  38 type ServerRequiredSuite struct{}
  39
  40 // Gocheck boilerplate
  41 var _ = Suite(&NoKeepServerSuite{})
  42
  43 // Test with no keepserver to simulate errors
  44 type NoKeepServerSuite struct{}
  45
  46 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  47
  48 // Wait (up to 1 second) for keepproxy to listen on a port. This
  49 // avoids a race condition where we hit a "connection refused" error
  50 // because we start testing the proxy too soon.
  51 func waitForListener() {
  52         const (
  53                 ms = 5
  54         )
  55         for i := 0; listener == nil && i < 10000; i += ms {
  56                 time.Sleep(ms * time.Millisecond)
  57         }
  58         if listener == nil {
  59                 panic("Timed out waiting for listener to start")
  60         }
  61 }
  62
  63 func closeListener() {
  64         if listener != nil {
  65                 listener.Close()
  66         }
  67 }
  68
  69 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  70         arvadostest.StartAPI()
  71         arvadostest.StartKeep(2, false)
  72 }
  73
  74 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  75         arvadostest.ResetEnv()
  76 }
  77
  78 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  79         arvadostest.StopKeep(2)
  80         arvadostest.StopAPI()
  81 }
  82
  83 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  84         arvadostest.StartAPI()
  85         // We need API to have some keep services listed, but the
  86         // services themselves should be unresponsive.
  87         arvadostest.StartKeep(2, false)
  88         arvadostest.StopKeep(2)
  89 }
  90
  91 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  92         arvadostest.ResetEnv()
  93 }
  94
  95 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  96         arvadostest.StopAPI()
  97 }
  98
  99 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
 100         args = append([]string{"keepproxy"}, args...)
 101         os.Args = append(args, "-listen=:0")
 102         listener = nil
 103         go main()
 104         waitForListener()
 105
 106         arv, err := arvadosclient.MakeArvadosClient()
 107         c.Assert(err, Equals, nil)
 108         if bogusClientToken {
 109                 arv.ApiToken = "bogus-token"
 110         }
 111         kc := keepclient.New(arv)
 112         sr := map[string]string{
 113                 TestProxyUUID: "http://" + listener.Addr().String(),
 114         }
 115         kc.SetServiceRoots(sr, sr, sr)
 116         kc.Arvados.External = true
 117
 118         return kc
 119 }
 120
 121 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 122         runProxy(c, nil, false)
 123         defer closeListener()
 124
 125         req, err := http.NewRequest("POST",
 126                 "http://"+listener.Addr().String()+"/",
 127                 strings.NewReader("TestViaHeader"))
 128         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 129         resp, err := (&http.Client{}).Do(req)
 130         c.Assert(err, Equals, nil)
 131         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 132         locator, err := ioutil.ReadAll(resp.Body)
 133         c.Assert(err, Equals, nil)
 134         resp.Body.Close()
 135
 136         req, err = http.NewRequest("GET",
 137                 "http://"+listener.Addr().String()+"/"+string(locator),
 138                 nil)
 139         c.Assert(err, Equals, nil)
 140         resp, err = (&http.Client{}).Do(req)
 141         c.Assert(err, Equals, nil)
 142         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 143         resp.Body.Close()
 144 }
 145
 146 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 147         kc := runProxy(c, nil, false)
 148         defer closeListener()
 149
 150         sr := map[string]string{
 151                 TestProxyUUID: "http://" + listener.Addr().String(),
 152         }
 153         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 154
 155         content := []byte("TestLoopDetection")
 156         _, _, err := kc.PutB(content)
 157         c.Check(err, ErrorMatches, `.*loop detected.*`)
 158
 159         hash := fmt.Sprintf("%x", md5.Sum(content))
 160         _, _, _, err = kc.Get(hash)
 161         c.Check(err, ErrorMatches, `.*loop detected.*`)
 162 }
 163
 164 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 165         kc := runProxy(c, nil, false)
 166         defer closeListener()
 167
 168         content := []byte("TestDesiredReplicas")
 169         hash := fmt.Sprintf("%x", md5.Sum(content))
 170
 171         for _, kc.Want_replicas = range []int{0, 1, 2} {
 172                 locator, rep, err := kc.PutB(content)
 173                 c.Check(err, Equals, nil)
 174                 c.Check(rep, Equals, kc.Want_replicas)
 175                 if rep > 0 {
 176                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 177                 }
 178         }
 179 }
 180
 181 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 182         kc := runProxy(c, nil, false)
 183         defer closeListener()
 184
 185         content := []byte("TestPutWrongContentLength")
 186         hash := fmt.Sprintf("%x", md5.Sum(content))
 187
 188         // If we use http.Client to send these requests to the network
 189         // server we just started, the Go http library automatically
 190         // fixes the invalid Content-Length header. In order to test
 191         // our server behavior, we have to call the handler directly
 192         // using an httptest.ResponseRecorder.
 193         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
 194
 195         type testcase struct {
 196                 sendLength   string
 197                 expectStatus int
 198         }
 199
 200         for _, t := range []testcase{
 201                 {"1", http.StatusBadRequest},
 202                 {"", http.StatusLengthRequired},
 203                 {"-1", http.StatusLengthRequired},
 204                 {"abcdef", http.StatusLengthRequired},
 205         } {
 206                 req, err := http.NewRequest("PUT",
 207                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 208                         bytes.NewReader(content))
 209                 c.Assert(err, IsNil)
 210                 req.Header.Set("Content-Length", t.sendLength)
 211                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 212                 req.Header.Set("Content-Type", "application/octet-stream")
 213
 214                 resp := httptest.NewRecorder()
 215                 rtr.ServeHTTP(resp, req)
 216                 c.Check(resp.Code, Equals, t.expectStatus)
 217         }
 218 }
 219
 220 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 221         kc := runProxy(c, nil, false)
 222         defer closeListener()
 223         router.(*proxyHandler).timeout = time.Nanosecond
 224
 225         buf := make([]byte, 1<<20)
 226         rand.Read(buf)
 227         var wg sync.WaitGroup
 228         for i := 0; i < 128; i++ {
 229                 wg.Add(1)
 230                 go func() {
 231                         defer wg.Done()
 232                         kc.PutB(buf)
 233                 }()
 234         }
 235         done := make(chan bool)
 236         go func() {
 237                 wg.Wait()
 238                 close(done)
 239         }()
 240         select {
 241         case <-done:
 242         case <-time.After(10 * time.Second):
 243                 c.Error("timeout")
 244         }
 245 }
 246
 247 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 248         kc := runProxy(c, nil, false)
 249         defer closeListener()
 250
 251         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 252         var hash2 string
 253
 254         {
 255                 _, _, err := kc.Ask(hash)
 256                 c.Check(err, Equals, keepclient.BlockNotFound)
 257                 c.Log("Finished Ask (expected BlockNotFound)")
 258         }
 259
 260         {
 261                 reader, _, _, err := kc.Get(hash)
 262                 c.Check(reader, Equals, nil)
 263                 c.Check(err, Equals, keepclient.BlockNotFound)
 264                 c.Log("Finished Get (expected BlockNotFound)")
 265         }
 266
 267         // Note in bug #5309 among other errors keepproxy would set
 268         // Content-Length incorrectly on the 404 BlockNotFound response, this
 269         // would result in a protocol violation that would prevent reuse of the
 270         // connection, which would manifest by the next attempt to use the
 271         // connection (in this case the PutB below) failing.  So to test for
 272         // that bug it's necessary to trigger an error response (such as
 273         // BlockNotFound) and then do something else with the same httpClient
 274         // connection.
 275
 276         {
 277                 var rep int
 278                 var err error
 279                 hash2, rep, err = kc.PutB([]byte("foo"))
 280                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 281                 c.Check(rep, Equals, 2)
 282                 c.Check(err, Equals, nil)
 283                 c.Log("Finished PutB (expected success)")
 284         }
 285
 286         {
 287                 blocklen, _, err := kc.Ask(hash2)
 288                 c.Assert(err, Equals, nil)
 289                 c.Check(blocklen, Equals, int64(3))
 290                 c.Log("Finished Ask (expected success)")
 291         }
 292
 293         {
 294                 reader, blocklen, _, err := kc.Get(hash2)
 295                 c.Assert(err, Equals, nil)
 296                 all, err := ioutil.ReadAll(reader)
 297                 c.Check(all, DeepEquals, []byte("foo"))
 298                 c.Check(blocklen, Equals, int64(3))
 299                 c.Log("Finished Get (expected success)")
 300         }
 301
 302         {
 303                 var rep int
 304                 var err error
 305                 hash2, rep, err = kc.PutB([]byte(""))
 306                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 307                 c.Check(rep, Equals, 2)
 308                 c.Check(err, Equals, nil)
 309                 c.Log("Finished PutB zero block")
 310         }
 311
 312         {
 313                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 314                 c.Assert(err, Equals, nil)
 315                 all, err := ioutil.ReadAll(reader)
 316                 c.Check(all, DeepEquals, []byte(""))
 317                 c.Check(blocklen, Equals, int64(0))
 318                 c.Log("Finished Get zero block")
 319         }
 320 }
 321
 322 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 323         kc := runProxy(c, nil, true)
 324         defer closeListener()
 325
 326         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 327
 328         {
 329                 _, _, err := kc.Ask(hash)
 330                 errNotFound, _ := err.(keepclient.ErrNotFound)
 331                 c.Check(errNotFound, NotNil)
 332                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 333                 c.Log("Ask 1")
 334         }
 335
 336         {
 337                 hash2, rep, err := kc.PutB([]byte("bar"))
 338                 c.Check(hash2, Equals, "")
 339                 c.Check(rep, Equals, 0)
 340                 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 341                 c.Log("PutB")
 342         }
 343
 344         {
 345                 blocklen, _, err := kc.Ask(hash)
 346                 errNotFound, _ := err.(keepclient.ErrNotFound)
 347                 c.Check(errNotFound, NotNil)
 348                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 349                 c.Check(blocklen, Equals, int64(0))
 350                 c.Log("Ask 2")
 351         }
 352
 353         {
 354                 _, blocklen, _, err := kc.Get(hash)
 355                 errNotFound, _ := err.(keepclient.ErrNotFound)
 356                 c.Check(errNotFound, NotNil)
 357                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 358                 c.Check(blocklen, Equals, int64(0))
 359                 c.Log("Get")
 360         }
 361 }
 362
 363 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 364         kc := runProxy(c, []string{"-no-get"}, false)
 365         defer closeListener()
 366
 367         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 368
 369         {
 370                 _, _, err := kc.Ask(hash)
 371                 errNotFound, _ := err.(keepclient.ErrNotFound)
 372                 c.Check(errNotFound, NotNil)
 373                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 374                 c.Log("Ask 1")
 375         }
 376
 377         {
 378                 hash2, rep, err := kc.PutB([]byte("baz"))
 379                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 380                 c.Check(rep, Equals, 2)
 381                 c.Check(err, Equals, nil)
 382                 c.Log("PutB")
 383         }
 384
 385         {
 386                 blocklen, _, err := kc.Ask(hash)
 387                 errNotFound, _ := err.(keepclient.ErrNotFound)
 388                 c.Check(errNotFound, NotNil)
 389                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 390                 c.Check(blocklen, Equals, int64(0))
 391                 c.Log("Ask 2")
 392         }
 393
 394         {
 395                 _, blocklen, _, err := kc.Get(hash)
 396                 errNotFound, _ := err.(keepclient.ErrNotFound)
 397                 c.Check(errNotFound, NotNil)
 398                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 399                 c.Check(blocklen, Equals, int64(0))
 400                 c.Log("Get")
 401         }
 402 }
 403
 404 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 405         kc := runProxy(c, []string{"-no-put"}, false)
 406         defer closeListener()
 407
 408         hash2, rep, err := kc.PutB([]byte("quux"))
 409         c.Check(hash2, Equals, "")
 410         c.Check(rep, Equals, 0)
 411         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 412 }
 413
 414 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 415         runProxy(c, nil, false)
 416         defer closeListener()
 417
 418         {
 419                 client := http.Client{}
 420                 req, err := http.NewRequest("OPTIONS",
 421                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 422                         nil)
 423                 req.Header.Add("Access-Control-Request-Method", "PUT")
 424                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 425                 resp, err := client.Do(req)
 426                 c.Check(err, Equals, nil)
 427                 c.Check(resp.StatusCode, Equals, 200)
 428                 body, err := ioutil.ReadAll(resp.Body)
 429                 c.Check(string(body), Equals, "")
 430                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 431                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 432         }
 433
 434         {
 435                 resp, err := http.Get(
 436                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 437                 c.Check(err, Equals, nil)
 438                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 439                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 440         }
 441 }
 442
 443 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 444         runProxy(c, nil, false)
 445         defer closeListener()
 446
 447         {
 448                 client := http.Client{}
 449                 req, err := http.NewRequest("POST",
 450                         "http://"+listener.Addr().String()+"/",
 451                         strings.NewReader("qux"))
 452                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 453                 req.Header.Add("Content-Type", "application/octet-stream")
 454                 resp, err := client.Do(req)
 455                 c.Check(err, Equals, nil)
 456                 body, err := ioutil.ReadAll(resp.Body)
 457                 c.Check(err, Equals, nil)
 458                 c.Check(string(body), Matches,
 459                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 460         }
 461 }
 462
 463 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 464         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 465                 Equals,
 466                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 467         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 468                 Equals,
 469                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 470         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 471                 Equals,
 472                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 473         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 474                 Equals,
 475                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 476
 477 }
 478
 479 // Test GetIndex
 480 //   Put one block, with 2 replicas
 481 //   With no prefix (expect the block locator, twice)
 482 //   With an existing prefix (expect the block locator, twice)
 483 //   With a valid but non-existing prefix (expect "\n")
 484 //   With an invalid prefix (expect error)
 485 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 486         kc := runProxy(c, nil, false)
 487         defer closeListener()
 488
 489         // Put "index-data" blocks
 490         data := []byte("index-data")
 491         hash := fmt.Sprintf("%x", md5.Sum(data))
 492
 493         hash2, rep, err := kc.PutB(data)
 494         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 495         c.Check(rep, Equals, 2)
 496         c.Check(err, Equals, nil)
 497
 498         reader, blocklen, _, err := kc.Get(hash)
 499         c.Assert(err, Equals, nil)
 500         c.Check(blocklen, Equals, int64(10))
 501         all, err := ioutil.ReadAll(reader)
 502         c.Check(all, DeepEquals, data)
 503
 504         // Put some more blocks
 505         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 506         c.Check(err, Equals, nil)
 507
 508         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 509
 510         // Invoke GetIndex
 511         for _, spec := range []struct {
 512                 prefix         string
 513                 expectTestHash bool
 514                 expectOther    bool
 515         }{
 516                 {"", true, true},         // with no prefix
 517                 {hash[:3], true, false},  // with matching prefix
 518                 {"abcdef", false, false}, // with no such prefix
 519         } {
 520                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 521                 c.Assert(err, Equals, nil)
 522                 indexResp, err := ioutil.ReadAll(indexReader)
 523                 c.Assert(err, Equals, nil)
 524                 locators := strings.Split(string(indexResp), "\n")
 525                 gotTestHash := 0
 526                 gotOther := 0
 527                 for _, locator := range locators {
 528                         if locator == "" {
 529                                 continue
 530                         }
 531                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 532                         if locator[:32] == hash {
 533                                 gotTestHash++
 534                         } else {
 535                                 gotOther++
 536                         }
 537                 }
 538                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 539                 c.Check(gotOther > 0, Equals, spec.expectOther)
 540         }
 541
 542         // GetIndex with invalid prefix
 543         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 544         c.Assert((err != nil), Equals, true)
 545 }
 546
 547 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 548         kc := runProxy(c, nil, false)
 549         defer closeListener()
 550
 551         // Put a test block
 552         hash, rep, err := kc.PutB([]byte("foo"))
 553         c.Check(err, Equals, nil)
 554         c.Check(rep, Equals, 2)
 555
 556         for _, token := range []string{
 557                 "nosuchtoken",
 558                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 559         } {
 560                 // Change token to given bad token
 561                 kc.Arvados.ApiToken = token
 562
 563                 // Ask should result in error
 564                 _, _, err = kc.Ask(hash)
 565                 c.Check(err, NotNil)
 566                 errNotFound, _ := err.(keepclient.ErrNotFound)
 567                 c.Check(errNotFound.Temporary(), Equals, false)
 568                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 569
 570                 // Get should result in error
 571                 _, _, _, err = kc.Get(hash)
 572                 c.Check(err, NotNil)
 573                 errNotFound, _ = err.(keepclient.ErrNotFound)
 574                 c.Check(errNotFound.Temporary(), Equals, false)
 575                 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
 576         }
 577 }
 578
 579 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 580         arv, err := arvadosclient.MakeArvadosClient()
 581         c.Assert(err, Equals, nil)
 582
 583         // keepclient with no such keep server
 584         kc := keepclient.New(arv)
 585         locals := map[string]string{
 586                 TestProxyUUID: "http://localhost:12345",
 587         }
 588         kc.SetServiceRoots(locals, nil, nil)
 589
 590         // Ask should result in temporary connection refused error
 591         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 592         _, _, err = kc.Ask(hash)
 593         c.Check(err, NotNil)
 594         errNotFound, _ := err.(*keepclient.ErrNotFound)
 595         c.Check(errNotFound.Temporary(), Equals, true)
 596         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 597
 598         // Get should result in temporary connection refused error
 599         _, _, _, err = kc.Get(hash)
 600         c.Check(err, NotNil)
 601         errNotFound, _ = err.(*keepclient.ErrNotFound)
 602         c.Check(errNotFound.Temporary(), Equals, true)
 603         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 604 }
 605
 606 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 607         kc := runProxy(c, nil, false)
 608         defer closeListener()
 609
 610         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 611         for _, f := range []func() error{
 612                 func() error {
 613                         _, _, err := kc.Ask(hash)
 614                         return err
 615                 },
 616                 func() error {
 617                         _, _, _, err := kc.Get(hash)
 618                         return err
 619                 },
 620         } {
 621                 err := f()
 622                 c.Assert(err, NotNil)
 623                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 624                 c.Check(errNotFound.Temporary(), Equals, true)
 625                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 626         }
 627 }
 628
 629 func (s *ServerRequiredSuite) TestPing(c *C) {
 630         kc := runProxy(c, nil, false)
 631         defer closeListener()
 632
 633         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
 634
 635         req, err := http.NewRequest("GET",
 636                 "http://"+listener.Addr().String()+"/_health/ping",
 637                 nil)
 638         c.Assert(err, IsNil)
 639         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 640
 641         resp := httptest.NewRecorder()
 642         rtr.ServeHTTP(resp, req)
 643         c.Check(resp.Code, Equals, 200)
 644         c.Assert(strings.Contains(resp.Body.String(), `{"health":"OK"}`), Equals, true)
 645 }