services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "log"
  14         "math/rand"
  15         "net/http"
  16         "net/http/httptest"
  17         "os"
  18         "strings"
  19         "sync"
  20         "testing"
  21         "time"
  22
  23         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  24         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  25         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  26
  27         . "gopkg.in/check.v1"
  28 )
  29
  30 // Gocheck boilerplate
  31 func Test(t *testing.T) {
  32         TestingT(t)
  33 }
  34
  35 // Gocheck boilerplate
  36 var _ = Suite(&ServerRequiredSuite{})
  37
  38 // Tests that require the Keep server running
  39 type ServerRequiredSuite struct{}
  40
  41 // Gocheck boilerplate
  42 var _ = Suite(&NoKeepServerSuite{})
  43
  44 // Test with no keepserver to simulate errors
  45 type NoKeepServerSuite struct{}
  46
  47 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  48
  49 // Wait (up to 1 second) for keepproxy to listen on a port. This
  50 // avoids a race condition where we hit a "connection refused" error
  51 // because we start testing the proxy too soon.
  52 func waitForListener() {
  53         const (
  54                 ms = 5
  55         )
  56         for i := 0; listener == nil && i < 10000; i += ms {
  57                 time.Sleep(ms * time.Millisecond)
  58         }
  59         if listener == nil {
  60                 log.Fatalf("Timed out waiting for listener to start")
  61         }
  62 }
  63
  64 func closeListener() {
  65         if listener != nil {
  66                 listener.Close()
  67         }
  68 }
  69
  70 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  71         arvadostest.StartAPI()
  72         arvadostest.StartKeep(2, false)
  73 }
  74
  75 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  76         arvadostest.ResetEnv()
  77 }
  78
  79 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  80         arvadostest.StopKeep(2)
  81         arvadostest.StopAPI()
  82 }
  83
  84 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  85         arvadostest.StartAPI()
  86         // We need API to have some keep services listed, but the
  87         // services themselves should be unresponsive.
  88         arvadostest.StartKeep(2, false)
  89         arvadostest.StopKeep(2)
  90 }
  91
  92 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  93         arvadostest.ResetEnv()
  94 }
  95
  96 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  97         arvadostest.StopAPI()
  98 }
  99
 100 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
 101         args = append([]string{"keepproxy"}, args...)
 102         os.Args = append(args, "-listen=:0")
 103         listener = nil
 104         go main()
 105         waitForListener()
 106
 107         arv, err := arvadosclient.MakeArvadosClient()
 108         c.Assert(err, Equals, nil)
 109         if bogusClientToken {
 110                 arv.ApiToken = "bogus-token"
 111         }
 112         kc := keepclient.New(arv)
 113         sr := map[string]string{
 114                 TestProxyUUID: "http://" + listener.Addr().String(),
 115         }
 116         kc.SetServiceRoots(sr, sr, sr)
 117         kc.Arvados.External = true
 118
 119         return kc
 120 }
 121
 122 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 123         runProxy(c, nil, false)
 124         defer closeListener()
 125
 126         req, err := http.NewRequest("POST",
 127                 "http://"+listener.Addr().String()+"/",
 128                 strings.NewReader("TestViaHeader"))
 129         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 130         resp, err := (&http.Client{}).Do(req)
 131         c.Assert(err, Equals, nil)
 132         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 133         locator, err := ioutil.ReadAll(resp.Body)
 134         c.Assert(err, Equals, nil)
 135         resp.Body.Close()
 136
 137         req, err = http.NewRequest("GET",
 138                 "http://"+listener.Addr().String()+"/"+string(locator),
 139                 nil)
 140         c.Assert(err, Equals, nil)
 141         resp, err = (&http.Client{}).Do(req)
 142         c.Assert(err, Equals, nil)
 143         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 144         resp.Body.Close()
 145 }
 146
 147 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 148         kc := runProxy(c, nil, false)
 149         defer closeListener()
 150
 151         sr := map[string]string{
 152                 TestProxyUUID: "http://" + listener.Addr().String(),
 153         }
 154         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 155
 156         content := []byte("TestLoopDetection")
 157         _, _, err := kc.PutB(content)
 158         c.Check(err, ErrorMatches, `.*loop detected.*`)
 159
 160         hash := fmt.Sprintf("%x", md5.Sum(content))
 161         _, _, _, err = kc.Get(hash)
 162         c.Check(err, ErrorMatches, `.*loop detected.*`)
 163 }
 164
 165 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 166         kc := runProxy(c, nil, false)
 167         defer closeListener()
 168
 169         content := []byte("TestDesiredReplicas")
 170         hash := fmt.Sprintf("%x", md5.Sum(content))
 171
 172         for _, kc.Want_replicas = range []int{0, 1, 2} {
 173                 locator, rep, err := kc.PutB(content)
 174                 c.Check(err, Equals, nil)
 175                 c.Check(rep, Equals, kc.Want_replicas)
 176                 if rep > 0 {
 177                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 178                 }
 179         }
 180 }
 181
 182 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 183         kc := runProxy(c, nil, false)
 184         defer closeListener()
 185
 186         content := []byte("TestPutWrongContentLength")
 187         hash := fmt.Sprintf("%x", md5.Sum(content))
 188
 189         // If we use http.Client to send these requests to the network
 190         // server we just started, the Go http library automatically
 191         // fixes the invalid Content-Length header. In order to test
 192         // our server behavior, we have to call the handler directly
 193         // using an httptest.ResponseRecorder.
 194         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
 195
 196         type testcase struct {
 197                 sendLength   string
 198                 expectStatus int
 199         }
 200
 201         for _, t := range []testcase{
 202                 {"1", http.StatusBadRequest},
 203                 {"", http.StatusLengthRequired},
 204                 {"-1", http.StatusLengthRequired},
 205                 {"abcdef", http.StatusLengthRequired},
 206         } {
 207                 req, err := http.NewRequest("PUT",
 208                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 209                         bytes.NewReader(content))
 210                 c.Assert(err, IsNil)
 211                 req.Header.Set("Content-Length", t.sendLength)
 212                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 213                 req.Header.Set("Content-Type", "application/octet-stream")
 214
 215                 resp := httptest.NewRecorder()
 216                 rtr.ServeHTTP(resp, req)
 217                 c.Check(resp.Code, Equals, t.expectStatus)
 218         }
 219 }
 220
 221 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 222         kc := runProxy(c, nil, false)
 223         defer closeListener()
 224         router.(*proxyHandler).timeout = time.Nanosecond
 225
 226         buf := make([]byte, 1<<20)
 227         rand.Read(buf)
 228         var wg sync.WaitGroup
 229         for i := 0; i < 128; i++ {
 230                 wg.Add(1)
 231                 go func() {
 232                         defer wg.Done()
 233                         kc.PutB(buf)
 234                 }()
 235         }
 236         done := make(chan bool)
 237         go func() {
 238                 wg.Wait()
 239                 close(done)
 240         }()
 241         select {
 242         case <-done:
 243         case <-time.After(10 * time.Second):
 244                 c.Error("timeout")
 245         }
 246 }
 247
 248 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 249         kc := runProxy(c, nil, false)
 250         defer closeListener()
 251
 252         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 253         var hash2 string
 254
 255         {
 256                 _, _, err := kc.Ask(hash)
 257                 c.Check(err, Equals, keepclient.BlockNotFound)
 258                 log.Print("Finished Ask (expected BlockNotFound)")
 259         }
 260
 261         {
 262                 reader, _, _, err := kc.Get(hash)
 263                 c.Check(reader, Equals, nil)
 264                 c.Check(err, Equals, keepclient.BlockNotFound)
 265                 log.Print("Finished Get (expected BlockNotFound)")
 266         }
 267
 268         // Note in bug #5309 among other errors keepproxy would set
 269         // Content-Length incorrectly on the 404 BlockNotFound response, this
 270         // would result in a protocol violation that would prevent reuse of the
 271         // connection, which would manifest by the next attempt to use the
 272         // connection (in this case the PutB below) failing.  So to test for
 273         // that bug it's necessary to trigger an error response (such as
 274         // BlockNotFound) and then do something else with the same httpClient
 275         // connection.
 276
 277         {
 278                 var rep int
 279                 var err error
 280                 hash2, rep, err = kc.PutB([]byte("foo"))
 281                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 282                 c.Check(rep, Equals, 2)
 283                 c.Check(err, Equals, nil)
 284                 log.Print("Finished PutB (expected success)")
 285         }
 286
 287         {
 288                 blocklen, _, err := kc.Ask(hash2)
 289                 c.Assert(err, Equals, nil)
 290                 c.Check(blocklen, Equals, int64(3))
 291                 log.Print("Finished Ask (expected success)")
 292         }
 293
 294         {
 295                 reader, blocklen, _, err := kc.Get(hash2)
 296                 c.Assert(err, Equals, nil)
 297                 all, err := ioutil.ReadAll(reader)
 298                 c.Check(all, DeepEquals, []byte("foo"))
 299                 c.Check(blocklen, Equals, int64(3))
 300                 log.Print("Finished Get (expected success)")
 301         }
 302
 303         {
 304                 var rep int
 305                 var err error
 306                 hash2, rep, err = kc.PutB([]byte(""))
 307                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 308                 c.Check(rep, Equals, 2)
 309                 c.Check(err, Equals, nil)
 310                 log.Print("Finished PutB zero block")
 311         }
 312
 313         {
 314                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 315                 c.Assert(err, Equals, nil)
 316                 all, err := ioutil.ReadAll(reader)
 317                 c.Check(all, DeepEquals, []byte(""))
 318                 c.Check(blocklen, Equals, int64(0))
 319                 log.Print("Finished Get zero block")
 320         }
 321 }
 322
 323 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 324         kc := runProxy(c, nil, true)
 325         defer closeListener()
 326
 327         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 328
 329         {
 330                 _, _, err := kc.Ask(hash)
 331                 errNotFound, _ := err.(keepclient.ErrNotFound)
 332                 c.Check(errNotFound, NotNil)
 333                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 334                 log.Print("Ask 1")
 335         }
 336
 337         {
 338                 hash2, rep, err := kc.PutB([]byte("bar"))
 339                 c.Check(hash2, Equals, "")
 340                 c.Check(rep, Equals, 0)
 341                 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 342                 log.Print("PutB")
 343         }
 344
 345         {
 346                 blocklen, _, err := kc.Ask(hash)
 347                 errNotFound, _ := err.(keepclient.ErrNotFound)
 348                 c.Check(errNotFound, NotNil)
 349                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 350                 c.Check(blocklen, Equals, int64(0))
 351                 log.Print("Ask 2")
 352         }
 353
 354         {
 355                 _, blocklen, _, err := kc.Get(hash)
 356                 errNotFound, _ := err.(keepclient.ErrNotFound)
 357                 c.Check(errNotFound, NotNil)
 358                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 359                 c.Check(blocklen, Equals, int64(0))
 360                 log.Print("Get")
 361         }
 362 }
 363
 364 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 365         kc := runProxy(c, []string{"-no-get"}, false)
 366         defer closeListener()
 367
 368         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 369
 370         {
 371                 _, _, err := kc.Ask(hash)
 372                 errNotFound, _ := err.(keepclient.ErrNotFound)
 373                 c.Check(errNotFound, NotNil)
 374                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 375                 log.Print("Ask 1")
 376         }
 377
 378         {
 379                 hash2, rep, err := kc.PutB([]byte("baz"))
 380                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 381                 c.Check(rep, Equals, 2)
 382                 c.Check(err, Equals, nil)
 383                 log.Print("PutB")
 384         }
 385
 386         {
 387                 blocklen, _, err := kc.Ask(hash)
 388                 errNotFound, _ := err.(keepclient.ErrNotFound)
 389                 c.Check(errNotFound, NotNil)
 390                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 391                 c.Check(blocklen, Equals, int64(0))
 392                 log.Print("Ask 2")
 393         }
 394
 395         {
 396                 _, blocklen, _, err := kc.Get(hash)
 397                 errNotFound, _ := err.(keepclient.ErrNotFound)
 398                 c.Check(errNotFound, NotNil)
 399                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 400                 c.Check(blocklen, Equals, int64(0))
 401                 log.Print("Get")
 402         }
 403 }
 404
 405 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 406         kc := runProxy(c, []string{"-no-put"}, false)
 407         defer closeListener()
 408
 409         hash2, rep, err := kc.PutB([]byte("quux"))
 410         c.Check(hash2, Equals, "")
 411         c.Check(rep, Equals, 0)
 412         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 413 }
 414
 415 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 416         runProxy(c, nil, false)
 417         defer closeListener()
 418
 419         {
 420                 client := http.Client{}
 421                 req, err := http.NewRequest("OPTIONS",
 422                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 423                         nil)
 424                 req.Header.Add("Access-Control-Request-Method", "PUT")
 425                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 426                 resp, err := client.Do(req)
 427                 c.Check(err, Equals, nil)
 428                 c.Check(resp.StatusCode, Equals, 200)
 429                 body, err := ioutil.ReadAll(resp.Body)
 430                 c.Check(string(body), Equals, "")
 431                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 432                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 433         }
 434
 435         {
 436                 resp, err := http.Get(
 437                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 438                 c.Check(err, Equals, nil)
 439                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 440                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 441         }
 442 }
 443
 444 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 445         runProxy(c, nil, false)
 446         defer closeListener()
 447
 448         {
 449                 client := http.Client{}
 450                 req, err := http.NewRequest("POST",
 451                         "http://"+listener.Addr().String()+"/",
 452                         strings.NewReader("qux"))
 453                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 454                 req.Header.Add("Content-Type", "application/octet-stream")
 455                 resp, err := client.Do(req)
 456                 c.Check(err, Equals, nil)
 457                 body, err := ioutil.ReadAll(resp.Body)
 458                 c.Check(err, Equals, nil)
 459                 c.Check(string(body), Matches,
 460                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 461         }
 462 }
 463
 464 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 465         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 466                 Equals,
 467                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 468         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 469                 Equals,
 470                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 471         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 472                 Equals,
 473                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 474         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 475                 Equals,
 476                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 477
 478 }
 479
 480 // Test GetIndex
 481 //   Put one block, with 2 replicas
 482 //   With no prefix (expect the block locator, twice)
 483 //   With an existing prefix (expect the block locator, twice)
 484 //   With a valid but non-existing prefix (expect "\n")
 485 //   With an invalid prefix (expect error)
 486 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 487         kc := runProxy(c, nil, false)
 488         defer closeListener()
 489
 490         // Put "index-data" blocks
 491         data := []byte("index-data")
 492         hash := fmt.Sprintf("%x", md5.Sum(data))
 493
 494         hash2, rep, err := kc.PutB(data)
 495         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 496         c.Check(rep, Equals, 2)
 497         c.Check(err, Equals, nil)
 498
 499         reader, blocklen, _, err := kc.Get(hash)
 500         c.Assert(err, Equals, nil)
 501         c.Check(blocklen, Equals, int64(10))
 502         all, err := ioutil.ReadAll(reader)
 503         c.Check(all, DeepEquals, data)
 504
 505         // Put some more blocks
 506         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 507         c.Check(err, Equals, nil)
 508
 509         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 510
 511         // Invoke GetIndex
 512         for _, spec := range []struct {
 513                 prefix         string
 514                 expectTestHash bool
 515                 expectOther    bool
 516         }{
 517                 {"", true, true},         // with no prefix
 518                 {hash[:3], true, false},  // with matching prefix
 519                 {"abcdef", false, false}, // with no such prefix
 520         } {
 521                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 522                 c.Assert(err, Equals, nil)
 523                 indexResp, err := ioutil.ReadAll(indexReader)
 524                 c.Assert(err, Equals, nil)
 525                 locators := strings.Split(string(indexResp), "\n")
 526                 gotTestHash := 0
 527                 gotOther := 0
 528                 for _, locator := range locators {
 529                         if locator == "" {
 530                                 continue
 531                         }
 532                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 533                         if locator[:32] == hash {
 534                                 gotTestHash++
 535                         } else {
 536                                 gotOther++
 537                         }
 538                 }
 539                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 540                 c.Check(gotOther > 0, Equals, spec.expectOther)
 541         }
 542
 543         // GetIndex with invalid prefix
 544         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 545         c.Assert((err != nil), Equals, true)
 546 }
 547
 548 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 549         kc := runProxy(c, nil, false)
 550         defer closeListener()
 551
 552         // Put a test block
 553         hash, rep, err := kc.PutB([]byte("foo"))
 554         c.Check(err, Equals, nil)
 555         c.Check(rep, Equals, 2)
 556
 557         for _, token := range []string{
 558                 "nosuchtoken",
 559                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 560         } {
 561                 // Change token to given bad token
 562                 kc.Arvados.ApiToken = token
 563
 564                 // Ask should result in error
 565                 _, _, err = kc.Ask(hash)
 566                 c.Check(err, NotNil)
 567                 errNotFound, _ := err.(keepclient.ErrNotFound)
 568                 c.Check(errNotFound.Temporary(), Equals, false)
 569                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 570
 571                 // Get should result in error
 572                 _, _, _, err = kc.Get(hash)
 573                 c.Check(err, NotNil)
 574                 errNotFound, _ = err.(keepclient.ErrNotFound)
 575                 c.Check(errNotFound.Temporary(), Equals, false)
 576                 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
 577         }
 578 }
 579
 580 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 581         arv, err := arvadosclient.MakeArvadosClient()
 582         c.Assert(err, Equals, nil)
 583
 584         // keepclient with no such keep server
 585         kc := keepclient.New(arv)
 586         locals := map[string]string{
 587                 TestProxyUUID: "http://localhost:12345",
 588         }
 589         kc.SetServiceRoots(locals, nil, nil)
 590
 591         // Ask should result in temporary connection refused error
 592         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 593         _, _, err = kc.Ask(hash)
 594         c.Check(err, NotNil)
 595         errNotFound, _ := err.(*keepclient.ErrNotFound)
 596         c.Check(errNotFound.Temporary(), Equals, true)
 597         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 598
 599         // Get should result in temporary connection refused error
 600         _, _, _, err = kc.Get(hash)
 601         c.Check(err, NotNil)
 602         errNotFound, _ = err.(*keepclient.ErrNotFound)
 603         c.Check(errNotFound.Temporary(), Equals, true)
 604         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 605 }
 606
 607 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 608         kc := runProxy(c, nil, false)
 609         defer closeListener()
 610
 611         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 612         for _, f := range []func() error{
 613                 func() error {
 614                         _, _, err := kc.Ask(hash)
 615                         return err
 616                 },
 617                 func() error {
 618                         _, _, _, err := kc.Get(hash)
 619                         return err
 620                 },
 621         } {
 622                 err := f()
 623                 c.Assert(err, NotNil)
 624                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 625                 c.Check(errNotFound.Temporary(), Equals, true)
 626                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 627         }
 628 }
 629
 630 func (s *ServerRequiredSuite) TestPing(c *C) {
 631         kc := runProxy(c, nil, false)
 632         defer closeListener()
 633
 634         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
 635
 636         req, err := http.NewRequest("GET",
 637                 "http://"+listener.Addr().String()+"/_health/ping",
 638                 nil)
 639         c.Assert(err, IsNil)
 640         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 641
 642         resp := httptest.NewRecorder()
 643         rtr.ServeHTTP(resp, req)
 644         c.Check(resp.Code, Equals, 200)
 645         c.Assert(strings.Contains(resp.Body.String(), `{"health":"OK"}`), Equals, true)
 646 }