services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "os"
  17         "strings"
  18         "sync"
  19         "testing"
  20         "time"
  21
  22         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25
  26         . "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 func Test(t *testing.T) {
  31         TestingT(t)
  32 }
  33
  34 // Gocheck boilerplate
  35 var _ = Suite(&ServerRequiredSuite{})
  36
  37 // Tests that require the Keep server running
  38 type ServerRequiredSuite struct{}
  39
  40 // Gocheck boilerplate
  41 var _ = Suite(&NoKeepServerSuite{})
  42
  43 // Test with no keepserver to simulate errors
  44 type NoKeepServerSuite struct{}
  45
  46 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  47
  48 // Wait (up to 1 second) for keepproxy to listen on a port. This
  49 // avoids a race condition where we hit a "connection refused" error
  50 // because we start testing the proxy too soon.
  51 func waitForListener() {
  52         const (
  53                 ms = 5
  54         )
  55         for i := 0; listener == nil && i < 10000; i += ms {
  56                 time.Sleep(ms * time.Millisecond)
  57         }
  58         if listener == nil {
  59                 panic("Timed out waiting for listener to start")
  60         }
  61 }
  62
  63 func closeListener() {
  64         if listener != nil {
  65                 listener.Close()
  66         }
  67 }
  68
  69 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  70         arvadostest.StartAPI()
  71         arvadostest.StartKeep(2, false)
  72 }
  73
  74 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  75         arvadostest.ResetEnv()
  76 }
  77
  78 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  79         arvadostest.StopKeep(2)
  80         arvadostest.StopAPI()
  81 }
  82
  83 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  84         arvadostest.StartAPI()
  85         // We need API to have some keep services listed, but the
  86         // services themselves should be unresponsive.
  87         arvadostest.StartKeep(2, false)
  88         arvadostest.StopKeep(2)
  89 }
  90
  91 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  92         arvadostest.ResetEnv()
  93 }
  94
  95 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  96         arvadostest.StopAPI()
  97 }
  98
  99 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
 100         args = append([]string{"keepproxy"}, args...)
 101         os.Args = append(args, "-listen=:0")
 102         listener = nil
 103         go main()
 104         waitForListener()
 105
 106         arv, err := arvadosclient.MakeArvadosClient()
 107         c.Assert(err, Equals, nil)
 108         if bogusClientToken {
 109                 arv.ApiToken = "bogus-token"
 110         }
 111         kc := keepclient.New(arv)
 112         sr := map[string]string{
 113                 TestProxyUUID: "http://" + listener.Addr().String(),
 114         }
 115         kc.SetServiceRoots(sr, sr, sr)
 116         kc.Arvados.External = true
 117
 118         return kc
 119 }
 120
 121 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 122         runProxy(c, nil, false)
 123         defer closeListener()
 124
 125         req, err := http.NewRequest("POST",
 126                 "http://"+listener.Addr().String()+"/",
 127                 strings.NewReader("TestViaHeader"))
 128         c.Assert(err, Equals, nil)
 129         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 130         resp, err := (&http.Client{}).Do(req)
 131         c.Assert(err, Equals, nil)
 132         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 133         locator, err := ioutil.ReadAll(resp.Body)
 134         c.Assert(err, Equals, nil)
 135         resp.Body.Close()
 136
 137         req, err = http.NewRequest("GET",
 138                 "http://"+listener.Addr().String()+"/"+string(locator),
 139                 nil)
 140         c.Assert(err, Equals, nil)
 141         resp, err = (&http.Client{}).Do(req)
 142         c.Assert(err, Equals, nil)
 143         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 144         resp.Body.Close()
 145 }
 146
 147 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 148         kc := runProxy(c, nil, false)
 149         defer closeListener()
 150
 151         sr := map[string]string{
 152                 TestProxyUUID: "http://" + listener.Addr().String(),
 153         }
 154         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 155
 156         content := []byte("TestLoopDetection")
 157         _, _, err := kc.PutB(content)
 158         c.Check(err, ErrorMatches, `.*loop detected.*`)
 159
 160         hash := fmt.Sprintf("%x", md5.Sum(content))
 161         _, _, _, err = kc.Get(hash)
 162         c.Check(err, ErrorMatches, `.*loop detected.*`)
 163 }
 164
 165 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 166         kc := runProxy(c, nil, false)
 167         defer closeListener()
 168
 169         // Set up fake keepstore to record request headers
 170         var hdr http.Header
 171         ts := httptest.NewServer(http.HandlerFunc(
 172                 func(w http.ResponseWriter, r *http.Request) {
 173                         hdr = r.Header
 174                         http.Error(w, "Error", http.StatusInternalServerError)
 175                 }))
 176         defer ts.Close()
 177
 178         // Point keepproxy router's keepclient to the fake keepstore
 179         sr := map[string]string{
 180                 TestProxyUUID: ts.URL,
 181         }
 182         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 183
 184         // Set up client to ask for storage classes to keepproxy
 185         kc.StorageClasses = []string{"secure"}
 186         content := []byte("Very important data")
 187         _, _, err := kc.PutB(content)
 188         c.Check(err, NotNil)
 189         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 190 }
 191
 192 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 193         kc := runProxy(c, nil, false)
 194         defer closeListener()
 195
 196         content := []byte("TestDesiredReplicas")
 197         hash := fmt.Sprintf("%x", md5.Sum(content))
 198
 199         for _, kc.Want_replicas = range []int{0, 1, 2} {
 200                 locator, rep, err := kc.PutB(content)
 201                 c.Check(err, Equals, nil)
 202                 c.Check(rep, Equals, kc.Want_replicas)
 203                 if rep > 0 {
 204                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 205                 }
 206         }
 207 }
 208
 209 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 210         kc := runProxy(c, nil, false)
 211         defer closeListener()
 212
 213         content := []byte("TestPutWrongContentLength")
 214         hash := fmt.Sprintf("%x", md5.Sum(content))
 215
 216         // If we use http.Client to send these requests to the network
 217         // server we just started, the Go http library automatically
 218         // fixes the invalid Content-Length header. In order to test
 219         // our server behavior, we have to call the handler directly
 220         // using an httptest.ResponseRecorder.
 221         rtr := MakeRESTRouter(kc, 10*time.Second, "")
 222
 223         type testcase struct {
 224                 sendLength   string
 225                 expectStatus int
 226         }
 227
 228         for _, t := range []testcase{
 229                 {"1", http.StatusBadRequest},
 230                 {"", http.StatusLengthRequired},
 231                 {"-1", http.StatusLengthRequired},
 232                 {"abcdef", http.StatusLengthRequired},
 233         } {
 234                 req, err := http.NewRequest("PUT",
 235                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 236                         bytes.NewReader(content))
 237                 c.Assert(err, IsNil)
 238                 req.Header.Set("Content-Length", t.sendLength)
 239                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 240                 req.Header.Set("Content-Type", "application/octet-stream")
 241
 242                 resp := httptest.NewRecorder()
 243                 rtr.ServeHTTP(resp, req)
 244                 c.Check(resp.Code, Equals, t.expectStatus)
 245         }
 246 }
 247
 248 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 249         kc := runProxy(c, nil, false)
 250         defer closeListener()
 251         router.(*proxyHandler).timeout = time.Nanosecond
 252
 253         buf := make([]byte, 1<<20)
 254         rand.Read(buf)
 255         var wg sync.WaitGroup
 256         for i := 0; i < 128; i++ {
 257                 wg.Add(1)
 258                 go func() {
 259                         defer wg.Done()
 260                         kc.PutB(buf)
 261                 }()
 262         }
 263         done := make(chan bool)
 264         go func() {
 265                 wg.Wait()
 266                 close(done)
 267         }()
 268         select {
 269         case <-done:
 270         case <-time.After(10 * time.Second):
 271                 c.Error("timeout")
 272         }
 273 }
 274
 275 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 276         kc := runProxy(c, nil, false)
 277         defer closeListener()
 278
 279         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 280         var hash2 string
 281
 282         {
 283                 _, _, err := kc.Ask(hash)
 284                 c.Check(err, Equals, keepclient.BlockNotFound)
 285                 c.Log("Finished Ask (expected BlockNotFound)")
 286         }
 287
 288         {
 289                 reader, _, _, err := kc.Get(hash)
 290                 c.Check(reader, Equals, nil)
 291                 c.Check(err, Equals, keepclient.BlockNotFound)
 292                 c.Log("Finished Get (expected BlockNotFound)")
 293         }
 294
 295         // Note in bug #5309 among other errors keepproxy would set
 296         // Content-Length incorrectly on the 404 BlockNotFound response, this
 297         // would result in a protocol violation that would prevent reuse of the
 298         // connection, which would manifest by the next attempt to use the
 299         // connection (in this case the PutB below) failing.  So to test for
 300         // that bug it's necessary to trigger an error response (such as
 301         // BlockNotFound) and then do something else with the same httpClient
 302         // connection.
 303
 304         {
 305                 var rep int
 306                 var err error
 307                 hash2, rep, err = kc.PutB([]byte("foo"))
 308                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 309                 c.Check(rep, Equals, 2)
 310                 c.Check(err, Equals, nil)
 311                 c.Log("Finished PutB (expected success)")
 312         }
 313
 314         {
 315                 blocklen, _, err := kc.Ask(hash2)
 316                 c.Assert(err, Equals, nil)
 317                 c.Check(blocklen, Equals, int64(3))
 318                 c.Log("Finished Ask (expected success)")
 319         }
 320
 321         {
 322                 reader, blocklen, _, err := kc.Get(hash2)
 323                 c.Assert(err, Equals, nil)
 324                 all, err := ioutil.ReadAll(reader)
 325                 c.Check(err, IsNil)
 326                 c.Check(all, DeepEquals, []byte("foo"))
 327                 c.Check(blocklen, Equals, int64(3))
 328                 c.Log("Finished Get (expected success)")
 329         }
 330
 331         {
 332                 var rep int
 333                 var err error
 334                 hash2, rep, err = kc.PutB([]byte(""))
 335                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 336                 c.Check(rep, Equals, 2)
 337                 c.Check(err, Equals, nil)
 338                 c.Log("Finished PutB zero block")
 339         }
 340
 341         {
 342                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 343                 c.Assert(err, Equals, nil)
 344                 all, err := ioutil.ReadAll(reader)
 345                 c.Check(err, IsNil)
 346                 c.Check(all, DeepEquals, []byte(""))
 347                 c.Check(blocklen, Equals, int64(0))
 348                 c.Log("Finished Get zero block")
 349         }
 350 }
 351
 352 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 353         kc := runProxy(c, nil, true)
 354         defer closeListener()
 355
 356         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 357
 358         _, _, err := kc.Ask(hash)
 359         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 360
 361         hash2, rep, err := kc.PutB([]byte("bar"))
 362         c.Check(hash2, Equals, "")
 363         c.Check(rep, Equals, 0)
 364         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 365
 366         blocklen, _, err := kc.Ask(hash)
 367         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 368         c.Check(err, ErrorMatches, ".*not found.*")
 369         c.Check(blocklen, Equals, int64(0))
 370
 371         _, blocklen, _, err = kc.Get(hash)
 372         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 373         c.Check(err, ErrorMatches, ".*not found.*")
 374         c.Check(blocklen, Equals, int64(0))
 375
 376 }
 377
 378 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 379         runProxy(c, nil, false)
 380         defer closeListener()
 381
 382         {
 383                 client := http.Client{}
 384                 req, err := http.NewRequest("OPTIONS",
 385                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 386                         nil)
 387                 c.Assert(err, IsNil)
 388                 req.Header.Add("Access-Control-Request-Method", "PUT")
 389                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 390                 resp, err := client.Do(req)
 391                 c.Check(err, Equals, nil)
 392                 c.Check(resp.StatusCode, Equals, 200)
 393                 body, err := ioutil.ReadAll(resp.Body)
 394                 c.Check(err, IsNil)
 395                 c.Check(string(body), Equals, "")
 396                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 397                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 398         }
 399
 400         {
 401                 resp, err := http.Get(
 402                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 403                 c.Check(err, Equals, nil)
 404                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 405                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 406         }
 407 }
 408
 409 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 410         runProxy(c, nil, false)
 411         defer closeListener()
 412
 413         {
 414                 client := http.Client{}
 415                 req, err := http.NewRequest("POST",
 416                         "http://"+listener.Addr().String()+"/",
 417                         strings.NewReader("qux"))
 418                 c.Check(err, IsNil)
 419                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 420                 req.Header.Add("Content-Type", "application/octet-stream")
 421                 resp, err := client.Do(req)
 422                 c.Check(err, Equals, nil)
 423                 body, err := ioutil.ReadAll(resp.Body)
 424                 c.Check(err, Equals, nil)
 425                 c.Check(string(body), Matches,
 426                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 427         }
 428 }
 429
 430 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 431         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 432                 Equals,
 433                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 434         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 435                 Equals,
 436                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 437         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 438                 Equals,
 439                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 440         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 441                 Equals,
 442                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 443
 444 }
 445
 446 // Test GetIndex
 447 //   Put one block, with 2 replicas
 448 //   With no prefix (expect the block locator, twice)
 449 //   With an existing prefix (expect the block locator, twice)
 450 //   With a valid but non-existing prefix (expect "\n")
 451 //   With an invalid prefix (expect error)
 452 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 453         kc := runProxy(c, nil, false)
 454         defer closeListener()
 455
 456         // Put "index-data" blocks
 457         data := []byte("index-data")
 458         hash := fmt.Sprintf("%x", md5.Sum(data))
 459
 460         hash2, rep, err := kc.PutB(data)
 461         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 462         c.Check(rep, Equals, 2)
 463         c.Check(err, Equals, nil)
 464
 465         reader, blocklen, _, err := kc.Get(hash)
 466         c.Assert(err, IsNil)
 467         c.Check(blocklen, Equals, int64(10))
 468         all, err := ioutil.ReadAll(reader)
 469         c.Assert(err, IsNil)
 470         c.Check(all, DeepEquals, data)
 471
 472         // Put some more blocks
 473         _, _, err = kc.PutB([]byte("some-more-index-data"))
 474         c.Check(err, IsNil)
 475
 476         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 477
 478         // Invoke GetIndex
 479         for _, spec := range []struct {
 480                 prefix         string
 481                 expectTestHash bool
 482                 expectOther    bool
 483         }{
 484                 {"", true, true},         // with no prefix
 485                 {hash[:3], true, false},  // with matching prefix
 486                 {"abcdef", false, false}, // with no such prefix
 487         } {
 488                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 489                 c.Assert(err, Equals, nil)
 490                 indexResp, err := ioutil.ReadAll(indexReader)
 491                 c.Assert(err, Equals, nil)
 492                 locators := strings.Split(string(indexResp), "\n")
 493                 gotTestHash := 0
 494                 gotOther := 0
 495                 for _, locator := range locators {
 496                         if locator == "" {
 497                                 continue
 498                         }
 499                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 500                         if locator[:32] == hash {
 501                                 gotTestHash++
 502                         } else {
 503                                 gotOther++
 504                         }
 505                 }
 506                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 507                 c.Check(gotOther > 0, Equals, spec.expectOther)
 508         }
 509
 510         // GetIndex with invalid prefix
 511         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 512         c.Assert((err != nil), Equals, true)
 513 }
 514
 515 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 516         kc := runProxy(c, nil, false)
 517         defer closeListener()
 518         hash, _, err := kc.PutB([]byte("shareddata"))
 519         c.Check(err, IsNil)
 520         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 521         rdr, _, _, err := kc.Get(hash)
 522         c.Assert(err, IsNil)
 523         data, err := ioutil.ReadAll(rdr)
 524         c.Check(err, IsNil)
 525         c.Check(data, DeepEquals, []byte("shareddata"))
 526 }
 527
 528 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 529         kc := runProxy(c, nil, false)
 530         defer closeListener()
 531
 532         // Put a test block
 533         hash, rep, err := kc.PutB([]byte("foo"))
 534         c.Check(err, IsNil)
 535         c.Check(rep, Equals, 2)
 536
 537         for _, badToken := range []string{
 538                 "nosuchtoken",
 539                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 540         } {
 541                 kc.Arvados.ApiToken = badToken
 542
 543                 // Ask and Get will fail only if the upstream
 544                 // keepstore server checks for valid signatures.
 545                 // Without knowing the blob signing key, there is no
 546                 // way for keepproxy to know whether a given token is
 547                 // permitted to read a block.  So these tests fail:
 548                 if false {
 549                         _, _, err = kc.Ask(hash)
 550                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 551                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 552                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 553
 554                         _, _, _, err = kc.Get(hash)
 555                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 556                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 557                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 558                 }
 559
 560                 _, _, err = kc.PutB([]byte("foo"))
 561                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 562         }
 563 }
 564
 565 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 566         kc := runProxy(c, nil, false)
 567         defer closeListener()
 568
 569         // Point keepproxy at a non-existent keepstore
 570         locals := map[string]string{
 571                 TestProxyUUID: "http://localhost:12345",
 572         }
 573         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
 574
 575         // Ask should result in temporary bad gateway error
 576         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 577         _, _, err := kc.Ask(hash)
 578         c.Check(err, NotNil)
 579         errNotFound, _ := err.(*keepclient.ErrNotFound)
 580         c.Check(errNotFound.Temporary(), Equals, true)
 581         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 582
 583         // Get should result in temporary bad gateway error
 584         _, _, _, err = kc.Get(hash)
 585         c.Check(err, NotNil)
 586         errNotFound, _ = err.(*keepclient.ErrNotFound)
 587         c.Check(errNotFound.Temporary(), Equals, true)
 588         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 589 }
 590
 591 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 592         kc := runProxy(c, nil, false)
 593         defer closeListener()
 594
 595         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 596         for _, f := range []func() error{
 597                 func() error {
 598                         _, _, err := kc.Ask(hash)
 599                         return err
 600                 },
 601                 func() error {
 602                         _, _, _, err := kc.Get(hash)
 603                         return err
 604                 },
 605         } {
 606                 err := f()
 607                 c.Assert(err, NotNil)
 608                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 609                 c.Check(errNotFound.Temporary(), Equals, true)
 610                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 611         }
 612 }
 613
 614 func (s *ServerRequiredSuite) TestPing(c *C) {
 615         kc := runProxy(c, nil, false)
 616         defer closeListener()
 617
 618         rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
 619
 620         req, err := http.NewRequest("GET",
 621                 "http://"+listener.Addr().String()+"/_health/ping",
 622                 nil)
 623         c.Assert(err, IsNil)
 624         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 625
 626         resp := httptest.NewRecorder()
 627         rtr.ServeHTTP(resp, req)
 628         c.Check(resp.Code, Equals, 200)
 629         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 630 }