services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "strings"
  17         "sync"
  18         "testing"
  19         "time"
  20
  21         "git.curoverse.com/arvados.git/lib/config"
  22         "git.curoverse.com/arvados.git/sdk/go/arvados"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  24         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  25         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  26         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  27         log "github.com/sirupsen/logrus"
  28
  29         . "gopkg.in/check.v1"
  30 )
  31
  32 // Gocheck boilerplate
  33 func Test(t *testing.T) {
  34         TestingT(t)
  35 }
  36
  37 // Gocheck boilerplate
  38 var _ = Suite(&ServerRequiredSuite{})
  39
  40 // Tests that require the Keep server running
  41 type ServerRequiredSuite struct{}
  42
  43 // Gocheck boilerplate
  44 var _ = Suite(&NoKeepServerSuite{})
  45
  46 // Test with no keepserver to simulate errors
  47 type NoKeepServerSuite struct{}
  48
  49 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  50
  51 // Wait (up to 1 second) for keepproxy to listen on a port. This
  52 // avoids a race condition where we hit a "connection refused" error
  53 // because we start testing the proxy too soon.
  54 func waitForListener() {
  55         const (
  56                 ms = 5
  57         )
  58         for i := 0; listener == nil && i < 10000; i += ms {
  59                 time.Sleep(ms * time.Millisecond)
  60         }
  61         if listener == nil {
  62                 panic("Timed out waiting for listener to start")
  63         }
  64 }
  65
  66 func closeListener() {
  67         if listener != nil {
  68                 listener.Close()
  69         }
  70 }
  71
  72 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  73         arvadostest.StartAPI()
  74         arvadostest.StartKeep(2, false)
  75 }
  76
  77 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  78         arvadostest.ResetEnv()
  79 }
  80
  81 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  82         arvadostest.StopKeep(2)
  83         arvadostest.StopAPI()
  84 }
  85
  86 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  87         arvadostest.StartAPI()
  88         // We need API to have some keep services listed, but the
  89         // services themselves should be unresponsive.
  90         arvadostest.StartKeep(2, false)
  91         arvadostest.StopKeep(2)
  92 }
  93
  94 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  95         arvadostest.ResetEnv()
  96 }
  97
  98 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  99         arvadostest.StopAPI()
 100 }
 101
 102 func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
 103         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
 104         c.Assert(err, Equals, nil)
 105         cluster, err := cfg.GetCluster("")
 106         c.Assert(err, Equals, nil)
 107
 108         cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
 109
 110         listener = nil
 111         go func() {
 112                 run(log.New(), cluster)
 113                 defer closeListener()
 114         }()
 115         waitForListener()
 116
 117         client := arvados.NewClientFromEnv()
 118         arv, err := arvadosclient.New(client)
 119         c.Assert(err, Equals, nil)
 120         if bogusClientToken {
 121                 arv.ApiToken = "bogus-token"
 122         }
 123         kc := keepclient.New(arv)
 124         sr := map[string]string{
 125                 TestProxyUUID: "http://" + listener.Addr().String(),
 126         }
 127         kc.SetServiceRoots(sr, sr, sr)
 128         kc.Arvados.External = true
 129
 130         return kc
 131 }
 132
 133 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 134         runProxy(c, false)
 135         defer closeListener()
 136
 137         req, err := http.NewRequest("POST",
 138                 "http://"+listener.Addr().String()+"/",
 139                 strings.NewReader("TestViaHeader"))
 140         c.Assert(err, Equals, nil)
 141         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 142         resp, err := (&http.Client{}).Do(req)
 143         c.Assert(err, Equals, nil)
 144         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 145         locator, err := ioutil.ReadAll(resp.Body)
 146         c.Assert(err, Equals, nil)
 147         resp.Body.Close()
 148
 149         req, err = http.NewRequest("GET",
 150                 "http://"+listener.Addr().String()+"/"+string(locator),
 151                 nil)
 152         c.Assert(err, Equals, nil)
 153         resp, err = (&http.Client{}).Do(req)
 154         c.Assert(err, Equals, nil)
 155         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 156         resp.Body.Close()
 157 }
 158
 159 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 160         kc := runProxy(c, false)
 161         defer closeListener()
 162
 163         sr := map[string]string{
 164                 TestProxyUUID: "http://" + listener.Addr().String(),
 165         }
 166         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 167
 168         content := []byte("TestLoopDetection")
 169         _, _, err := kc.PutB(content)
 170         c.Check(err, ErrorMatches, `.*loop detected.*`)
 171
 172         hash := fmt.Sprintf("%x", md5.Sum(content))
 173         _, _, _, err = kc.Get(hash)
 174         c.Check(err, ErrorMatches, `.*loop detected.*`)
 175 }
 176
 177 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 178         kc := runProxy(c, false)
 179         defer closeListener()
 180
 181         // Set up fake keepstore to record request headers
 182         var hdr http.Header
 183         ts := httptest.NewServer(http.HandlerFunc(
 184                 func(w http.ResponseWriter, r *http.Request) {
 185                         hdr = r.Header
 186                         http.Error(w, "Error", http.StatusInternalServerError)
 187                 }))
 188         defer ts.Close()
 189
 190         // Point keepproxy router's keepclient to the fake keepstore
 191         sr := map[string]string{
 192                 TestProxyUUID: ts.URL,
 193         }
 194         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 195
 196         // Set up client to ask for storage classes to keepproxy
 197         kc.StorageClasses = []string{"secure"}
 198         content := []byte("Very important data")
 199         _, _, err := kc.PutB(content)
 200         c.Check(err, NotNil)
 201         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 202 }
 203
 204 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 205         kc := runProxy(c, false)
 206         defer closeListener()
 207
 208         content := []byte("TestDesiredReplicas")
 209         hash := fmt.Sprintf("%x", md5.Sum(content))
 210
 211         for _, kc.Want_replicas = range []int{0, 1, 2} {
 212                 locator, rep, err := kc.PutB(content)
 213                 c.Check(err, Equals, nil)
 214                 c.Check(rep, Equals, kc.Want_replicas)
 215                 if rep > 0 {
 216                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 217                 }
 218         }
 219 }
 220
 221 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 222         kc := runProxy(c, false)
 223         defer closeListener()
 224
 225         content := []byte("TestPutWrongContentLength")
 226         hash := fmt.Sprintf("%x", md5.Sum(content))
 227
 228         // If we use http.Client to send these requests to the network
 229         // server we just started, the Go http library automatically
 230         // fixes the invalid Content-Length header. In order to test
 231         // our server behavior, we have to call the handler directly
 232         // using an httptest.ResponseRecorder.
 233         rtr := MakeRESTRouter(kc, 10*time.Second, "")
 234
 235         type testcase struct {
 236                 sendLength   string
 237                 expectStatus int
 238         }
 239
 240         for _, t := range []testcase{
 241                 {"1", http.StatusBadRequest},
 242                 {"", http.StatusLengthRequired},
 243                 {"-1", http.StatusLengthRequired},
 244                 {"abcdef", http.StatusLengthRequired},
 245         } {
 246                 req, err := http.NewRequest("PUT",
 247                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 248                         bytes.NewReader(content))
 249                 c.Assert(err, IsNil)
 250                 req.Header.Set("Content-Length", t.sendLength)
 251                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 252                 req.Header.Set("Content-Type", "application/octet-stream")
 253
 254                 resp := httptest.NewRecorder()
 255                 rtr.ServeHTTP(resp, req)
 256                 c.Check(resp.Code, Equals, t.expectStatus)
 257         }
 258 }
 259
 260 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 261         kc := runProxy(c, false)
 262         defer closeListener()
 263         router.(*proxyHandler).timeout = time.Nanosecond
 264
 265         buf := make([]byte, 1<<20)
 266         rand.Read(buf)
 267         var wg sync.WaitGroup
 268         for i := 0; i < 128; i++ {
 269                 wg.Add(1)
 270                 go func() {
 271                         defer wg.Done()
 272                         kc.PutB(buf)
 273                 }()
 274         }
 275         done := make(chan bool)
 276         go func() {
 277                 wg.Wait()
 278                 close(done)
 279         }()
 280         select {
 281         case <-done:
 282         case <-time.After(10 * time.Second):
 283                 c.Error("timeout")
 284         }
 285 }
 286
 287 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 288         kc := runProxy(c, false)
 289         defer closeListener()
 290
 291         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 292         var hash2 string
 293
 294         {
 295                 _, _, err := kc.Ask(hash)
 296                 c.Check(err, Equals, keepclient.BlockNotFound)
 297                 c.Log("Finished Ask (expected BlockNotFound)")
 298         }
 299
 300         {
 301                 reader, _, _, err := kc.Get(hash)
 302                 c.Check(reader, Equals, nil)
 303                 c.Check(err, Equals, keepclient.BlockNotFound)
 304                 c.Log("Finished Get (expected BlockNotFound)")
 305         }
 306
 307         // Note in bug #5309 among other errors keepproxy would set
 308         // Content-Length incorrectly on the 404 BlockNotFound response, this
 309         // would result in a protocol violation that would prevent reuse of the
 310         // connection, which would manifest by the next attempt to use the
 311         // connection (in this case the PutB below) failing.  So to test for
 312         // that bug it's necessary to trigger an error response (such as
 313         // BlockNotFound) and then do something else with the same httpClient
 314         // connection.
 315
 316         {
 317                 var rep int
 318                 var err error
 319                 hash2, rep, err = kc.PutB([]byte("foo"))
 320                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 321                 c.Check(rep, Equals, 2)
 322                 c.Check(err, Equals, nil)
 323                 c.Log("Finished PutB (expected success)")
 324         }
 325
 326         {
 327                 blocklen, _, err := kc.Ask(hash2)
 328                 c.Assert(err, Equals, nil)
 329                 c.Check(blocklen, Equals, int64(3))
 330                 c.Log("Finished Ask (expected success)")
 331         }
 332
 333         {
 334                 reader, blocklen, _, err := kc.Get(hash2)
 335                 c.Assert(err, Equals, nil)
 336                 all, err := ioutil.ReadAll(reader)
 337                 c.Check(err, IsNil)
 338                 c.Check(all, DeepEquals, []byte("foo"))
 339                 c.Check(blocklen, Equals, int64(3))
 340                 c.Log("Finished Get (expected success)")
 341         }
 342
 343         {
 344                 var rep int
 345                 var err error
 346                 hash2, rep, err = kc.PutB([]byte(""))
 347                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 348                 c.Check(rep, Equals, 2)
 349                 c.Check(err, Equals, nil)
 350                 c.Log("Finished PutB zero block")
 351         }
 352
 353         {
 354                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 355                 c.Assert(err, Equals, nil)
 356                 all, err := ioutil.ReadAll(reader)
 357                 c.Check(err, IsNil)
 358                 c.Check(all, DeepEquals, []byte(""))
 359                 c.Check(blocklen, Equals, int64(0))
 360                 c.Log("Finished Get zero block")
 361         }
 362 }
 363
 364 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 365         kc := runProxy(c, true)
 366         defer closeListener()
 367
 368         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 369
 370         _, _, err := kc.Ask(hash)
 371         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 372
 373         hash2, rep, err := kc.PutB([]byte("bar"))
 374         c.Check(hash2, Equals, "")
 375         c.Check(rep, Equals, 0)
 376         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 377
 378         blocklen, _, err := kc.Ask(hash)
 379         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 380         c.Check(err, ErrorMatches, ".*not found.*")
 381         c.Check(blocklen, Equals, int64(0))
 382
 383         _, blocklen, _, err = kc.Get(hash)
 384         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 385         c.Check(err, ErrorMatches, ".*not found.*")
 386         c.Check(blocklen, Equals, int64(0))
 387
 388 }
 389
 390 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 391         runProxy(c, false)
 392         defer closeListener()
 393
 394         {
 395                 client := http.Client{}
 396                 req, err := http.NewRequest("OPTIONS",
 397                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 398                         nil)
 399                 c.Assert(err, IsNil)
 400                 req.Header.Add("Access-Control-Request-Method", "PUT")
 401                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 402                 resp, err := client.Do(req)
 403                 c.Check(err, Equals, nil)
 404                 c.Check(resp.StatusCode, Equals, 200)
 405                 body, err := ioutil.ReadAll(resp.Body)
 406                 c.Check(err, IsNil)
 407                 c.Check(string(body), Equals, "")
 408                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 409                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 410         }
 411
 412         {
 413                 resp, err := http.Get(
 414                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 415                 c.Check(err, Equals, nil)
 416                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 417                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 418         }
 419 }
 420
 421 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 422         runProxy(c, false)
 423         defer closeListener()
 424
 425         {
 426                 client := http.Client{}
 427                 req, err := http.NewRequest("POST",
 428                         "http://"+listener.Addr().String()+"/",
 429                         strings.NewReader("qux"))
 430                 c.Check(err, IsNil)
 431                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 432                 req.Header.Add("Content-Type", "application/octet-stream")
 433                 resp, err := client.Do(req)
 434                 c.Check(err, Equals, nil)
 435                 body, err := ioutil.ReadAll(resp.Body)
 436                 c.Check(err, Equals, nil)
 437                 c.Check(string(body), Matches,
 438                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 439         }
 440 }
 441
 442 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 443         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 444                 Equals,
 445                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 446         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 447                 Equals,
 448                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 449         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 450                 Equals,
 451                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 452         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 453                 Equals,
 454                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 455
 456 }
 457
 458 // Test GetIndex
 459 //   Put one block, with 2 replicas
 460 //   With no prefix (expect the block locator, twice)
 461 //   With an existing prefix (expect the block locator, twice)
 462 //   With a valid but non-existing prefix (expect "\n")
 463 //   With an invalid prefix (expect error)
 464 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 465         kc := runProxy(c, false)
 466         defer closeListener()
 467
 468         // Put "index-data" blocks
 469         data := []byte("index-data")
 470         hash := fmt.Sprintf("%x", md5.Sum(data))
 471
 472         hash2, rep, err := kc.PutB(data)
 473         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 474         c.Check(rep, Equals, 2)
 475         c.Check(err, Equals, nil)
 476
 477         reader, blocklen, _, err := kc.Get(hash)
 478         c.Assert(err, IsNil)
 479         c.Check(blocklen, Equals, int64(10))
 480         all, err := ioutil.ReadAll(reader)
 481         c.Assert(err, IsNil)
 482         c.Check(all, DeepEquals, data)
 483
 484         // Put some more blocks
 485         _, _, err = kc.PutB([]byte("some-more-index-data"))
 486         c.Check(err, IsNil)
 487
 488         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 489
 490         // Invoke GetIndex
 491         for _, spec := range []struct {
 492                 prefix         string
 493                 expectTestHash bool
 494                 expectOther    bool
 495         }{
 496                 {"", true, true},         // with no prefix
 497                 {hash[:3], true, false},  // with matching prefix
 498                 {"abcdef", false, false}, // with no such prefix
 499         } {
 500                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 501                 c.Assert(err, Equals, nil)
 502                 indexResp, err := ioutil.ReadAll(indexReader)
 503                 c.Assert(err, Equals, nil)
 504                 locators := strings.Split(string(indexResp), "\n")
 505                 gotTestHash := 0
 506                 gotOther := 0
 507                 for _, locator := range locators {
 508                         if locator == "" {
 509                                 continue
 510                         }
 511                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 512                         if locator[:32] == hash {
 513                                 gotTestHash++
 514                         } else {
 515                                 gotOther++
 516                         }
 517                 }
 518                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 519                 c.Check(gotOther > 0, Equals, spec.expectOther)
 520         }
 521
 522         // GetIndex with invalid prefix
 523         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 524         c.Assert((err != nil), Equals, true)
 525 }
 526
 527 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 528         kc := runProxy(c, false)
 529         defer closeListener()
 530         hash, _, err := kc.PutB([]byte("shareddata"))
 531         c.Check(err, IsNil)
 532         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 533         rdr, _, _, err := kc.Get(hash)
 534         c.Assert(err, IsNil)
 535         data, err := ioutil.ReadAll(rdr)
 536         c.Check(err, IsNil)
 537         c.Check(data, DeepEquals, []byte("shareddata"))
 538 }
 539
 540 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 541         kc := runProxy(c, false)
 542         defer closeListener()
 543
 544         // Put a test block
 545         hash, rep, err := kc.PutB([]byte("foo"))
 546         c.Check(err, IsNil)
 547         c.Check(rep, Equals, 2)
 548
 549         for _, badToken := range []string{
 550                 "nosuchtoken",
 551                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 552         } {
 553                 kc.Arvados.ApiToken = badToken
 554
 555                 // Ask and Get will fail only if the upstream
 556                 // keepstore server checks for valid signatures.
 557                 // Without knowing the blob signing key, there is no
 558                 // way for keepproxy to know whether a given token is
 559                 // permitted to read a block.  So these tests fail:
 560                 if false {
 561                         _, _, err = kc.Ask(hash)
 562                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 563                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 564                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 565
 566                         _, _, _, err = kc.Get(hash)
 567                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 568                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 569                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 570                 }
 571
 572                 _, _, err = kc.PutB([]byte("foo"))
 573                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 574         }
 575 }
 576
 577 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 578         kc := runProxy(c, false)
 579         defer closeListener()
 580
 581         // Point keepproxy at a non-existent keepstore
 582         locals := map[string]string{
 583                 TestProxyUUID: "http://localhost:12345",
 584         }
 585         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
 586
 587         // Ask should result in temporary bad gateway error
 588         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 589         _, _, err := kc.Ask(hash)
 590         c.Check(err, NotNil)
 591         errNotFound, _ := err.(*keepclient.ErrNotFound)
 592         c.Check(errNotFound.Temporary(), Equals, true)
 593         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 594
 595         // Get should result in temporary bad gateway error
 596         _, _, _, err = kc.Get(hash)
 597         c.Check(err, NotNil)
 598         errNotFound, _ = err.(*keepclient.ErrNotFound)
 599         c.Check(errNotFound.Temporary(), Equals, true)
 600         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 601 }
 602
 603 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 604         kc := runProxy(c, false)
 605         defer closeListener()
 606
 607         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 608         for _, f := range []func() error{
 609                 func() error {
 610                         _, _, err := kc.Ask(hash)
 611                         return err
 612                 },
 613                 func() error {
 614                         _, _, _, err := kc.Get(hash)
 615                         return err
 616                 },
 617         } {
 618                 err := f()
 619                 c.Assert(err, NotNil)
 620                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 621                 c.Check(errNotFound.Temporary(), Equals, true)
 622                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 623         }
 624 }
 625
 626 func (s *ServerRequiredSuite) TestPing(c *C) {
 627         kc := runProxy(c, false)
 628         defer closeListener()
 629
 630         rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
 631
 632         req, err := http.NewRequest("GET",
 633                 "http://"+listener.Addr().String()+"/_health/ping",
 634                 nil)
 635         c.Assert(err, IsNil)
 636         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 637
 638         resp := httptest.NewRecorder()
 639         rtr.ServeHTTP(resp, req)
 640         c.Check(resp.Code, Equals, 200)
 641         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 642 }