services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "strings"
  17         "sync"
  18         "testing"
  19         "time"
  20
  21         "git.arvados.org/arvados.git/lib/config"
  22         "git.arvados.org/arvados.git/sdk/go/arvados"
  23         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  24         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         log "github.com/sirupsen/logrus"
  28
  29         . "gopkg.in/check.v1"
  30 )
  31
  32 // Gocheck boilerplate
  33 func Test(t *testing.T) {
  34         TestingT(t)
  35 }
  36
  37 // Gocheck boilerplate
  38 var _ = Suite(&ServerRequiredSuite{})
  39
  40 // Tests that require the Keep server running
  41 type ServerRequiredSuite struct{}
  42
  43 // Gocheck boilerplate
  44 var _ = Suite(&NoKeepServerSuite{})
  45
  46 // Test with no keepserver to simulate errors
  47 type NoKeepServerSuite struct{}
  48
  49 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  50
  51 // Wait (up to 1 second) for keepproxy to listen on a port. This
  52 // avoids a race condition where we hit a "connection refused" error
  53 // because we start testing the proxy too soon.
  54 func waitForListener() {
  55         const (
  56                 ms = 5
  57         )
  58         for i := 0; listener == nil && i < 10000; i += ms {
  59                 time.Sleep(ms * time.Millisecond)
  60         }
  61         if listener == nil {
  62                 panic("Timed out waiting for listener to start")
  63         }
  64 }
  65
  66 func closeListener() {
  67         if listener != nil {
  68                 listener.Close()
  69         }
  70 }
  71
  72 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  73         arvadostest.StartAPI()
  74         arvadostest.StartKeep(2, false)
  75 }
  76
  77 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  78         arvadostest.ResetEnv()
  79 }
  80
  81 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  82         arvadostest.StopKeep(2)
  83         arvadostest.StopAPI()
  84 }
  85
  86 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  87         arvadostest.StartAPI()
  88         // We need API to have some keep services listed, but the
  89         // services themselves should be unresponsive.
  90         arvadostest.StartKeep(2, false)
  91         arvadostest.StopKeep(2)
  92 }
  93
  94 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  95         arvadostest.ResetEnv()
  96 }
  97
  98 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  99         arvadostest.StopAPI()
 100 }
 101
 102 func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
 103         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
 104         c.Assert(err, Equals, nil)
 105         cluster, err := cfg.GetCluster("")
 106         c.Assert(err, Equals, nil)
 107
 108         cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
 109
 110         listener = nil
 111         go func() {
 112                 run(log.New(), cluster)
 113                 defer closeListener()
 114         }()
 115         waitForListener()
 116
 117         client := arvados.NewClientFromEnv()
 118         arv, err := arvadosclient.New(client)
 119         c.Assert(err, Equals, nil)
 120         if bogusClientToken {
 121                 arv.ApiToken = "bogus-token"
 122         }
 123         kc := keepclient.New(arv)
 124         sr := map[string]string{
 125                 TestProxyUUID: "http://" + listener.Addr().String(),
 126         }
 127         kc.SetServiceRoots(sr, sr, sr)
 128         kc.Arvados.External = true
 129
 130         return kc
 131 }
 132
 133 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 134         runProxy(c, false)
 135         defer closeListener()
 136
 137         req, err := http.NewRequest("POST",
 138                 "http://"+listener.Addr().String()+"/",
 139                 strings.NewReader("TestViaHeader"))
 140         c.Assert(err, Equals, nil)
 141         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 142         resp, err := (&http.Client{}).Do(req)
 143         c.Assert(err, Equals, nil)
 144         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 145         c.Assert(resp.StatusCode, Equals, http.StatusOK)
 146         locator, err := ioutil.ReadAll(resp.Body)
 147         c.Assert(err, Equals, nil)
 148         resp.Body.Close()
 149
 150         req, err = http.NewRequest("GET",
 151                 "http://"+listener.Addr().String()+"/"+string(locator),
 152                 nil)
 153         c.Assert(err, Equals, nil)
 154         resp, err = (&http.Client{}).Do(req)
 155         c.Assert(err, Equals, nil)
 156         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 157         resp.Body.Close()
 158 }
 159
 160 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 161         kc := runProxy(c, false)
 162         defer closeListener()
 163
 164         sr := map[string]string{
 165                 TestProxyUUID: "http://" + listener.Addr().String(),
 166         }
 167         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 168
 169         content := []byte("TestLoopDetection")
 170         _, _, err := kc.PutB(content)
 171         c.Check(err, ErrorMatches, `.*loop detected.*`)
 172
 173         hash := fmt.Sprintf("%x", md5.Sum(content))
 174         _, _, _, err = kc.Get(hash)
 175         c.Check(err, ErrorMatches, `.*loop detected.*`)
 176 }
 177
 178 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 179         kc := runProxy(c, false)
 180         defer closeListener()
 181
 182         // Set up fake keepstore to record request headers
 183         var hdr http.Header
 184         ts := httptest.NewServer(http.HandlerFunc(
 185                 func(w http.ResponseWriter, r *http.Request) {
 186                         hdr = r.Header
 187                         http.Error(w, "Error", http.StatusInternalServerError)
 188                 }))
 189         defer ts.Close()
 190
 191         // Point keepproxy router's keepclient to the fake keepstore
 192         sr := map[string]string{
 193                 TestProxyUUID: ts.URL,
 194         }
 195         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 196
 197         // Set up client to ask for storage classes to keepproxy
 198         kc.StorageClasses = []string{"secure"}
 199         content := []byte("Very important data")
 200         _, _, err := kc.PutB(content)
 201         c.Check(err, NotNil)
 202         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 203 }
 204
 205 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 206         kc := runProxy(c, false)
 207         defer closeListener()
 208
 209         content := []byte("TestDesiredReplicas")
 210         hash := fmt.Sprintf("%x", md5.Sum(content))
 211
 212         for _, kc.Want_replicas = range []int{0, 1, 2} {
 213                 locator, rep, err := kc.PutB(content)
 214                 c.Check(err, Equals, nil)
 215                 c.Check(rep, Equals, kc.Want_replicas)
 216                 if rep > 0 {
 217                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 218                 }
 219         }
 220 }
 221
 222 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 223         kc := runProxy(c, false)
 224         defer closeListener()
 225
 226         content := []byte("TestPutWrongContentLength")
 227         hash := fmt.Sprintf("%x", md5.Sum(content))
 228
 229         // If we use http.Client to send these requests to the network
 230         // server we just started, the Go http library automatically
 231         // fixes the invalid Content-Length header. In order to test
 232         // our server behavior, we have to call the handler directly
 233         // using an httptest.ResponseRecorder.
 234         rtr := MakeRESTRouter(kc, 10*time.Second, "")
 235
 236         type testcase struct {
 237                 sendLength   string
 238                 expectStatus int
 239         }
 240
 241         for _, t := range []testcase{
 242                 {"1", http.StatusBadRequest},
 243                 {"", http.StatusLengthRequired},
 244                 {"-1", http.StatusLengthRequired},
 245                 {"abcdef", http.StatusLengthRequired},
 246         } {
 247                 req, err := http.NewRequest("PUT",
 248                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 249                         bytes.NewReader(content))
 250                 c.Assert(err, IsNil)
 251                 req.Header.Set("Content-Length", t.sendLength)
 252                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 253                 req.Header.Set("Content-Type", "application/octet-stream")
 254
 255                 resp := httptest.NewRecorder()
 256                 rtr.ServeHTTP(resp, req)
 257                 c.Check(resp.Code, Equals, t.expectStatus)
 258         }
 259 }
 260
 261 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 262         kc := runProxy(c, false)
 263         defer closeListener()
 264         router.(*proxyHandler).timeout = time.Nanosecond
 265
 266         buf := make([]byte, 1<<20)
 267         rand.Read(buf)
 268         var wg sync.WaitGroup
 269         for i := 0; i < 128; i++ {
 270                 wg.Add(1)
 271                 go func() {
 272                         defer wg.Done()
 273                         kc.PutB(buf)
 274                 }()
 275         }
 276         done := make(chan bool)
 277         go func() {
 278                 wg.Wait()
 279                 close(done)
 280         }()
 281         select {
 282         case <-done:
 283         case <-time.After(10 * time.Second):
 284                 c.Error("timeout")
 285         }
 286 }
 287
 288 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 289         kc := runProxy(c, false)
 290         defer closeListener()
 291
 292         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 293         var hash2 string
 294
 295         {
 296                 _, _, err := kc.Ask(hash)
 297                 c.Check(err, Equals, keepclient.BlockNotFound)
 298                 c.Log("Finished Ask (expected BlockNotFound)")
 299         }
 300
 301         {
 302                 reader, _, _, err := kc.Get(hash)
 303                 c.Check(reader, Equals, nil)
 304                 c.Check(err, Equals, keepclient.BlockNotFound)
 305                 c.Log("Finished Get (expected BlockNotFound)")
 306         }
 307
 308         // Note in bug #5309 among other errors keepproxy would set
 309         // Content-Length incorrectly on the 404 BlockNotFound response, this
 310         // would result in a protocol violation that would prevent reuse of the
 311         // connection, which would manifest by the next attempt to use the
 312         // connection (in this case the PutB below) failing.  So to test for
 313         // that bug it's necessary to trigger an error response (such as
 314         // BlockNotFound) and then do something else with the same httpClient
 315         // connection.
 316
 317         {
 318                 var rep int
 319                 var err error
 320                 hash2, rep, err = kc.PutB([]byte("foo"))
 321                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 322                 c.Check(rep, Equals, 2)
 323                 c.Check(err, Equals, nil)
 324                 c.Log("Finished PutB (expected success)")
 325         }
 326
 327         {
 328                 blocklen, _, err := kc.Ask(hash2)
 329                 c.Assert(err, Equals, nil)
 330                 c.Check(blocklen, Equals, int64(3))
 331                 c.Log("Finished Ask (expected success)")
 332         }
 333
 334         {
 335                 reader, blocklen, _, err := kc.Get(hash2)
 336                 c.Assert(err, Equals, nil)
 337                 all, err := ioutil.ReadAll(reader)
 338                 c.Check(err, IsNil)
 339                 c.Check(all, DeepEquals, []byte("foo"))
 340                 c.Check(blocklen, Equals, int64(3))
 341                 c.Log("Finished Get (expected success)")
 342         }
 343
 344         {
 345                 var rep int
 346                 var err error
 347                 hash2, rep, err = kc.PutB([]byte(""))
 348                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 349                 c.Check(rep, Equals, 2)
 350                 c.Check(err, Equals, nil)
 351                 c.Log("Finished PutB zero block")
 352         }
 353
 354         {
 355                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 356                 c.Assert(err, Equals, nil)
 357                 all, err := ioutil.ReadAll(reader)
 358                 c.Check(err, IsNil)
 359                 c.Check(all, DeepEquals, []byte(""))
 360                 c.Check(blocklen, Equals, int64(0))
 361                 c.Log("Finished Get zero block")
 362         }
 363 }
 364
 365 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 366         kc := runProxy(c, true)
 367         defer closeListener()
 368
 369         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 370
 371         _, _, err := kc.Ask(hash)
 372         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 373
 374         hash2, rep, err := kc.PutB([]byte("bar"))
 375         c.Check(hash2, Equals, "")
 376         c.Check(rep, Equals, 0)
 377         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 378
 379         blocklen, _, err := kc.Ask(hash)
 380         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 381         c.Check(err, ErrorMatches, ".*not found.*")
 382         c.Check(blocklen, Equals, int64(0))
 383
 384         _, blocklen, _, err = kc.Get(hash)
 385         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 386         c.Check(err, ErrorMatches, ".*not found.*")
 387         c.Check(blocklen, Equals, int64(0))
 388
 389 }
 390
 391 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 392         runProxy(c, false)
 393         defer closeListener()
 394
 395         {
 396                 client := http.Client{}
 397                 req, err := http.NewRequest("OPTIONS",
 398                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 399                         nil)
 400                 c.Assert(err, IsNil)
 401                 req.Header.Add("Access-Control-Request-Method", "PUT")
 402                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 403                 resp, err := client.Do(req)
 404                 c.Check(err, Equals, nil)
 405                 c.Check(resp.StatusCode, Equals, 200)
 406                 body, err := ioutil.ReadAll(resp.Body)
 407                 c.Check(err, IsNil)
 408                 c.Check(string(body), Equals, "")
 409                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 410                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 411         }
 412
 413         {
 414                 resp, err := http.Get(
 415                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 416                 c.Check(err, Equals, nil)
 417                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 418                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 419         }
 420 }
 421
 422 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 423         runProxy(c, false)
 424         defer closeListener()
 425
 426         {
 427                 client := http.Client{}
 428                 req, err := http.NewRequest("POST",
 429                         "http://"+listener.Addr().String()+"/",
 430                         strings.NewReader("qux"))
 431                 c.Check(err, IsNil)
 432                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 433                 req.Header.Add("Content-Type", "application/octet-stream")
 434                 resp, err := client.Do(req)
 435                 c.Check(err, Equals, nil)
 436                 body, err := ioutil.ReadAll(resp.Body)
 437                 c.Check(err, Equals, nil)
 438                 c.Check(string(body), Matches,
 439                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 440         }
 441 }
 442
 443 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 444         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 445                 Equals,
 446                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 447         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 448                 Equals,
 449                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 450         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 451                 Equals,
 452                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 453         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 454                 Equals,
 455                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 456
 457 }
 458
 459 // Test GetIndex
 460 //   Put one block, with 2 replicas
 461 //   With no prefix (expect the block locator, twice)
 462 //   With an existing prefix (expect the block locator, twice)
 463 //   With a valid but non-existing prefix (expect "\n")
 464 //   With an invalid prefix (expect error)
 465 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 466         kc := runProxy(c, false)
 467         defer closeListener()
 468
 469         // Put "index-data" blocks
 470         data := []byte("index-data")
 471         hash := fmt.Sprintf("%x", md5.Sum(data))
 472
 473         hash2, rep, err := kc.PutB(data)
 474         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 475         c.Check(rep, Equals, 2)
 476         c.Check(err, Equals, nil)
 477
 478         reader, blocklen, _, err := kc.Get(hash)
 479         c.Assert(err, IsNil)
 480         c.Check(blocklen, Equals, int64(10))
 481         all, err := ioutil.ReadAll(reader)
 482         c.Assert(err, IsNil)
 483         c.Check(all, DeepEquals, data)
 484
 485         // Put some more blocks
 486         _, _, err = kc.PutB([]byte("some-more-index-data"))
 487         c.Check(err, IsNil)
 488
 489         kc.Arvados.ApiToken = arvadostest.SystemRootToken
 490
 491         // Invoke GetIndex
 492         for _, spec := range []struct {
 493                 prefix         string
 494                 expectTestHash bool
 495                 expectOther    bool
 496         }{
 497                 {"", true, true},         // with no prefix
 498                 {hash[:3], true, false},  // with matching prefix
 499                 {"abcdef", false, false}, // with no such prefix
 500         } {
 501                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 502                 c.Assert(err, Equals, nil)
 503                 indexResp, err := ioutil.ReadAll(indexReader)
 504                 c.Assert(err, Equals, nil)
 505                 locators := strings.Split(string(indexResp), "\n")
 506                 gotTestHash := 0
 507                 gotOther := 0
 508                 for _, locator := range locators {
 509                         if locator == "" {
 510                                 continue
 511                         }
 512                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 513                         if locator[:32] == hash {
 514                                 gotTestHash++
 515                         } else {
 516                                 gotOther++
 517                         }
 518                 }
 519                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 520                 c.Check(gotOther > 0, Equals, spec.expectOther)
 521         }
 522
 523         // GetIndex with invalid prefix
 524         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 525         c.Assert((err != nil), Equals, true)
 526 }
 527
 528 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 529         kc := runProxy(c, false)
 530         defer closeListener()
 531         hash, _, err := kc.PutB([]byte("shareddata"))
 532         c.Check(err, IsNil)
 533         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 534         rdr, _, _, err := kc.Get(hash)
 535         c.Assert(err, IsNil)
 536         data, err := ioutil.ReadAll(rdr)
 537         c.Check(err, IsNil)
 538         c.Check(data, DeepEquals, []byte("shareddata"))
 539 }
 540
 541 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 542         kc := runProxy(c, false)
 543         defer closeListener()
 544
 545         // Put a test block
 546         hash, rep, err := kc.PutB([]byte("foo"))
 547         c.Check(err, IsNil)
 548         c.Check(rep, Equals, 2)
 549
 550         for _, badToken := range []string{
 551                 "nosuchtoken",
 552                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 553         } {
 554                 kc.Arvados.ApiToken = badToken
 555
 556                 // Ask and Get will fail only if the upstream
 557                 // keepstore server checks for valid signatures.
 558                 // Without knowing the blob signing key, there is no
 559                 // way for keepproxy to know whether a given token is
 560                 // permitted to read a block.  So these tests fail:
 561                 if false {
 562                         _, _, err = kc.Ask(hash)
 563                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 564                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 565                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 566
 567                         _, _, _, err = kc.Get(hash)
 568                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 569                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 570                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 571                 }
 572
 573                 _, _, err = kc.PutB([]byte("foo"))
 574                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 575         }
 576 }
 577
 578 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 579         kc := runProxy(c, false)
 580         defer closeListener()
 581
 582         // Point keepproxy at a non-existent keepstore
 583         locals := map[string]string{
 584                 TestProxyUUID: "http://localhost:12345",
 585         }
 586         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
 587
 588         // Ask should result in temporary bad gateway error
 589         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 590         _, _, err := kc.Ask(hash)
 591         c.Check(err, NotNil)
 592         errNotFound, _ := err.(*keepclient.ErrNotFound)
 593         c.Check(errNotFound.Temporary(), Equals, true)
 594         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 595
 596         // Get should result in temporary bad gateway error
 597         _, _, _, err = kc.Get(hash)
 598         c.Check(err, NotNil)
 599         errNotFound, _ = err.(*keepclient.ErrNotFound)
 600         c.Check(errNotFound.Temporary(), Equals, true)
 601         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 602 }
 603
 604 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 605         kc := runProxy(c, false)
 606         defer closeListener()
 607
 608         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 609         for _, f := range []func() error{
 610                 func() error {
 611                         _, _, err := kc.Ask(hash)
 612                         return err
 613                 },
 614                 func() error {
 615                         _, _, _, err := kc.Get(hash)
 616                         return err
 617                 },
 618         } {
 619                 err := f()
 620                 c.Assert(err, NotNil)
 621                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 622                 c.Check(errNotFound.Temporary(), Equals, true)
 623                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 624         }
 625 }
 626
 627 func (s *ServerRequiredSuite) TestPing(c *C) {
 628         kc := runProxy(c, false)
 629         defer closeListener()
 630
 631         rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
 632
 633         req, err := http.NewRequest("GET",
 634                 "http://"+listener.Addr().String()+"/_health/ping",
 635                 nil)
 636         c.Assert(err, IsNil)
 637         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 638
 639         resp := httptest.NewRecorder()
 640         rtr.ServeHTTP(resp, req)
 641         c.Check(resp.Code, Equals, 200)
 642         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 643 }