services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "strings"
  17         "sync"
  18         "testing"
  19         "time"
  20
  21         "git.curoverse.com/arvados.git/lib/config"
  22         "git.curoverse.com/arvados.git/sdk/go/arvados"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  24         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  25         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  26         log "github.com/sirupsen/logrus"
  27
  28         . "gopkg.in/check.v1"
  29 )
  30
  31 // Gocheck boilerplate
  32 func Test(t *testing.T) {
  33         TestingT(t)
  34 }
  35
  36 // Gocheck boilerplate
  37 var _ = Suite(&ServerRequiredSuite{})
  38
  39 // Tests that require the Keep server running
  40 type ServerRequiredSuite struct{}
  41
  42 // Gocheck boilerplate
  43 var _ = Suite(&NoKeepServerSuite{})
  44
  45 // Test with no keepserver to simulate errors
  46 type NoKeepServerSuite struct{}
  47
  48 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  49
  50 // Wait (up to 1 second) for keepproxy to listen on a port. This
  51 // avoids a race condition where we hit a "connection refused" error
  52 // because we start testing the proxy too soon.
  53 func waitForListener() {
  54         const (
  55                 ms = 5
  56         )
  57         for i := 0; listener == nil && i < 10000; i += ms {
  58                 time.Sleep(ms * time.Millisecond)
  59         }
  60         if listener == nil {
  61                 panic("Timed out waiting for listener to start")
  62         }
  63 }
  64
  65 func closeListener() {
  66         if listener != nil {
  67                 listener.Close()
  68         }
  69 }
  70
  71 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  72         arvadostest.StartAPI()
  73         arvadostest.StartKeep(2, false)
  74 }
  75
  76 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  77         arvadostest.ResetEnv()
  78 }
  79
  80 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  81         arvadostest.StopKeep(2)
  82         arvadostest.StopAPI()
  83 }
  84
  85 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  86         arvadostest.StartAPI()
  87         // We need API to have some keep services listed, but the
  88         // services themselves should be unresponsive.
  89         arvadostest.StartKeep(2, false)
  90         arvadostest.StopKeep(2)
  91 }
  92
  93 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  94         arvadostest.ResetEnv()
  95 }
  96
  97 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  98         arvadostest.StopAPI()
  99 }
 100
 101 func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
 102         cfg, err := config.NewLoader(nil, nil).Load()
 103         c.Assert(err, Equals, nil)
 104         cluster, err := cfg.GetCluster("")
 105         c.Assert(err, Equals, nil)
 106
 107         cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
 108
 109         listener = nil
 110         go run(log.New(), cluster)
 111         waitForListener()
 112
 113         client := arvados.NewClientFromEnv()
 114         arv, err := arvadosclient.New(client)
 115         c.Assert(err, Equals, nil)
 116         if bogusClientToken {
 117                 arv.ApiToken = "bogus-token"
 118         }
 119         kc := keepclient.New(arv)
 120         sr := map[string]string{
 121                 TestProxyUUID: "http://" + listener.Addr().String(),
 122         }
 123         kc.SetServiceRoots(sr, sr, sr)
 124         kc.Arvados.External = true
 125
 126         return kc
 127 }
 128
 129 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 130         runProxy(c, false)
 131         defer closeListener()
 132
 133         req, err := http.NewRequest("POST",
 134                 "http://"+listener.Addr().String()+"/",
 135                 strings.NewReader("TestViaHeader"))
 136         c.Assert(err, Equals, nil)
 137         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 138         resp, err := (&http.Client{}).Do(req)
 139         c.Assert(err, Equals, nil)
 140         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 141         locator, err := ioutil.ReadAll(resp.Body)
 142         c.Assert(err, Equals, nil)
 143         resp.Body.Close()
 144
 145         req, err = http.NewRequest("GET",
 146                 "http://"+listener.Addr().String()+"/"+string(locator),
 147                 nil)
 148         c.Assert(err, Equals, nil)
 149         resp, err = (&http.Client{}).Do(req)
 150         c.Assert(err, Equals, nil)
 151         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 152         resp.Body.Close()
 153 }
 154
 155 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 156         kc := runProxy(c, false)
 157         defer closeListener()
 158
 159         sr := map[string]string{
 160                 TestProxyUUID: "http://" + listener.Addr().String(),
 161         }
 162         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 163
 164         content := []byte("TestLoopDetection")
 165         _, _, err := kc.PutB(content)
 166         c.Check(err, ErrorMatches, `.*loop detected.*`)
 167
 168         hash := fmt.Sprintf("%x", md5.Sum(content))
 169         _, _, _, err = kc.Get(hash)
 170         c.Check(err, ErrorMatches, `.*loop detected.*`)
 171 }
 172
 173 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 174         kc := runProxy(c, false)
 175         defer closeListener()
 176
 177         // Set up fake keepstore to record request headers
 178         var hdr http.Header
 179         ts := httptest.NewServer(http.HandlerFunc(
 180                 func(w http.ResponseWriter, r *http.Request) {
 181                         hdr = r.Header
 182                         http.Error(w, "Error", http.StatusInternalServerError)
 183                 }))
 184         defer ts.Close()
 185
 186         // Point keepproxy router's keepclient to the fake keepstore
 187         sr := map[string]string{
 188                 TestProxyUUID: ts.URL,
 189         }
 190         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 191
 192         // Set up client to ask for storage classes to keepproxy
 193         kc.StorageClasses = []string{"secure"}
 194         content := []byte("Very important data")
 195         _, _, err := kc.PutB(content)
 196         c.Check(err, NotNil)
 197         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 198 }
 199
 200 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 201         kc := runProxy(c, false)
 202         defer closeListener()
 203
 204         content := []byte("TestDesiredReplicas")
 205         hash := fmt.Sprintf("%x", md5.Sum(content))
 206
 207         for _, kc.Want_replicas = range []int{0, 1, 2} {
 208                 locator, rep, err := kc.PutB(content)
 209                 c.Check(err, Equals, nil)
 210                 c.Check(rep, Equals, kc.Want_replicas)
 211                 if rep > 0 {
 212                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 213                 }
 214         }
 215 }
 216
 217 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 218         kc := runProxy(c, false)
 219         defer closeListener()
 220
 221         content := []byte("TestPutWrongContentLength")
 222         hash := fmt.Sprintf("%x", md5.Sum(content))
 223
 224         // If we use http.Client to send these requests to the network
 225         // server we just started, the Go http library automatically
 226         // fixes the invalid Content-Length header. In order to test
 227         // our server behavior, we have to call the handler directly
 228         // using an httptest.ResponseRecorder.
 229         rtr := MakeRESTRouter(kc, 10*time.Second, "")
 230
 231         type testcase struct {
 232                 sendLength   string
 233                 expectStatus int
 234         }
 235
 236         for _, t := range []testcase{
 237                 {"1", http.StatusBadRequest},
 238                 {"", http.StatusLengthRequired},
 239                 {"-1", http.StatusLengthRequired},
 240                 {"abcdef", http.StatusLengthRequired},
 241         } {
 242                 req, err := http.NewRequest("PUT",
 243                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 244                         bytes.NewReader(content))
 245                 c.Assert(err, IsNil)
 246                 req.Header.Set("Content-Length", t.sendLength)
 247                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 248                 req.Header.Set("Content-Type", "application/octet-stream")
 249
 250                 resp := httptest.NewRecorder()
 251                 rtr.ServeHTTP(resp, req)
 252                 c.Check(resp.Code, Equals, t.expectStatus)
 253         }
 254 }
 255
 256 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 257         kc := runProxy(c, false)
 258         defer closeListener()
 259         router.(*proxyHandler).timeout = time.Nanosecond
 260
 261         buf := make([]byte, 1<<20)
 262         rand.Read(buf)
 263         var wg sync.WaitGroup
 264         for i := 0; i < 128; i++ {
 265                 wg.Add(1)
 266                 go func() {
 267                         defer wg.Done()
 268                         kc.PutB(buf)
 269                 }()
 270         }
 271         done := make(chan bool)
 272         go func() {
 273                 wg.Wait()
 274                 close(done)
 275         }()
 276         select {
 277         case <-done:
 278         case <-time.After(10 * time.Second):
 279                 c.Error("timeout")
 280         }
 281 }
 282
 283 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 284         kc := runProxy(c, false)
 285         defer closeListener()
 286
 287         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 288         var hash2 string
 289
 290         {
 291                 _, _, err := kc.Ask(hash)
 292                 c.Check(err, Equals, keepclient.BlockNotFound)
 293                 c.Log("Finished Ask (expected BlockNotFound)")
 294         }
 295
 296         {
 297                 reader, _, _, err := kc.Get(hash)
 298                 c.Check(reader, Equals, nil)
 299                 c.Check(err, Equals, keepclient.BlockNotFound)
 300                 c.Log("Finished Get (expected BlockNotFound)")
 301         }
 302
 303         // Note in bug #5309 among other errors keepproxy would set
 304         // Content-Length incorrectly on the 404 BlockNotFound response, this
 305         // would result in a protocol violation that would prevent reuse of the
 306         // connection, which would manifest by the next attempt to use the
 307         // connection (in this case the PutB below) failing.  So to test for
 308         // that bug it's necessary to trigger an error response (such as
 309         // BlockNotFound) and then do something else with the same httpClient
 310         // connection.
 311
 312         {
 313                 var rep int
 314                 var err error
 315                 hash2, rep, err = kc.PutB([]byte("foo"))
 316                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 317                 c.Check(rep, Equals, 2)
 318                 c.Check(err, Equals, nil)
 319                 c.Log("Finished PutB (expected success)")
 320         }
 321
 322         {
 323                 blocklen, _, err := kc.Ask(hash2)
 324                 c.Assert(err, Equals, nil)
 325                 c.Check(blocklen, Equals, int64(3))
 326                 c.Log("Finished Ask (expected success)")
 327         }
 328
 329         {
 330                 reader, blocklen, _, err := kc.Get(hash2)
 331                 c.Assert(err, Equals, nil)
 332                 all, err := ioutil.ReadAll(reader)
 333                 c.Check(err, IsNil)
 334                 c.Check(all, DeepEquals, []byte("foo"))
 335                 c.Check(blocklen, Equals, int64(3))
 336                 c.Log("Finished Get (expected success)")
 337         }
 338
 339         {
 340                 var rep int
 341                 var err error
 342                 hash2, rep, err = kc.PutB([]byte(""))
 343                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 344                 c.Check(rep, Equals, 2)
 345                 c.Check(err, Equals, nil)
 346                 c.Log("Finished PutB zero block")
 347         }
 348
 349         {
 350                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 351                 c.Assert(err, Equals, nil)
 352                 all, err := ioutil.ReadAll(reader)
 353                 c.Check(err, IsNil)
 354                 c.Check(all, DeepEquals, []byte(""))
 355                 c.Check(blocklen, Equals, int64(0))
 356                 c.Log("Finished Get zero block")
 357         }
 358 }
 359
 360 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 361         kc := runProxy(c, true)
 362         defer closeListener()
 363
 364         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 365
 366         _, _, err := kc.Ask(hash)
 367         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 368
 369         hash2, rep, err := kc.PutB([]byte("bar"))
 370         c.Check(hash2, Equals, "")
 371         c.Check(rep, Equals, 0)
 372         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 373
 374         blocklen, _, err := kc.Ask(hash)
 375         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 376         c.Check(err, ErrorMatches, ".*not found.*")
 377         c.Check(blocklen, Equals, int64(0))
 378
 379         _, blocklen, _, err = kc.Get(hash)
 380         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 381         c.Check(err, ErrorMatches, ".*not found.*")
 382         c.Check(blocklen, Equals, int64(0))
 383
 384 }
 385
 386 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 387         runProxy(c, false)
 388         defer closeListener()
 389
 390         {
 391                 client := http.Client{}
 392                 req, err := http.NewRequest("OPTIONS",
 393                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 394                         nil)
 395                 c.Assert(err, IsNil)
 396                 req.Header.Add("Access-Control-Request-Method", "PUT")
 397                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 398                 resp, err := client.Do(req)
 399                 c.Check(err, Equals, nil)
 400                 c.Check(resp.StatusCode, Equals, 200)
 401                 body, err := ioutil.ReadAll(resp.Body)
 402                 c.Check(err, IsNil)
 403                 c.Check(string(body), Equals, "")
 404                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 405                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 406         }
 407
 408         {
 409                 resp, err := http.Get(
 410                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 411                 c.Check(err, Equals, nil)
 412                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 413                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 414         }
 415 }
 416
 417 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 418         runProxy(c, false)
 419         defer closeListener()
 420
 421         {
 422                 client := http.Client{}
 423                 req, err := http.NewRequest("POST",
 424                         "http://"+listener.Addr().String()+"/",
 425                         strings.NewReader("qux"))
 426                 c.Check(err, IsNil)
 427                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 428                 req.Header.Add("Content-Type", "application/octet-stream")
 429                 resp, err := client.Do(req)
 430                 c.Check(err, Equals, nil)
 431                 body, err := ioutil.ReadAll(resp.Body)
 432                 c.Check(err, Equals, nil)
 433                 c.Check(string(body), Matches,
 434                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 435         }
 436 }
 437
 438 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 439         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 440                 Equals,
 441                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 442         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 443                 Equals,
 444                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 445         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 446                 Equals,
 447                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 448         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 449                 Equals,
 450                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 451
 452 }
 453
 454 // Test GetIndex
 455 //   Put one block, with 2 replicas
 456 //   With no prefix (expect the block locator, twice)
 457 //   With an existing prefix (expect the block locator, twice)
 458 //   With a valid but non-existing prefix (expect "\n")
 459 //   With an invalid prefix (expect error)
 460 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 461         kc := runProxy(c, false)
 462         defer closeListener()
 463
 464         // Put "index-data" blocks
 465         data := []byte("index-data")
 466         hash := fmt.Sprintf("%x", md5.Sum(data))
 467
 468         hash2, rep, err := kc.PutB(data)
 469         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 470         c.Check(rep, Equals, 2)
 471         c.Check(err, Equals, nil)
 472
 473         reader, blocklen, _, err := kc.Get(hash)
 474         c.Assert(err, IsNil)
 475         c.Check(blocklen, Equals, int64(10))
 476         all, err := ioutil.ReadAll(reader)
 477         c.Assert(err, IsNil)
 478         c.Check(all, DeepEquals, data)
 479
 480         // Put some more blocks
 481         _, _, err = kc.PutB([]byte("some-more-index-data"))
 482         c.Check(err, IsNil)
 483
 484         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 485
 486         // Invoke GetIndex
 487         for _, spec := range []struct {
 488                 prefix         string
 489                 expectTestHash bool
 490                 expectOther    bool
 491         }{
 492                 {"", true, true},         // with no prefix
 493                 {hash[:3], true, false},  // with matching prefix
 494                 {"abcdef", false, false}, // with no such prefix
 495         } {
 496                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 497                 c.Assert(err, Equals, nil)
 498                 indexResp, err := ioutil.ReadAll(indexReader)
 499                 c.Assert(err, Equals, nil)
 500                 locators := strings.Split(string(indexResp), "\n")
 501                 gotTestHash := 0
 502                 gotOther := 0
 503                 for _, locator := range locators {
 504                         if locator == "" {
 505                                 continue
 506                         }
 507                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 508                         if locator[:32] == hash {
 509                                 gotTestHash++
 510                         } else {
 511                                 gotOther++
 512                         }
 513                 }
 514                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 515                 c.Check(gotOther > 0, Equals, spec.expectOther)
 516         }
 517
 518         // GetIndex with invalid prefix
 519         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 520         c.Assert((err != nil), Equals, true)
 521 }
 522
 523 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 524         kc := runProxy(c, false)
 525         defer closeListener()
 526         hash, _, err := kc.PutB([]byte("shareddata"))
 527         c.Check(err, IsNil)
 528         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 529         rdr, _, _, err := kc.Get(hash)
 530         c.Assert(err, IsNil)
 531         data, err := ioutil.ReadAll(rdr)
 532         c.Check(err, IsNil)
 533         c.Check(data, DeepEquals, []byte("shareddata"))
 534 }
 535
 536 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 537         kc := runProxy(c, false)
 538         defer closeListener()
 539
 540         // Put a test block
 541         hash, rep, err := kc.PutB([]byte("foo"))
 542         c.Check(err, IsNil)
 543         c.Check(rep, Equals, 2)
 544
 545         for _, badToken := range []string{
 546                 "nosuchtoken",
 547                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 548         } {
 549                 kc.Arvados.ApiToken = badToken
 550
 551                 // Ask and Get will fail only if the upstream
 552                 // keepstore server checks for valid signatures.
 553                 // Without knowing the blob signing key, there is no
 554                 // way for keepproxy to know whether a given token is
 555                 // permitted to read a block.  So these tests fail:
 556                 if false {
 557                         _, _, err = kc.Ask(hash)
 558                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 559                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 560                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 561
 562                         _, _, _, err = kc.Get(hash)
 563                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 564                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 565                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 566                 }
 567
 568                 _, _, err = kc.PutB([]byte("foo"))
 569                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 570         }
 571 }
 572
 573 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 574         kc := runProxy(c, false)
 575         defer closeListener()
 576
 577         // Point keepproxy at a non-existent keepstore
 578         locals := map[string]string{
 579                 TestProxyUUID: "http://localhost:12345",
 580         }
 581         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
 582
 583         // Ask should result in temporary bad gateway error
 584         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 585         _, _, err := kc.Ask(hash)
 586         c.Check(err, NotNil)
 587         errNotFound, _ := err.(*keepclient.ErrNotFound)
 588         c.Check(errNotFound.Temporary(), Equals, true)
 589         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 590
 591         // Get should result in temporary bad gateway error
 592         _, _, _, err = kc.Get(hash)
 593         c.Check(err, NotNil)
 594         errNotFound, _ = err.(*keepclient.ErrNotFound)
 595         c.Check(errNotFound.Temporary(), Equals, true)
 596         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 597 }
 598
 599 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 600         kc := runProxy(c, false)
 601         defer closeListener()
 602
 603         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 604         for _, f := range []func() error{
 605                 func() error {
 606                         _, _, err := kc.Ask(hash)
 607                         return err
 608                 },
 609                 func() error {
 610                         _, _, _, err := kc.Get(hash)
 611                         return err
 612                 },
 613         } {
 614                 err := f()
 615                 c.Assert(err, NotNil)
 616                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 617                 c.Check(errNotFound.Temporary(), Equals, true)
 618                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 619         }
 620 }
 621
 622 func (s *ServerRequiredSuite) TestPing(c *C) {
 623         kc := runProxy(c, false)
 624         defer closeListener()
 625
 626         rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
 627
 628         req, err := http.NewRequest("GET",
 629                 "http://"+listener.Addr().String()+"/_health/ping",
 630                 nil)
 631         c.Assert(err, IsNil)
 632         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 633
 634         resp := httptest.NewRecorder()
 635         rtr.ServeHTTP(resp, req)
 636         c.Check(resp.Code, Equals, 200)
 637         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 638 }