services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "strings"
  17         "sync"
  18         "testing"
  19         "time"
  20
  21         "git.curoverse.com/arvados.git/lib/config"
  22         "git.curoverse.com/arvados.git/sdk/go/arvados"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  24         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  25         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  26         log "github.com/sirupsen/logrus"
  27
  28         . "gopkg.in/check.v1"
  29 )
  30
  31 // Gocheck boilerplate
  32 func Test(t *testing.T) {
  33         TestingT(t)
  34 }
  35
  36 // Gocheck boilerplate
  37 var _ = Suite(&ServerRequiredSuite{})
  38
  39 // Tests that require the Keep server running
  40 type ServerRequiredSuite struct{}
  41
  42 // Gocheck boilerplate
  43 var _ = Suite(&NoKeepServerSuite{})
  44
  45 // Test with no keepserver to simulate errors
  46 type NoKeepServerSuite struct{}
  47
  48 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  49
  50 // Wait (up to 1 second) for keepproxy to listen on a port. This
  51 // avoids a race condition where we hit a "connection refused" error
  52 // because we start testing the proxy too soon.
  53 func waitForListener() {
  54         const (
  55                 ms = 5
  56         )
  57         for i := 0; listener == nil && i < 10000; i += ms {
  58                 time.Sleep(ms * time.Millisecond)
  59         }
  60         if listener == nil {
  61                 panic("Timed out waiting for listener to start")
  62         }
  63 }
  64
  65 func closeListener() {
  66         if listener != nil {
  67                 listener.Close()
  68         }
  69 }
  70
  71 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  72         arvadostest.StartAPI()
  73         arvadostest.StartKeep(2, false)
  74 }
  75
  76 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  77         arvadostest.ResetEnv()
  78 }
  79
  80 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  81         arvadostest.StopKeep(2)
  82         arvadostest.StopAPI()
  83 }
  84
  85 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  86         arvadostest.StartAPI()
  87         // We need API to have some keep services listed, but the
  88         // services themselves should be unresponsive.
  89         arvadostest.StartKeep(2, false)
  90         arvadostest.StopKeep(2)
  91 }
  92
  93 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  94         arvadostest.ResetEnv()
  95 }
  96
  97 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  98         arvadostest.StopAPI()
  99 }
 100
 101 func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
 102         cfg, err := config.NewLoader(nil, nil).Load()
 103         c.Assert(err, Equals, nil)
 104         cluster, err := cfg.GetCluster("")
 105         c.Assert(err, Equals, nil)
 106
 107         cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
 108
 109         listener = nil
 110         go func() {
 111                 run(log.New(), cluster)
 112                 defer closeListener()
 113         }()
 114         waitForListener()
 115
 116         client := arvados.NewClientFromEnv()
 117         arv, err := arvadosclient.New(client)
 118         c.Assert(err, Equals, nil)
 119         if bogusClientToken {
 120                 arv.ApiToken = "bogus-token"
 121         }
 122         kc := keepclient.New(arv)
 123         sr := map[string]string{
 124                 TestProxyUUID: "http://" + listener.Addr().String(),
 125         }
 126         kc.SetServiceRoots(sr, sr, sr)
 127         kc.Arvados.External = true
 128
 129         return kc
 130 }
 131
 132 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 133         runProxy(c, false)
 134         defer closeListener()
 135
 136         req, err := http.NewRequest("POST",
 137                 "http://"+listener.Addr().String()+"/",
 138                 strings.NewReader("TestViaHeader"))
 139         c.Assert(err, Equals, nil)
 140         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 141         resp, err := (&http.Client{}).Do(req)
 142         c.Assert(err, Equals, nil)
 143         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 144         locator, err := ioutil.ReadAll(resp.Body)
 145         c.Assert(err, Equals, nil)
 146         resp.Body.Close()
 147
 148         req, err = http.NewRequest("GET",
 149                 "http://"+listener.Addr().String()+"/"+string(locator),
 150                 nil)
 151         c.Assert(err, Equals, nil)
 152         resp, err = (&http.Client{}).Do(req)
 153         c.Assert(err, Equals, nil)
 154         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 155         resp.Body.Close()
 156 }
 157
 158 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 159         kc := runProxy(c, false)
 160         defer closeListener()
 161
 162         sr := map[string]string{
 163                 TestProxyUUID: "http://" + listener.Addr().String(),
 164         }
 165         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 166
 167         content := []byte("TestLoopDetection")
 168         _, _, err := kc.PutB(content)
 169         c.Check(err, ErrorMatches, `.*loop detected.*`)
 170
 171         hash := fmt.Sprintf("%x", md5.Sum(content))
 172         _, _, _, err = kc.Get(hash)
 173         c.Check(err, ErrorMatches, `.*loop detected.*`)
 174 }
 175
 176 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 177         kc := runProxy(c, false)
 178         defer closeListener()
 179
 180         // Set up fake keepstore to record request headers
 181         var hdr http.Header
 182         ts := httptest.NewServer(http.HandlerFunc(
 183                 func(w http.ResponseWriter, r *http.Request) {
 184                         hdr = r.Header
 185                         http.Error(w, "Error", http.StatusInternalServerError)
 186                 }))
 187         defer ts.Close()
 188
 189         // Point keepproxy router's keepclient to the fake keepstore
 190         sr := map[string]string{
 191                 TestProxyUUID: ts.URL,
 192         }
 193         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 194
 195         // Set up client to ask for storage classes to keepproxy
 196         kc.StorageClasses = []string{"secure"}
 197         content := []byte("Very important data")
 198         _, _, err := kc.PutB(content)
 199         c.Check(err, NotNil)
 200         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 201 }
 202
 203 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 204         kc := runProxy(c, false)
 205         defer closeListener()
 206
 207         content := []byte("TestDesiredReplicas")
 208         hash := fmt.Sprintf("%x", md5.Sum(content))
 209
 210         for _, kc.Want_replicas = range []int{0, 1, 2} {
 211                 locator, rep, err := kc.PutB(content)
 212                 c.Check(err, Equals, nil)
 213                 c.Check(rep, Equals, kc.Want_replicas)
 214                 if rep > 0 {
 215                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 216                 }
 217         }
 218 }
 219
 220 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 221         kc := runProxy(c, false)
 222         defer closeListener()
 223
 224         content := []byte("TestPutWrongContentLength")
 225         hash := fmt.Sprintf("%x", md5.Sum(content))
 226
 227         // If we use http.Client to send these requests to the network
 228         // server we just started, the Go http library automatically
 229         // fixes the invalid Content-Length header. In order to test
 230         // our server behavior, we have to call the handler directly
 231         // using an httptest.ResponseRecorder.
 232         rtr := MakeRESTRouter(kc, 10*time.Second, "")
 233
 234         type testcase struct {
 235                 sendLength   string
 236                 expectStatus int
 237         }
 238
 239         for _, t := range []testcase{
 240                 {"1", http.StatusBadRequest},
 241                 {"", http.StatusLengthRequired},
 242                 {"-1", http.StatusLengthRequired},
 243                 {"abcdef", http.StatusLengthRequired},
 244         } {
 245                 req, err := http.NewRequest("PUT",
 246                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 247                         bytes.NewReader(content))
 248                 c.Assert(err, IsNil)
 249                 req.Header.Set("Content-Length", t.sendLength)
 250                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 251                 req.Header.Set("Content-Type", "application/octet-stream")
 252
 253                 resp := httptest.NewRecorder()
 254                 rtr.ServeHTTP(resp, req)
 255                 c.Check(resp.Code, Equals, t.expectStatus)
 256         }
 257 }
 258
 259 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 260         kc := runProxy(c, false)
 261         defer closeListener()
 262         router.(*proxyHandler).timeout = time.Nanosecond
 263
 264         buf := make([]byte, 1<<20)
 265         rand.Read(buf)
 266         var wg sync.WaitGroup
 267         for i := 0; i < 128; i++ {
 268                 wg.Add(1)
 269                 go func() {
 270                         defer wg.Done()
 271                         kc.PutB(buf)
 272                 }()
 273         }
 274         done := make(chan bool)
 275         go func() {
 276                 wg.Wait()
 277                 close(done)
 278         }()
 279         select {
 280         case <-done:
 281         case <-time.After(10 * time.Second):
 282                 c.Error("timeout")
 283         }
 284 }
 285
 286 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 287         kc := runProxy(c, false)
 288         defer closeListener()
 289
 290         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 291         var hash2 string
 292
 293         {
 294                 _, _, err := kc.Ask(hash)
 295                 c.Check(err, Equals, keepclient.BlockNotFound)
 296                 c.Log("Finished Ask (expected BlockNotFound)")
 297         }
 298
 299         {
 300                 reader, _, _, err := kc.Get(hash)
 301                 c.Check(reader, Equals, nil)
 302                 c.Check(err, Equals, keepclient.BlockNotFound)
 303                 c.Log("Finished Get (expected BlockNotFound)")
 304         }
 305
 306         // Note in bug #5309 among other errors keepproxy would set
 307         // Content-Length incorrectly on the 404 BlockNotFound response, this
 308         // would result in a protocol violation that would prevent reuse of the
 309         // connection, which would manifest by the next attempt to use the
 310         // connection (in this case the PutB below) failing.  So to test for
 311         // that bug it's necessary to trigger an error response (such as
 312         // BlockNotFound) and then do something else with the same httpClient
 313         // connection.
 314
 315         {
 316                 var rep int
 317                 var err error
 318                 hash2, rep, err = kc.PutB([]byte("foo"))
 319                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 320                 c.Check(rep, Equals, 2)
 321                 c.Check(err, Equals, nil)
 322                 c.Log("Finished PutB (expected success)")
 323         }
 324
 325         {
 326                 blocklen, _, err := kc.Ask(hash2)
 327                 c.Assert(err, Equals, nil)
 328                 c.Check(blocklen, Equals, int64(3))
 329                 c.Log("Finished Ask (expected success)")
 330         }
 331
 332         {
 333                 reader, blocklen, _, err := kc.Get(hash2)
 334                 c.Assert(err, Equals, nil)
 335                 all, err := ioutil.ReadAll(reader)
 336                 c.Check(err, IsNil)
 337                 c.Check(all, DeepEquals, []byte("foo"))
 338                 c.Check(blocklen, Equals, int64(3))
 339                 c.Log("Finished Get (expected success)")
 340         }
 341
 342         {
 343                 var rep int
 344                 var err error
 345                 hash2, rep, err = kc.PutB([]byte(""))
 346                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 347                 c.Check(rep, Equals, 2)
 348                 c.Check(err, Equals, nil)
 349                 c.Log("Finished PutB zero block")
 350         }
 351
 352         {
 353                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 354                 c.Assert(err, Equals, nil)
 355                 all, err := ioutil.ReadAll(reader)
 356                 c.Check(err, IsNil)
 357                 c.Check(all, DeepEquals, []byte(""))
 358                 c.Check(blocklen, Equals, int64(0))
 359                 c.Log("Finished Get zero block")
 360         }
 361 }
 362
 363 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 364         kc := runProxy(c, true)
 365         defer closeListener()
 366
 367         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 368
 369         _, _, err := kc.Ask(hash)
 370         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 371
 372         hash2, rep, err := kc.PutB([]byte("bar"))
 373         c.Check(hash2, Equals, "")
 374         c.Check(rep, Equals, 0)
 375         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 376
 377         blocklen, _, err := kc.Ask(hash)
 378         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 379         c.Check(err, ErrorMatches, ".*not found.*")
 380         c.Check(blocklen, Equals, int64(0))
 381
 382         _, blocklen, _, err = kc.Get(hash)
 383         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 384         c.Check(err, ErrorMatches, ".*not found.*")
 385         c.Check(blocklen, Equals, int64(0))
 386
 387 }
 388
 389 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 390         runProxy(c, false)
 391         defer closeListener()
 392
 393         {
 394                 client := http.Client{}
 395                 req, err := http.NewRequest("OPTIONS",
 396                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 397                         nil)
 398                 c.Assert(err, IsNil)
 399                 req.Header.Add("Access-Control-Request-Method", "PUT")
 400                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 401                 resp, err := client.Do(req)
 402                 c.Check(err, Equals, nil)
 403                 c.Check(resp.StatusCode, Equals, 200)
 404                 body, err := ioutil.ReadAll(resp.Body)
 405                 c.Check(err, IsNil)
 406                 c.Check(string(body), Equals, "")
 407                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 408                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 409         }
 410
 411         {
 412                 resp, err := http.Get(
 413                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 414                 c.Check(err, Equals, nil)
 415                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 416                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 417         }
 418 }
 419
 420 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 421         runProxy(c, false)
 422         defer closeListener()
 423
 424         {
 425                 client := http.Client{}
 426                 req, err := http.NewRequest("POST",
 427                         "http://"+listener.Addr().String()+"/",
 428                         strings.NewReader("qux"))
 429                 c.Check(err, IsNil)
 430                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 431                 req.Header.Add("Content-Type", "application/octet-stream")
 432                 resp, err := client.Do(req)
 433                 c.Check(err, Equals, nil)
 434                 body, err := ioutil.ReadAll(resp.Body)
 435                 c.Check(err, Equals, nil)
 436                 c.Check(string(body), Matches,
 437                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 438         }
 439 }
 440
 441 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 442         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 443                 Equals,
 444                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 445         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 446                 Equals,
 447                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 448         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 449                 Equals,
 450                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 451         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 452                 Equals,
 453                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 454
 455 }
 456
 457 // Test GetIndex
 458 //   Put one block, with 2 replicas
 459 //   With no prefix (expect the block locator, twice)
 460 //   With an existing prefix (expect the block locator, twice)
 461 //   With a valid but non-existing prefix (expect "\n")
 462 //   With an invalid prefix (expect error)
 463 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 464         kc := runProxy(c, false)
 465         defer closeListener()
 466
 467         // Put "index-data" blocks
 468         data := []byte("index-data")
 469         hash := fmt.Sprintf("%x", md5.Sum(data))
 470
 471         hash2, rep, err := kc.PutB(data)
 472         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 473         c.Check(rep, Equals, 2)
 474         c.Check(err, Equals, nil)
 475
 476         reader, blocklen, _, err := kc.Get(hash)
 477         c.Assert(err, IsNil)
 478         c.Check(blocklen, Equals, int64(10))
 479         all, err := ioutil.ReadAll(reader)
 480         c.Assert(err, IsNil)
 481         c.Check(all, DeepEquals, data)
 482
 483         // Put some more blocks
 484         _, _, err = kc.PutB([]byte("some-more-index-data"))
 485         c.Check(err, IsNil)
 486
 487         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 488
 489         // Invoke GetIndex
 490         for _, spec := range []struct {
 491                 prefix         string
 492                 expectTestHash bool
 493                 expectOther    bool
 494         }{
 495                 {"", true, true},         // with no prefix
 496                 {hash[:3], true, false},  // with matching prefix
 497                 {"abcdef", false, false}, // with no such prefix
 498         } {
 499                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 500                 c.Assert(err, Equals, nil)
 501                 indexResp, err := ioutil.ReadAll(indexReader)
 502                 c.Assert(err, Equals, nil)
 503                 locators := strings.Split(string(indexResp), "\n")
 504                 gotTestHash := 0
 505                 gotOther := 0
 506                 for _, locator := range locators {
 507                         if locator == "" {
 508                                 continue
 509                         }
 510                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 511                         if locator[:32] == hash {
 512                                 gotTestHash++
 513                         } else {
 514                                 gotOther++
 515                         }
 516                 }
 517                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 518                 c.Check(gotOther > 0, Equals, spec.expectOther)
 519         }
 520
 521         // GetIndex with invalid prefix
 522         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 523         c.Assert((err != nil), Equals, true)
 524 }
 525
 526 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 527         kc := runProxy(c, false)
 528         defer closeListener()
 529         hash, _, err := kc.PutB([]byte("shareddata"))
 530         c.Check(err, IsNil)
 531         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 532         rdr, _, _, err := kc.Get(hash)
 533         c.Assert(err, IsNil)
 534         data, err := ioutil.ReadAll(rdr)
 535         c.Check(err, IsNil)
 536         c.Check(data, DeepEquals, []byte("shareddata"))
 537 }
 538
 539 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 540         kc := runProxy(c, false)
 541         defer closeListener()
 542
 543         // Put a test block
 544         hash, rep, err := kc.PutB([]byte("foo"))
 545         c.Check(err, IsNil)
 546         c.Check(rep, Equals, 2)
 547
 548         for _, badToken := range []string{
 549                 "nosuchtoken",
 550                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 551         } {
 552                 kc.Arvados.ApiToken = badToken
 553
 554                 // Ask and Get will fail only if the upstream
 555                 // keepstore server checks for valid signatures.
 556                 // Without knowing the blob signing key, there is no
 557                 // way for keepproxy to know whether a given token is
 558                 // permitted to read a block.  So these tests fail:
 559                 if false {
 560                         _, _, err = kc.Ask(hash)
 561                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 562                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 563                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 564
 565                         _, _, _, err = kc.Get(hash)
 566                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 567                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 568                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 569                 }
 570
 571                 _, _, err = kc.PutB([]byte("foo"))
 572                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 573         }
 574 }
 575
 576 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 577         kc := runProxy(c, false)
 578         defer closeListener()
 579
 580         // Point keepproxy at a non-existent keepstore
 581         locals := map[string]string{
 582                 TestProxyUUID: "http://localhost:12345",
 583         }
 584         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
 585
 586         // Ask should result in temporary bad gateway error
 587         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 588         _, _, err := kc.Ask(hash)
 589         c.Check(err, NotNil)
 590         errNotFound, _ := err.(*keepclient.ErrNotFound)
 591         c.Check(errNotFound.Temporary(), Equals, true)
 592         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 593
 594         // Get should result in temporary bad gateway error
 595         _, _, _, err = kc.Get(hash)
 596         c.Check(err, NotNil)
 597         errNotFound, _ = err.(*keepclient.ErrNotFound)
 598         c.Check(errNotFound.Temporary(), Equals, true)
 599         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 600 }
 601
 602 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 603         kc := runProxy(c, false)
 604         defer closeListener()
 605
 606         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 607         for _, f := range []func() error{
 608                 func() error {
 609                         _, _, err := kc.Ask(hash)
 610                         return err
 611                 },
 612                 func() error {
 613                         _, _, _, err := kc.Get(hash)
 614                         return err
 615                 },
 616         } {
 617                 err := f()
 618                 c.Assert(err, NotNil)
 619                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 620                 c.Check(errNotFound.Temporary(), Equals, true)
 621                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 622         }
 623 }
 624
 625 func (s *ServerRequiredSuite) TestPing(c *C) {
 626         kc := runProxy(c, false)
 627         defer closeListener()
 628
 629         rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
 630
 631         req, err := http.NewRequest("GET",
 632                 "http://"+listener.Addr().String()+"/_health/ping",
 633                 nil)
 634         c.Assert(err, IsNil)
 635         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 636
 637         resp := httptest.NewRecorder()
 638         rtr.ServeHTTP(resp, req)
 639         c.Check(resp.Code, Equals, 200)
 640         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 641 }