services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "os"
  17         "strings"
  18         "sync"
  19         "testing"
  20         "time"
  21
  22         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25
  26         . "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 func Test(t *testing.T) {
  31         TestingT(t)
  32 }
  33
  34 // Gocheck boilerplate
  35 var _ = Suite(&ServerRequiredSuite{})
  36
  37 // Tests that require the Keep server running
  38 type ServerRequiredSuite struct{}
  39
  40 // Gocheck boilerplate
  41 var _ = Suite(&NoKeepServerSuite{})
  42
  43 // Test with no keepserver to simulate errors
  44 type NoKeepServerSuite struct{}
  45
  46 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  47
  48 // Wait (up to 1 second) for keepproxy to listen on a port. This
  49 // avoids a race condition where we hit a "connection refused" error
  50 // because we start testing the proxy too soon.
  51 func waitForListener() {
  52         const (
  53                 ms = 5
  54         )
  55         for i := 0; listener == nil && i < 10000; i += ms {
  56                 time.Sleep(ms * time.Millisecond)
  57         }
  58         if listener == nil {
  59                 panic("Timed out waiting for listener to start")
  60         }
  61 }
  62
  63 func closeListener() {
  64         if listener != nil {
  65                 listener.Close()
  66         }
  67 }
  68
  69 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  70         arvadostest.StartAPI()
  71         arvadostest.StartKeep(2, false)
  72 }
  73
  74 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  75         arvadostest.ResetEnv()
  76 }
  77
  78 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  79         arvadostest.StopKeep(2)
  80         arvadostest.StopAPI()
  81 }
  82
  83 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  84         arvadostest.StartAPI()
  85         // We need API to have some keep services listed, but the
  86         // services themselves should be unresponsive.
  87         arvadostest.StartKeep(2, false)
  88         arvadostest.StopKeep(2)
  89 }
  90
  91 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  92         arvadostest.ResetEnv()
  93 }
  94
  95 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  96         arvadostest.StopAPI()
  97 }
  98
  99 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
 100         args = append([]string{"keepproxy"}, args...)
 101         os.Args = append(args, "-listen=:0")
 102         listener = nil
 103         go main()
 104         waitForListener()
 105
 106         arv, err := arvadosclient.MakeArvadosClient()
 107         c.Assert(err, Equals, nil)
 108         if bogusClientToken {
 109                 arv.ApiToken = "bogus-token"
 110         }
 111         kc := keepclient.New(arv)
 112         sr := map[string]string{
 113                 TestProxyUUID: "http://" + listener.Addr().String(),
 114         }
 115         kc.SetServiceRoots(sr, sr, sr)
 116         kc.Arvados.External = true
 117
 118         return kc
 119 }
 120
 121 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 122         runProxy(c, nil, false)
 123         defer closeListener()
 124
 125         req, err := http.NewRequest("POST",
 126                 "http://"+listener.Addr().String()+"/",
 127                 strings.NewReader("TestViaHeader"))
 128         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 129         resp, err := (&http.Client{}).Do(req)
 130         c.Assert(err, Equals, nil)
 131         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 132         locator, err := ioutil.ReadAll(resp.Body)
 133         c.Assert(err, Equals, nil)
 134         resp.Body.Close()
 135
 136         req, err = http.NewRequest("GET",
 137                 "http://"+listener.Addr().String()+"/"+string(locator),
 138                 nil)
 139         c.Assert(err, Equals, nil)
 140         resp, err = (&http.Client{}).Do(req)
 141         c.Assert(err, Equals, nil)
 142         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 143         resp.Body.Close()
 144 }
 145
 146 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 147         kc := runProxy(c, nil, false)
 148         defer closeListener()
 149
 150         sr := map[string]string{
 151                 TestProxyUUID: "http://" + listener.Addr().String(),
 152         }
 153         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 154
 155         content := []byte("TestLoopDetection")
 156         _, _, err := kc.PutB(content)
 157         c.Check(err, ErrorMatches, `.*loop detected.*`)
 158
 159         hash := fmt.Sprintf("%x", md5.Sum(content))
 160         _, _, _, err = kc.Get(hash)
 161         c.Check(err, ErrorMatches, `.*loop detected.*`)
 162 }
 163
 164 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 165         kc := runProxy(c, nil, false)
 166         defer closeListener()
 167
 168         content := []byte("TestDesiredReplicas")
 169         hash := fmt.Sprintf("%x", md5.Sum(content))
 170
 171         for _, kc.Want_replicas = range []int{0, 1, 2} {
 172                 locator, rep, err := kc.PutB(content)
 173                 c.Check(err, Equals, nil)
 174                 c.Check(rep, Equals, kc.Want_replicas)
 175                 if rep > 0 {
 176                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 177                 }
 178         }
 179 }
 180
 181 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 182         kc := runProxy(c, nil, false)
 183         defer closeListener()
 184
 185         content := []byte("TestPutWrongContentLength")
 186         hash := fmt.Sprintf("%x", md5.Sum(content))
 187
 188         // If we use http.Client to send these requests to the network
 189         // server we just started, the Go http library automatically
 190         // fixes the invalid Content-Length header. In order to test
 191         // our server behavior, we have to call the handler directly
 192         // using an httptest.ResponseRecorder.
 193         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
 194
 195         type testcase struct {
 196                 sendLength   string
 197                 expectStatus int
 198         }
 199
 200         for _, t := range []testcase{
 201                 {"1", http.StatusBadRequest},
 202                 {"", http.StatusLengthRequired},
 203                 {"-1", http.StatusLengthRequired},
 204                 {"abcdef", http.StatusLengthRequired},
 205         } {
 206                 req, err := http.NewRequest("PUT",
 207                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 208                         bytes.NewReader(content))
 209                 c.Assert(err, IsNil)
 210                 req.Header.Set("Content-Length", t.sendLength)
 211                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 212                 req.Header.Set("Content-Type", "application/octet-stream")
 213
 214                 resp := httptest.NewRecorder()
 215                 rtr.ServeHTTP(resp, req)
 216                 c.Check(resp.Code, Equals, t.expectStatus)
 217         }
 218 }
 219
 220 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 221         kc := runProxy(c, nil, false)
 222         defer closeListener()
 223         router.(*proxyHandler).timeout = time.Nanosecond
 224
 225         buf := make([]byte, 1<<20)
 226         rand.Read(buf)
 227         var wg sync.WaitGroup
 228         for i := 0; i < 128; i++ {
 229                 wg.Add(1)
 230                 go func() {
 231                         defer wg.Done()
 232                         kc.PutB(buf)
 233                 }()
 234         }
 235         done := make(chan bool)
 236         go func() {
 237                 wg.Wait()
 238                 close(done)
 239         }()
 240         select {
 241         case <-done:
 242         case <-time.After(10 * time.Second):
 243                 c.Error("timeout")
 244         }
 245 }
 246
 247 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 248         kc := runProxy(c, nil, false)
 249         defer closeListener()
 250
 251         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 252         var hash2 string
 253
 254         {
 255                 _, _, err := kc.Ask(hash)
 256                 c.Check(err, Equals, keepclient.BlockNotFound)
 257                 c.Log("Finished Ask (expected BlockNotFound)")
 258         }
 259
 260         {
 261                 reader, _, _, err := kc.Get(hash)
 262                 c.Check(reader, Equals, nil)
 263                 c.Check(err, Equals, keepclient.BlockNotFound)
 264                 c.Log("Finished Get (expected BlockNotFound)")
 265         }
 266
 267         // Note in bug #5309 among other errors keepproxy would set
 268         // Content-Length incorrectly on the 404 BlockNotFound response, this
 269         // would result in a protocol violation that would prevent reuse of the
 270         // connection, which would manifest by the next attempt to use the
 271         // connection (in this case the PutB below) failing.  So to test for
 272         // that bug it's necessary to trigger an error response (such as
 273         // BlockNotFound) and then do something else with the same httpClient
 274         // connection.
 275
 276         {
 277                 var rep int
 278                 var err error
 279                 hash2, rep, err = kc.PutB([]byte("foo"))
 280                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 281                 c.Check(rep, Equals, 2)
 282                 c.Check(err, Equals, nil)
 283                 c.Log("Finished PutB (expected success)")
 284         }
 285
 286         {
 287                 blocklen, _, err := kc.Ask(hash2)
 288                 c.Assert(err, Equals, nil)
 289                 c.Check(blocklen, Equals, int64(3))
 290                 c.Log("Finished Ask (expected success)")
 291         }
 292
 293         {
 294                 reader, blocklen, _, err := kc.Get(hash2)
 295                 c.Assert(err, Equals, nil)
 296                 all, err := ioutil.ReadAll(reader)
 297                 c.Check(err, IsNil)
 298                 c.Check(all, DeepEquals, []byte("foo"))
 299                 c.Check(blocklen, Equals, int64(3))
 300                 c.Log("Finished Get (expected success)")
 301         }
 302
 303         {
 304                 var rep int
 305                 var err error
 306                 hash2, rep, err = kc.PutB([]byte(""))
 307                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 308                 c.Check(rep, Equals, 2)
 309                 c.Check(err, Equals, nil)
 310                 c.Log("Finished PutB zero block")
 311         }
 312
 313         {
 314                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 315                 c.Assert(err, Equals, nil)
 316                 all, err := ioutil.ReadAll(reader)
 317                 c.Check(err, IsNil)
 318                 c.Check(all, DeepEquals, []byte(""))
 319                 c.Check(blocklen, Equals, int64(0))
 320                 c.Log("Finished Get zero block")
 321         }
 322 }
 323
 324 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 325         kc := runProxy(c, nil, true)
 326         defer closeListener()
 327
 328         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 329
 330         _, _, err := kc.Ask(hash)
 331         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 332
 333         hash2, rep, err := kc.PutB([]byte("bar"))
 334         c.Check(hash2, Equals, "")
 335         c.Check(rep, Equals, 0)
 336         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 337
 338         blocklen, _, err := kc.Ask(hash)
 339         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 340         c.Check(err, ErrorMatches, ".*not found.*")
 341         c.Check(blocklen, Equals, int64(0))
 342
 343         _, blocklen, _, err = kc.Get(hash)
 344         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 345         c.Check(err, ErrorMatches, ".*not found.*")
 346         c.Check(blocklen, Equals, int64(0))
 347
 348 }
 349
 350 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 351         kc := runProxy(c, []string{"-no-get"}, false)
 352         defer closeListener()
 353
 354         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 355
 356         {
 357                 _, _, err := kc.Ask(hash)
 358                 errNotFound, _ := err.(keepclient.ErrNotFound)
 359                 c.Check(errNotFound, NotNil)
 360                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 361                 c.Log("Ask 1")
 362         }
 363
 364         {
 365                 hash2, rep, err := kc.PutB([]byte("baz"))
 366                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 367                 c.Check(rep, Equals, 2)
 368                 c.Check(err, Equals, nil)
 369                 c.Log("PutB")
 370         }
 371
 372         {
 373                 blocklen, _, err := kc.Ask(hash)
 374                 errNotFound, _ := err.(keepclient.ErrNotFound)
 375                 c.Check(errNotFound, NotNil)
 376                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 377                 c.Check(blocklen, Equals, int64(0))
 378                 c.Log("Ask 2")
 379         }
 380
 381         {
 382                 _, blocklen, _, err := kc.Get(hash)
 383                 errNotFound, _ := err.(keepclient.ErrNotFound)
 384                 c.Check(errNotFound, NotNil)
 385                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 386                 c.Check(blocklen, Equals, int64(0))
 387                 c.Log("Get")
 388         }
 389 }
 390
 391 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 392         kc := runProxy(c, []string{"-no-put"}, false)
 393         defer closeListener()
 394
 395         hash2, rep, err := kc.PutB([]byte("quux"))
 396         c.Check(hash2, Equals, "")
 397         c.Check(rep, Equals, 0)
 398         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 399 }
 400
 401 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 402         runProxy(c, nil, false)
 403         defer closeListener()
 404
 405         {
 406                 client := http.Client{}
 407                 req, err := http.NewRequest("OPTIONS",
 408                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 409                         nil)
 410                 req.Header.Add("Access-Control-Request-Method", "PUT")
 411                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 412                 resp, err := client.Do(req)
 413                 c.Check(err, Equals, nil)
 414                 c.Check(resp.StatusCode, Equals, 200)
 415                 body, err := ioutil.ReadAll(resp.Body)
 416                 c.Check(err, IsNil)
 417                 c.Check(string(body), Equals, "")
 418                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 419                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 420         }
 421
 422         {
 423                 resp, err := http.Get(
 424                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 425                 c.Check(err, Equals, nil)
 426                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 427                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 428         }
 429 }
 430
 431 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 432         runProxy(c, nil, false)
 433         defer closeListener()
 434
 435         {
 436                 client := http.Client{}
 437                 req, err := http.NewRequest("POST",
 438                         "http://"+listener.Addr().String()+"/",
 439                         strings.NewReader("qux"))
 440                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 441                 req.Header.Add("Content-Type", "application/octet-stream")
 442                 resp, err := client.Do(req)
 443                 c.Check(err, Equals, nil)
 444                 body, err := ioutil.ReadAll(resp.Body)
 445                 c.Check(err, Equals, nil)
 446                 c.Check(string(body), Matches,
 447                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 448         }
 449 }
 450
 451 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 452         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 453                 Equals,
 454                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 455         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 456                 Equals,
 457                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 458         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 459                 Equals,
 460                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 461         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 462                 Equals,
 463                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 464
 465 }
 466
 467 // Test GetIndex
 468 //   Put one block, with 2 replicas
 469 //   With no prefix (expect the block locator, twice)
 470 //   With an existing prefix (expect the block locator, twice)
 471 //   With a valid but non-existing prefix (expect "\n")
 472 //   With an invalid prefix (expect error)
 473 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 474         kc := runProxy(c, nil, false)
 475         defer closeListener()
 476
 477         // Put "index-data" blocks
 478         data := []byte("index-data")
 479         hash := fmt.Sprintf("%x", md5.Sum(data))
 480
 481         hash2, rep, err := kc.PutB(data)
 482         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 483         c.Check(rep, Equals, 2)
 484         c.Check(err, Equals, nil)
 485
 486         reader, blocklen, _, err := kc.Get(hash)
 487         c.Assert(err, Equals, nil)
 488         c.Check(blocklen, Equals, int64(10))
 489         all, err := ioutil.ReadAll(reader)
 490         c.Check(all, DeepEquals, data)
 491
 492         // Put some more blocks
 493         _, _, err = kc.PutB([]byte("some-more-index-data"))
 494         c.Check(err, IsNil)
 495
 496         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 497
 498         // Invoke GetIndex
 499         for _, spec := range []struct {
 500                 prefix         string
 501                 expectTestHash bool
 502                 expectOther    bool
 503         }{
 504                 {"", true, true},         // with no prefix
 505                 {hash[:3], true, false},  // with matching prefix
 506                 {"abcdef", false, false}, // with no such prefix
 507         } {
 508                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 509                 c.Assert(err, Equals, nil)
 510                 indexResp, err := ioutil.ReadAll(indexReader)
 511                 c.Assert(err, Equals, nil)
 512                 locators := strings.Split(string(indexResp), "\n")
 513                 gotTestHash := 0
 514                 gotOther := 0
 515                 for _, locator := range locators {
 516                         if locator == "" {
 517                                 continue
 518                         }
 519                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 520                         if locator[:32] == hash {
 521                                 gotTestHash++
 522                         } else {
 523                                 gotOther++
 524                         }
 525                 }
 526                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 527                 c.Check(gotOther > 0, Equals, spec.expectOther)
 528         }
 529
 530         // GetIndex with invalid prefix
 531         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 532         c.Assert((err != nil), Equals, true)
 533 }
 534
 535 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 536         kc := runProxy(c, nil, false)
 537         defer closeListener()
 538         hash, _, err := kc.PutB([]byte("shareddata"))
 539         c.Check(err, IsNil)
 540         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 541         rdr, _, _, err := kc.Get(hash)
 542         c.Assert(err, IsNil)
 543         data, err := ioutil.ReadAll(rdr)
 544         c.Check(err, IsNil)
 545         c.Check(data, DeepEquals, []byte("shareddata"))
 546 }
 547
 548 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 549         kc := runProxy(c, nil, false)
 550         defer closeListener()
 551
 552         // Put a test block
 553         hash, rep, err := kc.PutB([]byte("foo"))
 554         c.Check(err, IsNil)
 555         c.Check(rep, Equals, 2)
 556
 557         for _, badToken := range []string{
 558                 "nosuchtoken",
 559                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 560         } {
 561                 kc.Arvados.ApiToken = badToken
 562
 563                 // Ask and Get will fail only if the upstream
 564                 // keepstore server checks for valid signatures.
 565                 // Without knowing the blob signing key, there is no
 566                 // way for keepproxy to know whether a given token is
 567                 // permitted to read a block.  So these tests fail:
 568                 if false {
 569                         _, _, err = kc.Ask(hash)
 570                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 571                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 572                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 573
 574                         _, _, _, err = kc.Get(hash)
 575                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 576                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 577                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 578                 }
 579
 580                 _, _, err = kc.PutB([]byte("foo"))
 581                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 582         }
 583 }
 584
 585 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 586         arv, err := arvadosclient.MakeArvadosClient()
 587         c.Assert(err, Equals, nil)
 588
 589         // keepclient with no such keep server
 590         kc := keepclient.New(arv)
 591         locals := map[string]string{
 592                 TestProxyUUID: "http://localhost:12345",
 593         }
 594         kc.SetServiceRoots(locals, nil, nil)
 595
 596         // Ask should result in temporary connection refused error
 597         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 598         _, _, err = kc.Ask(hash)
 599         c.Check(err, NotNil)
 600         errNotFound, _ := err.(*keepclient.ErrNotFound)
 601         c.Check(errNotFound.Temporary(), Equals, true)
 602         c.Assert(err, ErrorMatches, ".*connection refused.*")
 603
 604         // Get should result in temporary connection refused error
 605         _, _, _, err = kc.Get(hash)
 606         c.Check(err, NotNil)
 607         errNotFound, _ = err.(*keepclient.ErrNotFound)
 608         c.Check(errNotFound.Temporary(), Equals, true)
 609         c.Assert(err, ErrorMatches, ".*connection refused.*")
 610 }
 611
 612 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 613         kc := runProxy(c, nil, false)
 614         defer closeListener()
 615
 616         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 617         for _, f := range []func() error{
 618                 func() error {
 619                         _, _, err := kc.Ask(hash)
 620                         return err
 621                 },
 622                 func() error {
 623                         _, _, _, err := kc.Get(hash)
 624                         return err
 625                 },
 626         } {
 627                 err := f()
 628                 c.Assert(err, NotNil)
 629                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 630                 c.Check(errNotFound.Temporary(), Equals, true)
 631                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 632         }
 633 }
 634
 635 func (s *ServerRequiredSuite) TestPing(c *C) {
 636         kc := runProxy(c, nil, false)
 637         defer closeListener()
 638
 639         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
 640
 641         req, err := http.NewRequest("GET",
 642                 "http://"+listener.Addr().String()+"/_health/ping",
 643                 nil)
 644         c.Assert(err, IsNil)
 645         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 646
 647         resp := httptest.NewRecorder()
 648         rtr.ServeHTTP(resp, req)
 649         c.Check(resp.Code, Equals, 200)
 650         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 651 }