services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "os"
  17         "strings"
  18         "sync"
  19         "testing"
  20         "time"
  21
  22         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25
  26         . "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 func Test(t *testing.T) {
  31         TestingT(t)
  32 }
  33
  34 // Gocheck boilerplate
  35 var _ = Suite(&ServerRequiredSuite{})
  36
  37 // Tests that require the Keep server running
  38 type ServerRequiredSuite struct{}
  39
  40 // Gocheck boilerplate
  41 var _ = Suite(&NoKeepServerSuite{})
  42
  43 // Test with no keepserver to simulate errors
  44 type NoKeepServerSuite struct{}
  45
  46 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  47
  48 // Wait (up to 1 second) for keepproxy to listen on a port. This
  49 // avoids a race condition where we hit a "connection refused" error
  50 // because we start testing the proxy too soon.
  51 func waitForListener() {
  52         const (
  53                 ms = 5
  54         )
  55         for i := 0; listener == nil && i < 10000; i += ms {
  56                 time.Sleep(ms * time.Millisecond)
  57         }
  58         if listener == nil {
  59                 panic("Timed out waiting for listener to start")
  60         }
  61 }
  62
  63 func closeListener() {
  64         if listener != nil {
  65                 listener.Close()
  66         }
  67 }
  68
  69 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  70         arvadostest.StartAPI()
  71         arvadostest.StartKeep(2, false)
  72 }
  73
  74 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  75         arvadostest.ResetEnv()
  76 }
  77
  78 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  79         arvadostest.StopKeep(2)
  80         arvadostest.StopAPI()
  81 }
  82
  83 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  84         arvadostest.StartAPI()
  85         // We need API to have some keep services listed, but the
  86         // services themselves should be unresponsive.
  87         arvadostest.StartKeep(2, false)
  88         arvadostest.StopKeep(2)
  89 }
  90
  91 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  92         arvadostest.ResetEnv()
  93 }
  94
  95 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  96         arvadostest.StopAPI()
  97 }
  98
  99 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
 100         args = append([]string{"keepproxy"}, args...)
 101         os.Args = append(args, "-listen=:0")
 102         listener = nil
 103         go main()
 104         waitForListener()
 105
 106         arv, err := arvadosclient.MakeArvadosClient()
 107         c.Assert(err, Equals, nil)
 108         if bogusClientToken {
 109                 arv.ApiToken = "bogus-token"
 110         }
 111         kc := keepclient.New(arv)
 112         sr := map[string]string{
 113                 TestProxyUUID: "http://" + listener.Addr().String(),
 114         }
 115         kc.SetServiceRoots(sr, sr, sr)
 116         kc.Arvados.External = true
 117
 118         return kc
 119 }
 120
 121 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 122         runProxy(c, nil, false)
 123         defer closeListener()
 124
 125         req, err := http.NewRequest("POST",
 126                 "http://"+listener.Addr().String()+"/",
 127                 strings.NewReader("TestViaHeader"))
 128         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 129         resp, err := (&http.Client{}).Do(req)
 130         c.Assert(err, Equals, nil)
 131         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 132         locator, err := ioutil.ReadAll(resp.Body)
 133         c.Assert(err, Equals, nil)
 134         resp.Body.Close()
 135
 136         req, err = http.NewRequest("GET",
 137                 "http://"+listener.Addr().String()+"/"+string(locator),
 138                 nil)
 139         c.Assert(err, Equals, nil)
 140         resp, err = (&http.Client{}).Do(req)
 141         c.Assert(err, Equals, nil)
 142         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 143         resp.Body.Close()
 144 }
 145
 146 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 147         kc := runProxy(c, nil, false)
 148         defer closeListener()
 149
 150         sr := map[string]string{
 151                 TestProxyUUID: "http://" + listener.Addr().String(),
 152         }
 153         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 154
 155         content := []byte("TestLoopDetection")
 156         _, _, err := kc.PutB(content)
 157         c.Check(err, ErrorMatches, `.*loop detected.*`)
 158
 159         hash := fmt.Sprintf("%x", md5.Sum(content))
 160         _, _, _, err = kc.Get(hash)
 161         c.Check(err, ErrorMatches, `.*loop detected.*`)
 162 }
 163
 164 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 165         kc := runProxy(c, nil, false)
 166         defer closeListener()
 167
 168         content := []byte("TestDesiredReplicas")
 169         hash := fmt.Sprintf("%x", md5.Sum(content))
 170
 171         for _, kc.Want_replicas = range []int{0, 1, 2} {
 172                 locator, rep, err := kc.PutB(content)
 173                 c.Check(err, Equals, nil)
 174                 c.Check(rep, Equals, kc.Want_replicas)
 175                 if rep > 0 {
 176                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 177                 }
 178         }
 179 }
 180
 181 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 182         kc := runProxy(c, nil, false)
 183         defer closeListener()
 184
 185         content := []byte("TestPutWrongContentLength")
 186         hash := fmt.Sprintf("%x", md5.Sum(content))
 187
 188         // If we use http.Client to send these requests to the network
 189         // server we just started, the Go http library automatically
 190         // fixes the invalid Content-Length header. In order to test
 191         // our server behavior, we have to call the handler directly
 192         // using an httptest.ResponseRecorder.
 193         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
 194
 195         type testcase struct {
 196                 sendLength   string
 197                 expectStatus int
 198         }
 199
 200         for _, t := range []testcase{
 201                 {"1", http.StatusBadRequest},
 202                 {"", http.StatusLengthRequired},
 203                 {"-1", http.StatusLengthRequired},
 204                 {"abcdef", http.StatusLengthRequired},
 205         } {
 206                 req, err := http.NewRequest("PUT",
 207                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 208                         bytes.NewReader(content))
 209                 c.Assert(err, IsNil)
 210                 req.Header.Set("Content-Length", t.sendLength)
 211                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 212                 req.Header.Set("Content-Type", "application/octet-stream")
 213
 214                 resp := httptest.NewRecorder()
 215                 rtr.ServeHTTP(resp, req)
 216                 c.Check(resp.Code, Equals, t.expectStatus)
 217         }
 218 }
 219
 220 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 221         kc := runProxy(c, nil, false)
 222         defer closeListener()
 223         router.(*proxyHandler).timeout = time.Nanosecond
 224
 225         buf := make([]byte, 1<<20)
 226         rand.Read(buf)
 227         var wg sync.WaitGroup
 228         for i := 0; i < 128; i++ {
 229                 wg.Add(1)
 230                 go func() {
 231                         defer wg.Done()
 232                         kc.PutB(buf)
 233                 }()
 234         }
 235         done := make(chan bool)
 236         go func() {
 237                 wg.Wait()
 238                 close(done)
 239         }()
 240         select {
 241         case <-done:
 242         case <-time.After(10 * time.Second):
 243                 c.Error("timeout")
 244         }
 245 }
 246
 247 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 248         kc := runProxy(c, nil, false)
 249         defer closeListener()
 250
 251         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 252         var hash2 string
 253
 254         {
 255                 _, _, err := kc.Ask(hash)
 256                 c.Check(err, Equals, keepclient.BlockNotFound)
 257                 c.Log("Finished Ask (expected BlockNotFound)")
 258         }
 259
 260         {
 261                 reader, _, _, err := kc.Get(hash)
 262                 c.Check(reader, Equals, nil)
 263                 c.Check(err, Equals, keepclient.BlockNotFound)
 264                 c.Log("Finished Get (expected BlockNotFound)")
 265         }
 266
 267         // Note in bug #5309 among other errors keepproxy would set
 268         // Content-Length incorrectly on the 404 BlockNotFound response, this
 269         // would result in a protocol violation that would prevent reuse of the
 270         // connection, which would manifest by the next attempt to use the
 271         // connection (in this case the PutB below) failing.  So to test for
 272         // that bug it's necessary to trigger an error response (such as
 273         // BlockNotFound) and then do something else with the same httpClient
 274         // connection.
 275
 276         {
 277                 var rep int
 278                 var err error
 279                 hash2, rep, err = kc.PutB([]byte("foo"))
 280                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 281                 c.Check(rep, Equals, 2)
 282                 c.Check(err, Equals, nil)
 283                 c.Log("Finished PutB (expected success)")
 284         }
 285
 286         {
 287                 blocklen, _, err := kc.Ask(hash2)
 288                 c.Assert(err, Equals, nil)
 289                 c.Check(blocklen, Equals, int64(3))
 290                 c.Log("Finished Ask (expected success)")
 291         }
 292
 293         {
 294                 reader, blocklen, _, err := kc.Get(hash2)
 295                 c.Assert(err, Equals, nil)
 296                 all, err := ioutil.ReadAll(reader)
 297                 c.Check(all, DeepEquals, []byte("foo"))
 298                 c.Check(blocklen, Equals, int64(3))
 299                 c.Log("Finished Get (expected success)")
 300         }
 301
 302         {
 303                 var rep int
 304                 var err error
 305                 hash2, rep, err = kc.PutB([]byte(""))
 306                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 307                 c.Check(rep, Equals, 2)
 308                 c.Check(err, Equals, nil)
 309                 c.Log("Finished PutB zero block")
 310         }
 311
 312         {
 313                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 314                 c.Assert(err, Equals, nil)
 315                 all, err := ioutil.ReadAll(reader)
 316                 c.Check(all, DeepEquals, []byte(""))
 317                 c.Check(blocklen, Equals, int64(0))
 318                 c.Log("Finished Get zero block")
 319         }
 320 }
 321
 322 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 323         kc := runProxy(c, nil, true)
 324         defer closeListener()
 325
 326         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 327
 328         _, _, err := kc.Ask(hash)
 329         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 330
 331         hash2, rep, err := kc.PutB([]byte("bar"))
 332         c.Check(hash2, Equals, "")
 333         c.Check(rep, Equals, 0)
 334         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 335
 336         blocklen, _, err := kc.Ask(hash)
 337         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 338         c.Check(err, ErrorMatches, ".*not found.*")
 339         c.Check(blocklen, Equals, int64(0))
 340
 341         _, blocklen, _, err = kc.Get(hash)
 342         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 343         c.Check(err, ErrorMatches, ".*not found.*")
 344         c.Check(blocklen, Equals, int64(0))
 345
 346 }
 347
 348 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 349         kc := runProxy(c, []string{"-no-get"}, false)
 350         defer closeListener()
 351
 352         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 353
 354         {
 355                 _, _, err := kc.Ask(hash)
 356                 errNotFound, _ := err.(keepclient.ErrNotFound)
 357                 c.Check(errNotFound, NotNil)
 358                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 359                 c.Log("Ask 1")
 360         }
 361
 362         {
 363                 hash2, rep, err := kc.PutB([]byte("baz"))
 364                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 365                 c.Check(rep, Equals, 2)
 366                 c.Check(err, Equals, nil)
 367                 c.Log("PutB")
 368         }
 369
 370         {
 371                 blocklen, _, err := kc.Ask(hash)
 372                 errNotFound, _ := err.(keepclient.ErrNotFound)
 373                 c.Check(errNotFound, NotNil)
 374                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 375                 c.Check(blocklen, Equals, int64(0))
 376                 c.Log("Ask 2")
 377         }
 378
 379         {
 380                 _, blocklen, _, err := kc.Get(hash)
 381                 errNotFound, _ := err.(keepclient.ErrNotFound)
 382                 c.Check(errNotFound, NotNil)
 383                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 384                 c.Check(blocklen, Equals, int64(0))
 385                 c.Log("Get")
 386         }
 387 }
 388
 389 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 390         kc := runProxy(c, []string{"-no-put"}, false)
 391         defer closeListener()
 392
 393         hash2, rep, err := kc.PutB([]byte("quux"))
 394         c.Check(hash2, Equals, "")
 395         c.Check(rep, Equals, 0)
 396         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 397 }
 398
 399 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 400         runProxy(c, nil, false)
 401         defer closeListener()
 402
 403         {
 404                 client := http.Client{}
 405                 req, err := http.NewRequest("OPTIONS",
 406                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 407                         nil)
 408                 req.Header.Add("Access-Control-Request-Method", "PUT")
 409                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 410                 resp, err := client.Do(req)
 411                 c.Check(err, Equals, nil)
 412                 c.Check(resp.StatusCode, Equals, 200)
 413                 body, err := ioutil.ReadAll(resp.Body)
 414                 c.Check(string(body), Equals, "")
 415                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 416                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 417         }
 418
 419         {
 420                 resp, err := http.Get(
 421                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 422                 c.Check(err, Equals, nil)
 423                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 424                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 425         }
 426 }
 427
 428 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 429         runProxy(c, nil, false)
 430         defer closeListener()
 431
 432         {
 433                 client := http.Client{}
 434                 req, err := http.NewRequest("POST",
 435                         "http://"+listener.Addr().String()+"/",
 436                         strings.NewReader("qux"))
 437                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 438                 req.Header.Add("Content-Type", "application/octet-stream")
 439                 resp, err := client.Do(req)
 440                 c.Check(err, Equals, nil)
 441                 body, err := ioutil.ReadAll(resp.Body)
 442                 c.Check(err, Equals, nil)
 443                 c.Check(string(body), Matches,
 444                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 445         }
 446 }
 447
 448 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 449         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 450                 Equals,
 451                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 452         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 453                 Equals,
 454                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 455         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 456                 Equals,
 457                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 458         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 459                 Equals,
 460                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 461
 462 }
 463
 464 // Test GetIndex
 465 //   Put one block, with 2 replicas
 466 //   With no prefix (expect the block locator, twice)
 467 //   With an existing prefix (expect the block locator, twice)
 468 //   With a valid but non-existing prefix (expect "\n")
 469 //   With an invalid prefix (expect error)
 470 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 471         kc := runProxy(c, nil, false)
 472         defer closeListener()
 473
 474         // Put "index-data" blocks
 475         data := []byte("index-data")
 476         hash := fmt.Sprintf("%x", md5.Sum(data))
 477
 478         hash2, rep, err := kc.PutB(data)
 479         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 480         c.Check(rep, Equals, 2)
 481         c.Check(err, Equals, nil)
 482
 483         reader, blocklen, _, err := kc.Get(hash)
 484         c.Assert(err, Equals, nil)
 485         c.Check(blocklen, Equals, int64(10))
 486         all, err := ioutil.ReadAll(reader)
 487         c.Check(all, DeepEquals, data)
 488
 489         // Put some more blocks
 490         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 491         c.Check(err, Equals, nil)
 492
 493         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 494
 495         // Invoke GetIndex
 496         for _, spec := range []struct {
 497                 prefix         string
 498                 expectTestHash bool
 499                 expectOther    bool
 500         }{
 501                 {"", true, true},         // with no prefix
 502                 {hash[:3], true, false},  // with matching prefix
 503                 {"abcdef", false, false}, // with no such prefix
 504         } {
 505                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 506                 c.Assert(err, Equals, nil)
 507                 indexResp, err := ioutil.ReadAll(indexReader)
 508                 c.Assert(err, Equals, nil)
 509                 locators := strings.Split(string(indexResp), "\n")
 510                 gotTestHash := 0
 511                 gotOther := 0
 512                 for _, locator := range locators {
 513                         if locator == "" {
 514                                 continue
 515                         }
 516                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 517                         if locator[:32] == hash {
 518                                 gotTestHash++
 519                         } else {
 520                                 gotOther++
 521                         }
 522                 }
 523                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 524                 c.Check(gotOther > 0, Equals, spec.expectOther)
 525         }
 526
 527         // GetIndex with invalid prefix
 528         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 529         c.Assert((err != nil), Equals, true)
 530 }
 531
 532 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 533         kc := runProxy(c, nil, false)
 534         defer closeListener()
 535         hash, _, err := kc.PutB([]byte("shareddata"))
 536         c.Check(err, IsNil)
 537         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 538         rdr, _, _, err := kc.Get(hash)
 539         c.Assert(err, IsNil)
 540         data, err := ioutil.ReadAll(rdr)
 541         c.Check(err, IsNil)
 542         c.Check(data, DeepEquals, []byte("shareddata"))
 543 }
 544
 545 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 546         kc := runProxy(c, nil, false)
 547         defer closeListener()
 548
 549         // Put a test block
 550         hash, rep, err := kc.PutB([]byte("foo"))
 551         c.Check(err, IsNil)
 552         c.Check(rep, Equals, 2)
 553
 554         for _, badToken := range []string{
 555                 "nosuchtoken",
 556                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 557         } {
 558                 kc.Arvados.ApiToken = badToken
 559
 560                 // Ask and Get will fail only if the upstream
 561                 // keepstore server checks for valid signatures.
 562                 // Without knowing the blob signing key, there is no
 563                 // way for keepproxy to know whether a given token is
 564                 // permitted to read a block.  So these tests fail:
 565                 if false {
 566                         _, _, err = kc.Ask(hash)
 567                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 568                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 569                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 570
 571                         _, _, _, err = kc.Get(hash)
 572                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 573                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 574                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 575                 }
 576
 577                 _, _, err = kc.PutB([]byte("foo"))
 578                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 579         }
 580 }
 581
 582 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 583         arv, err := arvadosclient.MakeArvadosClient()
 584         c.Assert(err, Equals, nil)
 585
 586         // keepclient with no such keep server
 587         kc := keepclient.New(arv)
 588         locals := map[string]string{
 589                 TestProxyUUID: "http://localhost:12345",
 590         }
 591         kc.SetServiceRoots(locals, nil, nil)
 592
 593         // Ask should result in temporary connection refused error
 594         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 595         _, _, err = kc.Ask(hash)
 596         c.Check(err, NotNil)
 597         errNotFound, _ := err.(*keepclient.ErrNotFound)
 598         c.Check(errNotFound.Temporary(), Equals, true)
 599         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 600
 601         // Get should result in temporary connection refused error
 602         _, _, _, err = kc.Get(hash)
 603         c.Check(err, NotNil)
 604         errNotFound, _ = err.(*keepclient.ErrNotFound)
 605         c.Check(errNotFound.Temporary(), Equals, true)
 606         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 607 }
 608
 609 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 610         kc := runProxy(c, nil, false)
 611         defer closeListener()
 612
 613         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 614         for _, f := range []func() error{
 615                 func() error {
 616                         _, _, err := kc.Ask(hash)
 617                         return err
 618                 },
 619                 func() error {
 620                         _, _, _, err := kc.Get(hash)
 621                         return err
 622                 },
 623         } {
 624                 err := f()
 625                 c.Assert(err, NotNil)
 626                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 627                 c.Check(errNotFound.Temporary(), Equals, true)
 628                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 629         }
 630 }
 631
 632 func (s *ServerRequiredSuite) TestPing(c *C) {
 633         kc := runProxy(c, nil, false)
 634         defer closeListener()
 635
 636         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
 637
 638         req, err := http.NewRequest("GET",
 639                 "http://"+listener.Addr().String()+"/_health/ping",
 640                 nil)
 641         c.Assert(err, IsNil)
 642         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 643
 644         resp := httptest.NewRecorder()
 645         rtr.ServeHTTP(resp, req)
 646         c.Check(resp.Code, Equals, 200)
 647         c.Assert(strings.Contains(resp.Body.String(), `{"health":"OK"}`), Equals, true)
 648 }