services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "strings"
  17         "sync"
  18         "testing"
  19         "time"
  20
  21         "git.arvados.org/arvados.git/lib/config"
  22         "git.arvados.org/arvados.git/sdk/go/arvados"
  23         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  24         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         log "github.com/sirupsen/logrus"
  28
  29         . "gopkg.in/check.v1"
  30 )
  31
  32 // Gocheck boilerplate
  33 func Test(t *testing.T) {
  34         TestingT(t)
  35 }
  36
  37 // Gocheck boilerplate
  38 var _ = Suite(&ServerRequiredSuite{})
  39
  40 // Tests that require the Keep server running
  41 type ServerRequiredSuite struct{}
  42
  43 // Gocheck boilerplate
  44 var _ = Suite(&ServerRequiredConfigYmlSuite{})
  45
  46 // Tests that require the Keep servers running as defined in config.yml
  47 type ServerRequiredConfigYmlSuite struct{}
  48
  49 // Gocheck boilerplate
  50 var _ = Suite(&NoKeepServerSuite{})
  51
  52 // Test with no keepserver to simulate errors
  53 type NoKeepServerSuite struct{}
  54
  55 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  56
  57 // Wait (up to 1 second) for keepproxy to listen on a port. This
  58 // avoids a race condition where we hit a "connection refused" error
  59 // because we start testing the proxy too soon.
  60 func waitForListener() {
  61         const (
  62                 ms = 5
  63         )
  64         for i := 0; listener == nil && i < 10000; i += ms {
  65                 time.Sleep(ms * time.Millisecond)
  66         }
  67         if listener == nil {
  68                 panic("Timed out waiting for listener to start")
  69         }
  70 }
  71
  72 func closeListener() {
  73         if listener != nil {
  74                 listener.Close()
  75         }
  76 }
  77
  78 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  79         arvadostest.StartAPI()
  80         arvadostest.StartKeep(2, false)
  81 }
  82
  83 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  84         arvadostest.ResetEnv()
  85 }
  86
  87 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  88         arvadostest.StopKeep(2)
  89         arvadostest.StopAPI()
  90 }
  91
  92 func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
  93         arvadostest.StartAPI()
  94         // config.yml defines 4 keepstores
  95         arvadostest.StartKeep(4, false)
  96 }
  97
  98 func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
  99         arvadostest.ResetEnv()
 100 }
 101
 102 func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
 103         arvadostest.StopKeep(4)
 104         arvadostest.StopAPI()
 105 }
 106
 107 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
 108         arvadostest.StartAPI()
 109         // We need API to have some keep services listed, but the
 110         // services themselves should be unresponsive.
 111         arvadostest.StartKeep(2, false)
 112         arvadostest.StopKeep(2)
 113 }
 114
 115 func (s *NoKeepServerSuite) SetUpTest(c *C) {
 116         arvadostest.ResetEnv()
 117 }
 118
 119 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
 120         arvadostest.StopAPI()
 121 }
 122
 123 func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool) *keepclient.KeepClient {
 124         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
 125         c.Assert(err, Equals, nil)
 126         cluster, err := cfg.GetCluster("")
 127         c.Assert(err, Equals, nil)
 128
 129         if !loadKeepstoresFromConfig {
 130                 // Do not load Keepstore InternalURLs from the config file
 131                 cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
 132         }
 133
 134         cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: ":0"}: {}}
 135
 136         listener = nil
 137         go func() {
 138                 run(log.New(), cluster)
 139                 defer closeListener()
 140         }()
 141         waitForListener()
 142
 143         client := arvados.NewClientFromEnv()
 144         arv, err := arvadosclient.New(client)
 145         c.Assert(err, Equals, nil)
 146         if bogusClientToken {
 147                 arv.ApiToken = "bogus-token"
 148         }
 149         kc := keepclient.New(arv)
 150         sr := map[string]string{
 151                 TestProxyUUID: "http://" + listener.Addr().String(),
 152         }
 153         kc.SetServiceRoots(sr, sr, sr)
 154         kc.Arvados.External = true
 155
 156         return kc
 157 }
 158
 159 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 160         runProxy(c, false, false)
 161         defer closeListener()
 162
 163         req, err := http.NewRequest("POST",
 164                 "http://"+listener.Addr().String()+"/",
 165                 strings.NewReader("TestViaHeader"))
 166         c.Assert(err, Equals, nil)
 167         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 168         resp, err := (&http.Client{}).Do(req)
 169         c.Assert(err, Equals, nil)
 170         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 171         c.Assert(resp.StatusCode, Equals, http.StatusOK)
 172         locator, err := ioutil.ReadAll(resp.Body)
 173         c.Assert(err, Equals, nil)
 174         resp.Body.Close()
 175
 176         req, err = http.NewRequest("GET",
 177                 "http://"+listener.Addr().String()+"/"+string(locator),
 178                 nil)
 179         c.Assert(err, Equals, nil)
 180         resp, err = (&http.Client{}).Do(req)
 181         c.Assert(err, Equals, nil)
 182         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 183         resp.Body.Close()
 184 }
 185
 186 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 187         kc := runProxy(c, false, false)
 188         defer closeListener()
 189
 190         sr := map[string]string{
 191                 TestProxyUUID: "http://" + listener.Addr().String(),
 192         }
 193         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 194
 195         content := []byte("TestLoopDetection")
 196         _, _, err := kc.PutB(content)
 197         c.Check(err, ErrorMatches, `.*loop detected.*`)
 198
 199         hash := fmt.Sprintf("%x", md5.Sum(content))
 200         _, _, _, err = kc.Get(hash)
 201         c.Check(err, ErrorMatches, `.*loop detected.*`)
 202 }
 203
 204 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 205         kc := runProxy(c, false, false)
 206         defer closeListener()
 207
 208         // Set up fake keepstore to record request headers
 209         var hdr http.Header
 210         ts := httptest.NewServer(http.HandlerFunc(
 211                 func(w http.ResponseWriter, r *http.Request) {
 212                         hdr = r.Header
 213                         http.Error(w, "Error", http.StatusInternalServerError)
 214                 }))
 215         defer ts.Close()
 216
 217         // Point keepproxy router's keepclient to the fake keepstore
 218         sr := map[string]string{
 219                 TestProxyUUID: ts.URL,
 220         }
 221         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 222
 223         // Set up client to ask for storage classes to keepproxy
 224         kc.StorageClasses = []string{"secure"}
 225         content := []byte("Very important data")
 226         _, _, err := kc.PutB(content)
 227         c.Check(err, NotNil)
 228         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 229 }
 230
 231 func (s *ServerRequiredSuite) TestStorageClassesConfirmedHeader(c *C) {
 232         runProxy(c, false, false)
 233         defer closeListener()
 234
 235         content := []byte("foo")
 236         hash := fmt.Sprintf("%x", md5.Sum(content))
 237         client := &http.Client{}
 238
 239         req, err := http.NewRequest("PUT",
 240                 fmt.Sprintf("http://%s/%s", listener.Addr().String(), hash),
 241                 bytes.NewReader(content))
 242         c.Assert(err, IsNil)
 243         req.Header.Set("X-Keep-Storage-Classes", "default")
 244         req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 245         req.Header.Set("Content-Type", "application/octet-stream")
 246
 247         resp, err := client.Do(req)
 248         c.Assert(err, IsNil)
 249         c.Assert(resp.StatusCode, Equals, http.StatusOK)
 250         c.Assert(resp.Header.Get("X-Keep-Storage-Classes-Confirmed"), Equals, "default=2")
 251 }
 252
 253 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 254         kc := runProxy(c, false, false)
 255         defer closeListener()
 256
 257         content := []byte("TestDesiredReplicas")
 258         hash := fmt.Sprintf("%x", md5.Sum(content))
 259
 260         for _, kc.Want_replicas = range []int{0, 1, 2} {
 261                 locator, rep, err := kc.PutB(content)
 262                 c.Check(err, Equals, nil)
 263                 c.Check(rep, Equals, kc.Want_replicas)
 264                 if rep > 0 {
 265                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 266                 }
 267         }
 268 }
 269
 270 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 271         kc := runProxy(c, false, false)
 272         defer closeListener()
 273
 274         content := []byte("TestPutWrongContentLength")
 275         hash := fmt.Sprintf("%x", md5.Sum(content))
 276
 277         // If we use http.Client to send these requests to the network
 278         // server we just started, the Go http library automatically
 279         // fixes the invalid Content-Length header. In order to test
 280         // our server behavior, we have to call the handler directly
 281         // using an httptest.ResponseRecorder.
 282         rtr := MakeRESTRouter(kc, 10*time.Second, "")
 283
 284         type testcase struct {
 285                 sendLength   string
 286                 expectStatus int
 287         }
 288
 289         for _, t := range []testcase{
 290                 {"1", http.StatusBadRequest},
 291                 {"", http.StatusLengthRequired},
 292                 {"-1", http.StatusLengthRequired},
 293                 {"abcdef", http.StatusLengthRequired},
 294         } {
 295                 req, err := http.NewRequest("PUT",
 296                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 297                         bytes.NewReader(content))
 298                 c.Assert(err, IsNil)
 299                 req.Header.Set("Content-Length", t.sendLength)
 300                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 301                 req.Header.Set("Content-Type", "application/octet-stream")
 302
 303                 resp := httptest.NewRecorder()
 304                 rtr.ServeHTTP(resp, req)
 305                 c.Check(resp.Code, Equals, t.expectStatus)
 306         }
 307 }
 308
 309 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 310         kc := runProxy(c, false, false)
 311         defer closeListener()
 312         router.(*proxyHandler).timeout = time.Nanosecond
 313
 314         buf := make([]byte, 1<<20)
 315         rand.Read(buf)
 316         var wg sync.WaitGroup
 317         for i := 0; i < 128; i++ {
 318                 wg.Add(1)
 319                 go func() {
 320                         defer wg.Done()
 321                         kc.PutB(buf)
 322                 }()
 323         }
 324         done := make(chan bool)
 325         go func() {
 326                 wg.Wait()
 327                 close(done)
 328         }()
 329         select {
 330         case <-done:
 331         case <-time.After(10 * time.Second):
 332                 c.Error("timeout")
 333         }
 334 }
 335
 336 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 337         kc := runProxy(c, false, false)
 338         defer closeListener()
 339
 340         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 341         var hash2 string
 342
 343         {
 344                 _, _, err := kc.Ask(hash)
 345                 c.Check(err, Equals, keepclient.BlockNotFound)
 346                 c.Log("Finished Ask (expected BlockNotFound)")
 347         }
 348
 349         {
 350                 reader, _, _, err := kc.Get(hash)
 351                 c.Check(reader, Equals, nil)
 352                 c.Check(err, Equals, keepclient.BlockNotFound)
 353                 c.Log("Finished Get (expected BlockNotFound)")
 354         }
 355
 356         // Note in bug #5309 among other errors keepproxy would set
 357         // Content-Length incorrectly on the 404 BlockNotFound response, this
 358         // would result in a protocol violation that would prevent reuse of the
 359         // connection, which would manifest by the next attempt to use the
 360         // connection (in this case the PutB below) failing.  So to test for
 361         // that bug it's necessary to trigger an error response (such as
 362         // BlockNotFound) and then do something else with the same httpClient
 363         // connection.
 364
 365         {
 366                 var rep int
 367                 var err error
 368                 hash2, rep, err = kc.PutB([]byte("foo"))
 369                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 370                 c.Check(rep, Equals, 2)
 371                 c.Check(err, Equals, nil)
 372                 c.Log("Finished PutB (expected success)")
 373         }
 374
 375         {
 376                 blocklen, _, err := kc.Ask(hash2)
 377                 c.Assert(err, Equals, nil)
 378                 c.Check(blocklen, Equals, int64(3))
 379                 c.Log("Finished Ask (expected success)")
 380         }
 381
 382         {
 383                 reader, blocklen, _, err := kc.Get(hash2)
 384                 c.Assert(err, Equals, nil)
 385                 all, err := ioutil.ReadAll(reader)
 386                 c.Check(err, IsNil)
 387                 c.Check(all, DeepEquals, []byte("foo"))
 388                 c.Check(blocklen, Equals, int64(3))
 389                 c.Log("Finished Get (expected success)")
 390         }
 391
 392         {
 393                 var rep int
 394                 var err error
 395                 hash2, rep, err = kc.PutB([]byte(""))
 396                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 397                 c.Check(rep, Equals, 2)
 398                 c.Check(err, Equals, nil)
 399                 c.Log("Finished PutB zero block")
 400         }
 401
 402         {
 403                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 404                 c.Assert(err, Equals, nil)
 405                 all, err := ioutil.ReadAll(reader)
 406                 c.Check(err, IsNil)
 407                 c.Check(all, DeepEquals, []byte(""))
 408                 c.Check(blocklen, Equals, int64(0))
 409                 c.Log("Finished Get zero block")
 410         }
 411 }
 412
 413 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 414         kc := runProxy(c, true, false)
 415         defer closeListener()
 416
 417         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 418
 419         _, _, err := kc.Ask(hash)
 420         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 421
 422         hash2, rep, err := kc.PutB([]byte("bar"))
 423         c.Check(hash2, Equals, "")
 424         c.Check(rep, Equals, 0)
 425         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 426
 427         blocklen, _, err := kc.Ask(hash)
 428         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 429         c.Check(err, ErrorMatches, ".*not found.*")
 430         c.Check(blocklen, Equals, int64(0))
 431
 432         _, blocklen, _, err = kc.Get(hash)
 433         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 434         c.Check(err, ErrorMatches, ".*not found.*")
 435         c.Check(blocklen, Equals, int64(0))
 436
 437 }
 438
 439 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 440         runProxy(c, false, false)
 441         defer closeListener()
 442
 443         {
 444                 client := http.Client{}
 445                 req, err := http.NewRequest("OPTIONS",
 446                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 447                         nil)
 448                 c.Assert(err, IsNil)
 449                 req.Header.Add("Access-Control-Request-Method", "PUT")
 450                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 451                 resp, err := client.Do(req)
 452                 c.Check(err, Equals, nil)
 453                 c.Check(resp.StatusCode, Equals, 200)
 454                 body, err := ioutil.ReadAll(resp.Body)
 455                 c.Check(err, IsNil)
 456                 c.Check(string(body), Equals, "")
 457                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 458                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 459         }
 460
 461         {
 462                 resp, err := http.Get(
 463                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 464                 c.Check(err, Equals, nil)
 465                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 466                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 467         }
 468 }
 469
 470 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 471         runProxy(c, false, false)
 472         defer closeListener()
 473
 474         {
 475                 client := http.Client{}
 476                 req, err := http.NewRequest("POST",
 477                         "http://"+listener.Addr().String()+"/",
 478                         strings.NewReader("qux"))
 479                 c.Check(err, IsNil)
 480                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 481                 req.Header.Add("Content-Type", "application/octet-stream")
 482                 resp, err := client.Do(req)
 483                 c.Check(err, Equals, nil)
 484                 body, err := ioutil.ReadAll(resp.Body)
 485                 c.Check(err, Equals, nil)
 486                 c.Check(string(body), Matches,
 487                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 488         }
 489 }
 490
 491 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 492         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 493                 Equals,
 494                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 495         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 496                 Equals,
 497                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 498         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 499                 Equals,
 500                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 501         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 502                 Equals,
 503                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 504
 505 }
 506
 507 // Test GetIndex
 508 //   Put one block, with 2 replicas
 509 //   With no prefix (expect the block locator, twice)
 510 //   With an existing prefix (expect the block locator, twice)
 511 //   With a valid but non-existing prefix (expect "\n")
 512 //   With an invalid prefix (expect error)
 513 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 514         getIndexWorker(c, false)
 515 }
 516
 517 // Test GetIndex
 518 //   Uses config.yml
 519 //   Put one block, with 2 replicas
 520 //   With no prefix (expect the block locator, twice)
 521 //   With an existing prefix (expect the block locator, twice)
 522 //   With a valid but non-existing prefix (expect "\n")
 523 //   With an invalid prefix (expect error)
 524 func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
 525         getIndexWorker(c, true)
 526 }
 527
 528 func getIndexWorker(c *C, useConfig bool) {
 529         kc := runProxy(c, false, useConfig)
 530         defer closeListener()
 531
 532         // Put "index-data" blocks
 533         data := []byte("index-data")
 534         hash := fmt.Sprintf("%x", md5.Sum(data))
 535
 536         hash2, rep, err := kc.PutB(data)
 537         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 538         c.Check(rep, Equals, 2)
 539         c.Check(err, Equals, nil)
 540
 541         reader, blocklen, _, err := kc.Get(hash)
 542         c.Assert(err, IsNil)
 543         c.Check(blocklen, Equals, int64(10))
 544         all, err := ioutil.ReadAll(reader)
 545         c.Assert(err, IsNil)
 546         c.Check(all, DeepEquals, data)
 547
 548         // Put some more blocks
 549         _, _, err = kc.PutB([]byte("some-more-index-data"))
 550         c.Check(err, IsNil)
 551
 552         kc.Arvados.ApiToken = arvadostest.SystemRootToken
 553
 554         // Invoke GetIndex
 555         for _, spec := range []struct {
 556                 prefix         string
 557                 expectTestHash bool
 558                 expectOther    bool
 559         }{
 560                 {"", true, true},         // with no prefix
 561                 {hash[:3], true, false},  // with matching prefix
 562                 {"abcdef", false, false}, // with no such prefix
 563         } {
 564                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 565                 c.Assert(err, Equals, nil)
 566                 indexResp, err := ioutil.ReadAll(indexReader)
 567                 c.Assert(err, Equals, nil)
 568                 locators := strings.Split(string(indexResp), "\n")
 569                 gotTestHash := 0
 570                 gotOther := 0
 571                 for _, locator := range locators {
 572                         if locator == "" {
 573                                 continue
 574                         }
 575                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 576                         if locator[:32] == hash {
 577                                 gotTestHash++
 578                         } else {
 579                                 gotOther++
 580                         }
 581                 }
 582                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 583                 c.Check(gotOther > 0, Equals, spec.expectOther)
 584         }
 585
 586         // GetIndex with invalid prefix
 587         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 588         c.Assert((err != nil), Equals, true)
 589 }
 590
 591 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 592         kc := runProxy(c, false, false)
 593         defer closeListener()
 594         hash, _, err := kc.PutB([]byte("shareddata"))
 595         c.Check(err, IsNil)
 596         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 597         rdr, _, _, err := kc.Get(hash)
 598         c.Assert(err, IsNil)
 599         data, err := ioutil.ReadAll(rdr)
 600         c.Check(err, IsNil)
 601         c.Check(data, DeepEquals, []byte("shareddata"))
 602 }
 603
 604 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 605         kc := runProxy(c, false, false)
 606         defer closeListener()
 607
 608         // Put a test block
 609         hash, rep, err := kc.PutB([]byte("foo"))
 610         c.Check(err, IsNil)
 611         c.Check(rep, Equals, 2)
 612
 613         for _, badToken := range []string{
 614                 "nosuchtoken",
 615                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 616         } {
 617                 kc.Arvados.ApiToken = badToken
 618
 619                 // Ask and Get will fail only if the upstream
 620                 // keepstore server checks for valid signatures.
 621                 // Without knowing the blob signing key, there is no
 622                 // way for keepproxy to know whether a given token is
 623                 // permitted to read a block.  So these tests fail:
 624                 if false {
 625                         _, _, err = kc.Ask(hash)
 626                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 627                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 628                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 629
 630                         _, _, _, err = kc.Get(hash)
 631                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 632                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 633                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 634                 }
 635
 636                 _, _, err = kc.PutB([]byte("foo"))
 637                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 638         }
 639 }
 640
 641 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 642         kc := runProxy(c, false, false)
 643         defer closeListener()
 644
 645         // Point keepproxy at a non-existent keepstore
 646         locals := map[string]string{
 647                 TestProxyUUID: "http://localhost:12345",
 648         }
 649         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
 650
 651         // Ask should result in temporary bad gateway error
 652         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 653         _, _, err := kc.Ask(hash)
 654         c.Check(err, NotNil)
 655         errNotFound, _ := err.(*keepclient.ErrNotFound)
 656         c.Check(errNotFound.Temporary(), Equals, true)
 657         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 658
 659         // Get should result in temporary bad gateway error
 660         _, _, _, err = kc.Get(hash)
 661         c.Check(err, NotNil)
 662         errNotFound, _ = err.(*keepclient.ErrNotFound)
 663         c.Check(errNotFound.Temporary(), Equals, true)
 664         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 665 }
 666
 667 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 668         kc := runProxy(c, false, false)
 669         defer closeListener()
 670
 671         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 672         for _, f := range []func() error{
 673                 func() error {
 674                         _, _, err := kc.Ask(hash)
 675                         return err
 676                 },
 677                 func() error {
 678                         _, _, _, err := kc.Get(hash)
 679                         return err
 680                 },
 681         } {
 682                 err := f()
 683                 c.Assert(err, NotNil)
 684                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 685                 c.Check(errNotFound.Temporary(), Equals, true)
 686                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 687         }
 688 }
 689
 690 func (s *ServerRequiredSuite) TestPing(c *C) {
 691         kc := runProxy(c, false, false)
 692         defer closeListener()
 693
 694         rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
 695
 696         req, err := http.NewRequest("GET",
 697                 "http://"+listener.Addr().String()+"/_health/ping",
 698                 nil)
 699         c.Assert(err, IsNil)
 700         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 701
 702         resp := httptest.NewRecorder()
 703         rtr.ServeHTTP(resp, req)
 704         c.Check(resp.Code, Equals, 200)
 705         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 706 }