services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "strings"
  17         "sync"
  18         "testing"
  19         "time"
  20
  21         "git.arvados.org/arvados.git/lib/config"
  22         "git.arvados.org/arvados.git/sdk/go/arvados"
  23         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  24         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         log "github.com/sirupsen/logrus"
  28
  29         . "gopkg.in/check.v1"
  30 )
  31
  32 // Gocheck boilerplate
  33 func Test(t *testing.T) {
  34         TestingT(t)
  35 }
  36
  37 // Gocheck boilerplate
  38 var _ = Suite(&ServerRequiredSuite{})
  39
  40 // Tests that require the Keep server running
  41 type ServerRequiredSuite struct{}
  42
  43 // Gocheck boilerplate
  44 var _ = Suite(&ServerRequiredConfigYmlSuite{})
  45
  46 // Tests that require the Keep servers running as defined in config.yml
  47 type ServerRequiredConfigYmlSuite struct{}
  48
  49 // Gocheck boilerplate
  50 var _ = Suite(&NoKeepServerSuite{})
  51
  52 // Test with no keepserver to simulate errors
  53 type NoKeepServerSuite struct{}
  54
  55 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  56
  57 // Wait (up to 1 second) for keepproxy to listen on a port. This
  58 // avoids a race condition where we hit a "connection refused" error
  59 // because we start testing the proxy too soon.
  60 func waitForListener() {
  61         const (
  62                 ms = 5
  63         )
  64         for i := 0; listener == nil && i < 10000; i += ms {
  65                 time.Sleep(ms * time.Millisecond)
  66         }
  67         if listener == nil {
  68                 panic("Timed out waiting for listener to start")
  69         }
  70 }
  71
  72 func closeListener() {
  73         if listener != nil {
  74                 listener.Close()
  75         }
  76 }
  77
  78 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  79         arvadostest.StartAPI()
  80         arvadostest.StartKeep(2, false)
  81 }
  82
  83 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  84         arvadostest.ResetEnv()
  85 }
  86
  87 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  88         arvadostest.StopKeep(2)
  89         arvadostest.StopAPI()
  90 }
  91
  92 func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
  93         arvadostest.StartAPI()
  94         // config.yml defines 4 keepstores
  95         arvadostest.StartKeep(4, false)
  96 }
  97
  98 func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
  99         arvadostest.ResetEnv()
 100 }
 101
 102 func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
 103         arvadostest.StopKeep(4)
 104         arvadostest.StopAPI()
 105 }
 106
 107 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
 108         arvadostest.StartAPI()
 109         // We need API to have some keep services listed, but the
 110         // services themselves should be unresponsive.
 111         arvadostest.StartKeep(2, false)
 112         arvadostest.StopKeep(2)
 113 }
 114
 115 func (s *NoKeepServerSuite) SetUpTest(c *C) {
 116         arvadostest.ResetEnv()
 117 }
 118
 119 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
 120         arvadostest.StopAPI()
 121 }
 122
 123 func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool) *keepclient.KeepClient {
 124         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
 125         c.Assert(err, Equals, nil)
 126         cluster, err := cfg.GetCluster("")
 127         c.Assert(err, Equals, nil)
 128
 129         if !loadKeepstoresFromConfig {
 130                 // Do not load Keepstore InternalURLs from the config file
 131                 cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
 132         }
 133
 134         cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
 135
 136         listener = nil
 137         go func() {
 138                 run(log.New(), cluster)
 139                 defer closeListener()
 140         }()
 141         waitForListener()
 142
 143         client := arvados.NewClientFromEnv()
 144         arv, err := arvadosclient.New(client)
 145         c.Assert(err, Equals, nil)
 146         if bogusClientToken {
 147                 arv.ApiToken = "bogus-token"
 148         }
 149         kc := keepclient.New(arv)
 150         sr := map[string]string{
 151                 TestProxyUUID: "http://" + listener.Addr().String(),
 152         }
 153         kc.SetServiceRoots(sr, sr, sr)
 154         kc.Arvados.External = true
 155
 156         return kc
 157 }
 158
 159 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 160         runProxy(c, false, false)
 161         defer closeListener()
 162
 163         req, err := http.NewRequest("POST",
 164                 "http://"+listener.Addr().String()+"/",
 165                 strings.NewReader("TestViaHeader"))
 166         c.Assert(err, Equals, nil)
 167         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 168         resp, err := (&http.Client{}).Do(req)
 169         c.Assert(err, Equals, nil)
 170         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 171         c.Assert(resp.StatusCode, Equals, http.StatusOK)
 172         locator, err := ioutil.ReadAll(resp.Body)
 173         c.Assert(err, Equals, nil)
 174         resp.Body.Close()
 175
 176         req, err = http.NewRequest("GET",
 177                 "http://"+listener.Addr().String()+"/"+string(locator),
 178                 nil)
 179         c.Assert(err, Equals, nil)
 180         resp, err = (&http.Client{}).Do(req)
 181         c.Assert(err, Equals, nil)
 182         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 183         resp.Body.Close()
 184 }
 185
 186 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 187         kc := runProxy(c, false, false)
 188         defer closeListener()
 189
 190         sr := map[string]string{
 191                 TestProxyUUID: "http://" + listener.Addr().String(),
 192         }
 193         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 194
 195         content := []byte("TestLoopDetection")
 196         _, _, err := kc.PutB(content)
 197         c.Check(err, ErrorMatches, `.*loop detected.*`)
 198
 199         hash := fmt.Sprintf("%x", md5.Sum(content))
 200         _, _, _, err = kc.Get(hash)
 201         c.Check(err, ErrorMatches, `.*loop detected.*`)
 202 }
 203
 204 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 205         kc := runProxy(c, false, false)
 206         defer closeListener()
 207
 208         // Set up fake keepstore to record request headers
 209         var hdr http.Header
 210         ts := httptest.NewServer(http.HandlerFunc(
 211                 func(w http.ResponseWriter, r *http.Request) {
 212                         hdr = r.Header
 213                         http.Error(w, "Error", http.StatusInternalServerError)
 214                 }))
 215         defer ts.Close()
 216
 217         // Point keepproxy router's keepclient to the fake keepstore
 218         sr := map[string]string{
 219                 TestProxyUUID: ts.URL,
 220         }
 221         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 222
 223         // Set up client to ask for storage classes to keepproxy
 224         kc.StorageClasses = []string{"secure"}
 225         content := []byte("Very important data")
 226         _, _, err := kc.PutB(content)
 227         c.Check(err, NotNil)
 228         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 229 }
 230
 231 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 232         kc := runProxy(c, false, false)
 233         defer closeListener()
 234
 235         content := []byte("TestDesiredReplicas")
 236         hash := fmt.Sprintf("%x", md5.Sum(content))
 237
 238         for _, kc.Want_replicas = range []int{0, 1, 2} {
 239                 locator, rep, err := kc.PutB(content)
 240                 c.Check(err, Equals, nil)
 241                 c.Check(rep, Equals, kc.Want_replicas)
 242                 if rep > 0 {
 243                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 244                 }
 245         }
 246 }
 247
 248 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 249         kc := runProxy(c, false, false)
 250         defer closeListener()
 251
 252         content := []byte("TestPutWrongContentLength")
 253         hash := fmt.Sprintf("%x", md5.Sum(content))
 254
 255         // If we use http.Client to send these requests to the network
 256         // server we just started, the Go http library automatically
 257         // fixes the invalid Content-Length header. In order to test
 258         // our server behavior, we have to call the handler directly
 259         // using an httptest.ResponseRecorder.
 260         rtr := MakeRESTRouter(kc, 10*time.Second, "")
 261
 262         type testcase struct {
 263                 sendLength   string
 264                 expectStatus int
 265         }
 266
 267         for _, t := range []testcase{
 268                 {"1", http.StatusBadRequest},
 269                 {"", http.StatusLengthRequired},
 270                 {"-1", http.StatusLengthRequired},
 271                 {"abcdef", http.StatusLengthRequired},
 272         } {
 273                 req, err := http.NewRequest("PUT",
 274                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 275                         bytes.NewReader(content))
 276                 c.Assert(err, IsNil)
 277                 req.Header.Set("Content-Length", t.sendLength)
 278                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 279                 req.Header.Set("Content-Type", "application/octet-stream")
 280
 281                 resp := httptest.NewRecorder()
 282                 rtr.ServeHTTP(resp, req)
 283                 c.Check(resp.Code, Equals, t.expectStatus)
 284         }
 285 }
 286
 287 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 288         kc := runProxy(c, false, false)
 289         defer closeListener()
 290         router.(*proxyHandler).timeout = time.Nanosecond
 291
 292         buf := make([]byte, 1<<20)
 293         rand.Read(buf)
 294         var wg sync.WaitGroup
 295         for i := 0; i < 128; i++ {
 296                 wg.Add(1)
 297                 go func() {
 298                         defer wg.Done()
 299                         kc.PutB(buf)
 300                 }()
 301         }
 302         done := make(chan bool)
 303         go func() {
 304                 wg.Wait()
 305                 close(done)
 306         }()
 307         select {
 308         case <-done:
 309         case <-time.After(10 * time.Second):
 310                 c.Error("timeout")
 311         }
 312 }
 313
 314 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 315         kc := runProxy(c, false, false)
 316         defer closeListener()
 317
 318         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 319         var hash2 string
 320
 321         {
 322                 _, _, err := kc.Ask(hash)
 323                 c.Check(err, Equals, keepclient.BlockNotFound)
 324                 c.Log("Finished Ask (expected BlockNotFound)")
 325         }
 326
 327         {
 328                 reader, _, _, err := kc.Get(hash)
 329                 c.Check(reader, Equals, nil)
 330                 c.Check(err, Equals, keepclient.BlockNotFound)
 331                 c.Log("Finished Get (expected BlockNotFound)")
 332         }
 333
 334         // Note in bug #5309 among other errors keepproxy would set
 335         // Content-Length incorrectly on the 404 BlockNotFound response, this
 336         // would result in a protocol violation that would prevent reuse of the
 337         // connection, which would manifest by the next attempt to use the
 338         // connection (in this case the PutB below) failing.  So to test for
 339         // that bug it's necessary to trigger an error response (such as
 340         // BlockNotFound) and then do something else with the same httpClient
 341         // connection.
 342
 343         {
 344                 var rep int
 345                 var err error
 346                 hash2, rep, err = kc.PutB([]byte("foo"))
 347                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 348                 c.Check(rep, Equals, 2)
 349                 c.Check(err, Equals, nil)
 350                 c.Log("Finished PutB (expected success)")
 351         }
 352
 353         {
 354                 blocklen, _, err := kc.Ask(hash2)
 355                 c.Assert(err, Equals, nil)
 356                 c.Check(blocklen, Equals, int64(3))
 357                 c.Log("Finished Ask (expected success)")
 358         }
 359
 360         {
 361                 reader, blocklen, _, err := kc.Get(hash2)
 362                 c.Assert(err, Equals, nil)
 363                 all, err := ioutil.ReadAll(reader)
 364                 c.Check(err, IsNil)
 365                 c.Check(all, DeepEquals, []byte("foo"))
 366                 c.Check(blocklen, Equals, int64(3))
 367                 c.Log("Finished Get (expected success)")
 368         }
 369
 370         {
 371                 var rep int
 372                 var err error
 373                 hash2, rep, err = kc.PutB([]byte(""))
 374                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 375                 c.Check(rep, Equals, 2)
 376                 c.Check(err, Equals, nil)
 377                 c.Log("Finished PutB zero block")
 378         }
 379
 380         {
 381                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 382                 c.Assert(err, Equals, nil)
 383                 all, err := ioutil.ReadAll(reader)
 384                 c.Check(err, IsNil)
 385                 c.Check(all, DeepEquals, []byte(""))
 386                 c.Check(blocklen, Equals, int64(0))
 387                 c.Log("Finished Get zero block")
 388         }
 389 }
 390
 391 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 392         kc := runProxy(c, true, false)
 393         defer closeListener()
 394
 395         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 396
 397         _, _, err := kc.Ask(hash)
 398         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 399
 400         hash2, rep, err := kc.PutB([]byte("bar"))
 401         c.Check(hash2, Equals, "")
 402         c.Check(rep, Equals, 0)
 403         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 404
 405         blocklen, _, err := kc.Ask(hash)
 406         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 407         c.Check(err, ErrorMatches, ".*not found.*")
 408         c.Check(blocklen, Equals, int64(0))
 409
 410         _, blocklen, _, err = kc.Get(hash)
 411         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 412         c.Check(err, ErrorMatches, ".*not found.*")
 413         c.Check(blocklen, Equals, int64(0))
 414
 415 }
 416
 417 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 418         runProxy(c, false, false)
 419         defer closeListener()
 420
 421         {
 422                 client := http.Client{}
 423                 req, err := http.NewRequest("OPTIONS",
 424                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 425                         nil)
 426                 c.Assert(err, IsNil)
 427                 req.Header.Add("Access-Control-Request-Method", "PUT")
 428                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 429                 resp, err := client.Do(req)
 430                 c.Check(err, Equals, nil)
 431                 c.Check(resp.StatusCode, Equals, 200)
 432                 body, err := ioutil.ReadAll(resp.Body)
 433                 c.Check(err, IsNil)
 434                 c.Check(string(body), Equals, "")
 435                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 436                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 437         }
 438
 439         {
 440                 resp, err := http.Get(
 441                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 442                 c.Check(err, Equals, nil)
 443                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 444                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 445         }
 446 }
 447
 448 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 449         runProxy(c, false, false)
 450         defer closeListener()
 451
 452         {
 453                 client := http.Client{}
 454                 req, err := http.NewRequest("POST",
 455                         "http://"+listener.Addr().String()+"/",
 456                         strings.NewReader("qux"))
 457                 c.Check(err, IsNil)
 458                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 459                 req.Header.Add("Content-Type", "application/octet-stream")
 460                 resp, err := client.Do(req)
 461                 c.Check(err, Equals, nil)
 462                 body, err := ioutil.ReadAll(resp.Body)
 463                 c.Check(err, Equals, nil)
 464                 c.Check(string(body), Matches,
 465                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 466         }
 467 }
 468
 469 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 470         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 471                 Equals,
 472                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 473         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 474                 Equals,
 475                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 476         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 477                 Equals,
 478                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 479         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 480                 Equals,
 481                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 482
 483 }
 484
 485 // Test GetIndex
 486 //   Put one block, with 2 replicas
 487 //   With no prefix (expect the block locator, twice)
 488 //   With an existing prefix (expect the block locator, twice)
 489 //   With a valid but non-existing prefix (expect "\n")
 490 //   With an invalid prefix (expect error)
 491 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 492         getIndexWorker(c, false)
 493 }
 494
 495 // Test GetIndex
 496 //   Uses config.yml
 497 //   Put one block, with 2 replicas
 498 //   With no prefix (expect the block locator, twice)
 499 //   With an existing prefix (expect the block locator, twice)
 500 //   With a valid but non-existing prefix (expect "\n")
 501 //   With an invalid prefix (expect error)
 502 func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
 503         getIndexWorker(c, true)
 504 }
 505
 506 func getIndexWorker(c *C, useConfig bool) {
 507         kc := runProxy(c, false, useConfig)
 508         defer closeListener()
 509
 510         // Put "index-data" blocks
 511         data := []byte("index-data")
 512         hash := fmt.Sprintf("%x", md5.Sum(data))
 513
 514         hash2, rep, err := kc.PutB(data)
 515         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 516         c.Check(rep, Equals, 2)
 517         c.Check(err, Equals, nil)
 518
 519         reader, blocklen, _, err := kc.Get(hash)
 520         c.Assert(err, IsNil)
 521         c.Check(blocklen, Equals, int64(10))
 522         all, err := ioutil.ReadAll(reader)
 523         c.Assert(err, IsNil)
 524         c.Check(all, DeepEquals, data)
 525
 526         // Put some more blocks
 527         _, _, err = kc.PutB([]byte("some-more-index-data"))
 528         c.Check(err, IsNil)
 529
 530         kc.Arvados.ApiToken = arvadostest.SystemRootToken
 531
 532         // Invoke GetIndex
 533         for _, spec := range []struct {
 534                 prefix         string
 535                 expectTestHash bool
 536                 expectOther    bool
 537         }{
 538                 {"", true, true},         // with no prefix
 539                 {hash[:3], true, false},  // with matching prefix
 540                 {"abcdef", false, false}, // with no such prefix
 541         } {
 542                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 543                 c.Assert(err, Equals, nil)
 544                 indexResp, err := ioutil.ReadAll(indexReader)
 545                 c.Assert(err, Equals, nil)
 546                 locators := strings.Split(string(indexResp), "\n")
 547                 gotTestHash := 0
 548                 gotOther := 0
 549                 for _, locator := range locators {
 550                         if locator == "" {
 551                                 continue
 552                         }
 553                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 554                         if locator[:32] == hash {
 555                                 gotTestHash++
 556                         } else {
 557                                 gotOther++
 558                         }
 559                 }
 560                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 561                 c.Check(gotOther > 0, Equals, spec.expectOther)
 562         }
 563
 564         // GetIndex with invalid prefix
 565         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 566         c.Assert((err != nil), Equals, true)
 567 }
 568
 569 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 570         kc := runProxy(c, false, false)
 571         defer closeListener()
 572         hash, _, err := kc.PutB([]byte("shareddata"))
 573         c.Check(err, IsNil)
 574         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 575         rdr, _, _, err := kc.Get(hash)
 576         c.Assert(err, IsNil)
 577         data, err := ioutil.ReadAll(rdr)
 578         c.Check(err, IsNil)
 579         c.Check(data, DeepEquals, []byte("shareddata"))
 580 }
 581
 582 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 583         kc := runProxy(c, false, false)
 584         defer closeListener()
 585
 586         // Put a test block
 587         hash, rep, err := kc.PutB([]byte("foo"))
 588         c.Check(err, IsNil)
 589         c.Check(rep, Equals, 2)
 590
 591         for _, badToken := range []string{
 592                 "nosuchtoken",
 593                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 594         } {
 595                 kc.Arvados.ApiToken = badToken
 596
 597                 // Ask and Get will fail only if the upstream
 598                 // keepstore server checks for valid signatures.
 599                 // Without knowing the blob signing key, there is no
 600                 // way for keepproxy to know whether a given token is
 601                 // permitted to read a block.  So these tests fail:
 602                 if false {
 603                         _, _, err = kc.Ask(hash)
 604                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 605                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 606                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 607
 608                         _, _, _, err = kc.Get(hash)
 609                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 610                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 611                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 612                 }
 613
 614                 _, _, err = kc.PutB([]byte("foo"))
 615                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 616         }
 617 }
 618
 619 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 620         kc := runProxy(c, false, false)
 621         defer closeListener()
 622
 623         // Point keepproxy at a non-existent keepstore
 624         locals := map[string]string{
 625                 TestProxyUUID: "http://localhost:12345",
 626         }
 627         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
 628
 629         // Ask should result in temporary bad gateway error
 630         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 631         _, _, err := kc.Ask(hash)
 632         c.Check(err, NotNil)
 633         errNotFound, _ := err.(*keepclient.ErrNotFound)
 634         c.Check(errNotFound.Temporary(), Equals, true)
 635         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 636
 637         // Get should result in temporary bad gateway error
 638         _, _, _, err = kc.Get(hash)
 639         c.Check(err, NotNil)
 640         errNotFound, _ = err.(*keepclient.ErrNotFound)
 641         c.Check(errNotFound.Temporary(), Equals, true)
 642         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 643 }
 644
 645 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 646         kc := runProxy(c, false, false)
 647         defer closeListener()
 648
 649         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 650         for _, f := range []func() error{
 651                 func() error {
 652                         _, _, err := kc.Ask(hash)
 653                         return err
 654                 },
 655                 func() error {
 656                         _, _, _, err := kc.Get(hash)
 657                         return err
 658                 },
 659         } {
 660                 err := f()
 661                 c.Assert(err, NotNil)
 662                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 663                 c.Check(errNotFound.Temporary(), Equals, true)
 664                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 665         }
 666 }
 667
 668 func (s *ServerRequiredSuite) TestPing(c *C) {
 669         kc := runProxy(c, false, false)
 670         defer closeListener()
 671
 672         rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
 673
 674         req, err := http.NewRequest("GET",
 675                 "http://"+listener.Addr().String()+"/_health/ping",
 676                 nil)
 677         c.Assert(err, IsNil)
 678         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 679
 680         resp := httptest.NewRecorder()
 681         rtr.ServeHTTP(resp, req)
 682         c.Check(resp.Code, Equals, 200)
 683         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 684 }