services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "os"
  17         "strings"
  18         "sync"
  19         "testing"
  20         "time"
  21
  22         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25
  26         . "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 func Test(t *testing.T) {
  31         TestingT(t)
  32 }
  33
  34 // Gocheck boilerplate
  35 var _ = Suite(&ServerRequiredSuite{})
  36
  37 // Tests that require the Keep server running
  38 type ServerRequiredSuite struct{}
  39
  40 // Gocheck boilerplate
  41 var _ = Suite(&NoKeepServerSuite{})
  42
  43 // Test with no keepserver to simulate errors
  44 type NoKeepServerSuite struct{}
  45
  46 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  47
  48 // Wait (up to 1 second) for keepproxy to listen on a port. This
  49 // avoids a race condition where we hit a "connection refused" error
  50 // because we start testing the proxy too soon.
  51 func waitForListener() {
  52         const (
  53                 ms = 5
  54         )
  55         for i := 0; listener == nil && i < 10000; i += ms {
  56                 time.Sleep(ms * time.Millisecond)
  57         }
  58         if listener == nil {
  59                 panic("Timed out waiting for listener to start")
  60         }
  61 }
  62
  63 func closeListener() {
  64         if listener != nil {
  65                 listener.Close()
  66         }
  67 }
  68
  69 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  70         arvadostest.StartAPI()
  71         arvadostest.StartKeep(2, false)
  72 }
  73
  74 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  75         arvadostest.ResetEnv()
  76 }
  77
  78 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  79         arvadostest.StopKeep(2)
  80         arvadostest.StopAPI()
  81 }
  82
  83 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  84         arvadostest.StartAPI()
  85         // We need API to have some keep services listed, but the
  86         // services themselves should be unresponsive.
  87         arvadostest.StartKeep(2, false)
  88         arvadostest.StopKeep(2)
  89 }
  90
  91 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  92         arvadostest.ResetEnv()
  93 }
  94
  95 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  96         arvadostest.StopAPI()
  97 }
  98
  99 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
 100         args = append([]string{"keepproxy"}, args...)
 101         os.Args = append(args, "-listen=:0")
 102         listener = nil
 103         go main()
 104         waitForListener()
 105
 106         arv, err := arvadosclient.MakeArvadosClient()
 107         c.Assert(err, Equals, nil)
 108         if bogusClientToken {
 109                 arv.ApiToken = "bogus-token"
 110         }
 111         kc := keepclient.New(arv)
 112         sr := map[string]string{
 113                 TestProxyUUID: "http://" + listener.Addr().String(),
 114         }
 115         kc.SetServiceRoots(sr, sr, sr)
 116         kc.Arvados.External = true
 117
 118         return kc
 119 }
 120
 121 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 122         runProxy(c, nil, false)
 123         defer closeListener()
 124
 125         req, err := http.NewRequest("POST",
 126                 "http://"+listener.Addr().String()+"/",
 127                 strings.NewReader("TestViaHeader"))
 128         c.Assert(err, Equals, nil)
 129         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 130         resp, err := (&http.Client{}).Do(req)
 131         c.Assert(err, Equals, nil)
 132         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 133         locator, err := ioutil.ReadAll(resp.Body)
 134         c.Assert(err, Equals, nil)
 135         resp.Body.Close()
 136
 137         req, err = http.NewRequest("GET",
 138                 "http://"+listener.Addr().String()+"/"+string(locator),
 139                 nil)
 140         c.Assert(err, Equals, nil)
 141         resp, err = (&http.Client{}).Do(req)
 142         c.Assert(err, Equals, nil)
 143         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 144         resp.Body.Close()
 145 }
 146
 147 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 148         kc := runProxy(c, nil, false)
 149         defer closeListener()
 150
 151         sr := map[string]string{
 152                 TestProxyUUID: "http://" + listener.Addr().String(),
 153         }
 154         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 155
 156         content := []byte("TestLoopDetection")
 157         _, _, err := kc.PutB(content)
 158         c.Check(err, ErrorMatches, `.*loop detected.*`)
 159
 160         hash := fmt.Sprintf("%x", md5.Sum(content))
 161         _, _, _, err = kc.Get(hash)
 162         c.Check(err, ErrorMatches, `.*loop detected.*`)
 163 }
 164
 165 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 166         kc := runProxy(c, nil, false)
 167         defer closeListener()
 168
 169         // Set up fake keepstore to record request headers
 170         var hdr http.Header
 171         ts := httptest.NewServer(http.HandlerFunc(
 172                 func(w http.ResponseWriter, r *http.Request) {
 173                         hdr = r.Header
 174                         http.Error(w, "Error", http.StatusInternalServerError)
 175                 }))
 176         defer ts.Close()
 177
 178         // Point keepproxy router's keepclient to the fake keepstore
 179         sr := map[string]string{
 180                 TestProxyUUID: ts.URL,
 181         }
 182         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 183
 184         // Set up client to ask for storage classes to keepproxy
 185         kc.StorageClasses = []string{"secure"}
 186         content := []byte("Very important data")
 187         _, _, err := kc.PutB(content)
 188         c.Check(err, NotNil)
 189         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 190 }
 191
 192 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 193         kc := runProxy(c, nil, false)
 194         defer closeListener()
 195
 196         content := []byte("TestDesiredReplicas")
 197         hash := fmt.Sprintf("%x", md5.Sum(content))
 198
 199         for _, kc.Want_replicas = range []int{0, 1, 2} {
 200                 locator, rep, err := kc.PutB(content)
 201                 c.Check(err, Equals, nil)
 202                 c.Check(rep, Equals, kc.Want_replicas)
 203                 if rep > 0 {
 204                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 205                 }
 206         }
 207 }
 208
 209 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 210         kc := runProxy(c, nil, false)
 211         defer closeListener()
 212
 213         content := []byte("TestPutWrongContentLength")
 214         hash := fmt.Sprintf("%x", md5.Sum(content))
 215
 216         // If we use http.Client to send these requests to the network
 217         // server we just started, the Go http library automatically
 218         // fixes the invalid Content-Length header. In order to test
 219         // our server behavior, we have to call the handler directly
 220         // using an httptest.ResponseRecorder.
 221         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
 222
 223         type testcase struct {
 224                 sendLength   string
 225                 expectStatus int
 226         }
 227
 228         for _, t := range []testcase{
 229                 {"1", http.StatusBadRequest},
 230                 {"", http.StatusLengthRequired},
 231                 {"-1", http.StatusLengthRequired},
 232                 {"abcdef", http.StatusLengthRequired},
 233         } {
 234                 req, err := http.NewRequest("PUT",
 235                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 236                         bytes.NewReader(content))
 237                 c.Assert(err, IsNil)
 238                 req.Header.Set("Content-Length", t.sendLength)
 239                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 240                 req.Header.Set("Content-Type", "application/octet-stream")
 241
 242                 resp := httptest.NewRecorder()
 243                 rtr.ServeHTTP(resp, req)
 244                 c.Check(resp.Code, Equals, t.expectStatus)
 245         }
 246 }
 247
 248 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 249         kc := runProxy(c, nil, false)
 250         defer closeListener()
 251         router.(*proxyHandler).timeout = time.Nanosecond
 252
 253         buf := make([]byte, 1<<20)
 254         rand.Read(buf)
 255         var wg sync.WaitGroup
 256         for i := 0; i < 128; i++ {
 257                 wg.Add(1)
 258                 go func() {
 259                         defer wg.Done()
 260                         kc.PutB(buf)
 261                 }()
 262         }
 263         done := make(chan bool)
 264         go func() {
 265                 wg.Wait()
 266                 close(done)
 267         }()
 268         select {
 269         case <-done:
 270         case <-time.After(10 * time.Second):
 271                 c.Error("timeout")
 272         }
 273 }
 274
 275 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 276         kc := runProxy(c, nil, false)
 277         defer closeListener()
 278
 279         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 280         var hash2 string
 281
 282         {
 283                 _, _, err := kc.Ask(hash)
 284                 c.Check(err, Equals, keepclient.BlockNotFound)
 285                 c.Log("Finished Ask (expected BlockNotFound)")
 286         }
 287
 288         {
 289                 reader, _, _, err := kc.Get(hash)
 290                 c.Check(reader, Equals, nil)
 291                 c.Check(err, Equals, keepclient.BlockNotFound)
 292                 c.Log("Finished Get (expected BlockNotFound)")
 293         }
 294
 295         // Note in bug #5309 among other errors keepproxy would set
 296         // Content-Length incorrectly on the 404 BlockNotFound response, this
 297         // would result in a protocol violation that would prevent reuse of the
 298         // connection, which would manifest by the next attempt to use the
 299         // connection (in this case the PutB below) failing.  So to test for
 300         // that bug it's necessary to trigger an error response (such as
 301         // BlockNotFound) and then do something else with the same httpClient
 302         // connection.
 303
 304         {
 305                 var rep int
 306                 var err error
 307                 hash2, rep, err = kc.PutB([]byte("foo"))
 308                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 309                 c.Check(rep, Equals, 2)
 310                 c.Check(err, Equals, nil)
 311                 c.Log("Finished PutB (expected success)")
 312         }
 313
 314         {
 315                 blocklen, _, err := kc.Ask(hash2)
 316                 c.Assert(err, Equals, nil)
 317                 c.Check(blocklen, Equals, int64(3))
 318                 c.Log("Finished Ask (expected success)")
 319         }
 320
 321         {
 322                 reader, blocklen, _, err := kc.Get(hash2)
 323                 c.Assert(err, Equals, nil)
 324                 all, err := ioutil.ReadAll(reader)
 325                 c.Check(err, IsNil)
 326                 c.Check(all, DeepEquals, []byte("foo"))
 327                 c.Check(blocklen, Equals, int64(3))
 328                 c.Log("Finished Get (expected success)")
 329         }
 330
 331         {
 332                 var rep int
 333                 var err error
 334                 hash2, rep, err = kc.PutB([]byte(""))
 335                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 336                 c.Check(rep, Equals, 2)
 337                 c.Check(err, Equals, nil)
 338                 c.Log("Finished PutB zero block")
 339         }
 340
 341         {
 342                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 343                 c.Assert(err, Equals, nil)
 344                 all, err := ioutil.ReadAll(reader)
 345                 c.Check(err, IsNil)
 346                 c.Check(all, DeepEquals, []byte(""))
 347                 c.Check(blocklen, Equals, int64(0))
 348                 c.Log("Finished Get zero block")
 349         }
 350 }
 351
 352 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 353         kc := runProxy(c, nil, true)
 354         defer closeListener()
 355
 356         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 357
 358         _, _, err := kc.Ask(hash)
 359         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 360
 361         hash2, rep, err := kc.PutB([]byte("bar"))
 362         c.Check(hash2, Equals, "")
 363         c.Check(rep, Equals, 0)
 364         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 365
 366         blocklen, _, err := kc.Ask(hash)
 367         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 368         c.Check(err, ErrorMatches, ".*not found.*")
 369         c.Check(blocklen, Equals, int64(0))
 370
 371         _, blocklen, _, err = kc.Get(hash)
 372         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 373         c.Check(err, ErrorMatches, ".*not found.*")
 374         c.Check(blocklen, Equals, int64(0))
 375
 376 }
 377
 378 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 379         kc := runProxy(c, []string{"-no-get"}, false)
 380         defer closeListener()
 381
 382         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 383
 384         {
 385                 _, _, err := kc.Ask(hash)
 386                 errNotFound, _ := err.(keepclient.ErrNotFound)
 387                 c.Check(errNotFound, NotNil)
 388                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 389                 c.Log("Ask 1")
 390         }
 391
 392         {
 393                 hash2, rep, err := kc.PutB([]byte("baz"))
 394                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 395                 c.Check(rep, Equals, 2)
 396                 c.Check(err, Equals, nil)
 397                 c.Log("PutB")
 398         }
 399
 400         {
 401                 blocklen, _, err := kc.Ask(hash)
 402                 errNotFound, _ := err.(keepclient.ErrNotFound)
 403                 c.Check(errNotFound, NotNil)
 404                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 405                 c.Check(blocklen, Equals, int64(0))
 406                 c.Log("Ask 2")
 407         }
 408
 409         {
 410                 _, blocklen, _, err := kc.Get(hash)
 411                 errNotFound, _ := err.(keepclient.ErrNotFound)
 412                 c.Check(errNotFound, NotNil)
 413                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 414                 c.Check(blocklen, Equals, int64(0))
 415                 c.Log("Get")
 416         }
 417 }
 418
 419 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 420         kc := runProxy(c, []string{"-no-put"}, false)
 421         defer closeListener()
 422
 423         hash2, rep, err := kc.PutB([]byte("quux"))
 424         c.Check(hash2, Equals, "")
 425         c.Check(rep, Equals, 0)
 426         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 427 }
 428
 429 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 430         runProxy(c, nil, false)
 431         defer closeListener()
 432
 433         {
 434                 client := http.Client{}
 435                 req, err := http.NewRequest("OPTIONS",
 436                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 437                         nil)
 438                 c.Assert(err, IsNil)
 439                 req.Header.Add("Access-Control-Request-Method", "PUT")
 440                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 441                 resp, err := client.Do(req)
 442                 c.Check(err, Equals, nil)
 443                 c.Check(resp.StatusCode, Equals, 200)
 444                 body, err := ioutil.ReadAll(resp.Body)
 445                 c.Check(err, IsNil)
 446                 c.Check(string(body), Equals, "")
 447                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 448                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 449         }
 450
 451         {
 452                 resp, err := http.Get(
 453                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 454                 c.Check(err, Equals, nil)
 455                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 456                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 457         }
 458 }
 459
 460 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 461         runProxy(c, nil, false)
 462         defer closeListener()
 463
 464         {
 465                 client := http.Client{}
 466                 req, err := http.NewRequest("POST",
 467                         "http://"+listener.Addr().String()+"/",
 468                         strings.NewReader("qux"))
 469                 c.Check(err, IsNil)
 470                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 471                 req.Header.Add("Content-Type", "application/octet-stream")
 472                 resp, err := client.Do(req)
 473                 c.Check(err, Equals, nil)
 474                 body, err := ioutil.ReadAll(resp.Body)
 475                 c.Check(err, Equals, nil)
 476                 c.Check(string(body), Matches,
 477                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 478         }
 479 }
 480
 481 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 482         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 483                 Equals,
 484                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 485         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 486                 Equals,
 487                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 488         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 489                 Equals,
 490                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 491         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 492                 Equals,
 493                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 494
 495 }
 496
 497 // Test GetIndex
 498 //   Put one block, with 2 replicas
 499 //   With no prefix (expect the block locator, twice)
 500 //   With an existing prefix (expect the block locator, twice)
 501 //   With a valid but non-existing prefix (expect "\n")
 502 //   With an invalid prefix (expect error)
 503 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 504         kc := runProxy(c, nil, false)
 505         defer closeListener()
 506
 507         // Put "index-data" blocks
 508         data := []byte("index-data")
 509         hash := fmt.Sprintf("%x", md5.Sum(data))
 510
 511         hash2, rep, err := kc.PutB(data)
 512         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 513         c.Check(rep, Equals, 2)
 514         c.Check(err, Equals, nil)
 515
 516         reader, blocklen, _, err := kc.Get(hash)
 517         c.Assert(err, IsNil)
 518         c.Check(blocklen, Equals, int64(10))
 519         all, err := ioutil.ReadAll(reader)
 520         c.Assert(err, IsNil)
 521         c.Check(all, DeepEquals, data)
 522
 523         // Put some more blocks
 524         _, _, err = kc.PutB([]byte("some-more-index-data"))
 525         c.Check(err, IsNil)
 526
 527         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 528
 529         // Invoke GetIndex
 530         for _, spec := range []struct {
 531                 prefix         string
 532                 expectTestHash bool
 533                 expectOther    bool
 534         }{
 535                 {"", true, true},         // with no prefix
 536                 {hash[:3], true, false},  // with matching prefix
 537                 {"abcdef", false, false}, // with no such prefix
 538         } {
 539                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 540                 c.Assert(err, Equals, nil)
 541                 indexResp, err := ioutil.ReadAll(indexReader)
 542                 c.Assert(err, Equals, nil)
 543                 locators := strings.Split(string(indexResp), "\n")
 544                 gotTestHash := 0
 545                 gotOther := 0
 546                 for _, locator := range locators {
 547                         if locator == "" {
 548                                 continue
 549                         }
 550                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 551                         if locator[:32] == hash {
 552                                 gotTestHash++
 553                         } else {
 554                                 gotOther++
 555                         }
 556                 }
 557                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 558                 c.Check(gotOther > 0, Equals, spec.expectOther)
 559         }
 560
 561         // GetIndex with invalid prefix
 562         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 563         c.Assert((err != nil), Equals, true)
 564 }
 565
 566 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 567         kc := runProxy(c, nil, false)
 568         defer closeListener()
 569         hash, _, err := kc.PutB([]byte("shareddata"))
 570         c.Check(err, IsNil)
 571         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 572         rdr, _, _, err := kc.Get(hash)
 573         c.Assert(err, IsNil)
 574         data, err := ioutil.ReadAll(rdr)
 575         c.Check(err, IsNil)
 576         c.Check(data, DeepEquals, []byte("shareddata"))
 577 }
 578
 579 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 580         kc := runProxy(c, nil, false)
 581         defer closeListener()
 582
 583         // Put a test block
 584         hash, rep, err := kc.PutB([]byte("foo"))
 585         c.Check(err, IsNil)
 586         c.Check(rep, Equals, 2)
 587
 588         for _, badToken := range []string{
 589                 "nosuchtoken",
 590                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 591         } {
 592                 kc.Arvados.ApiToken = badToken
 593
 594                 // Ask and Get will fail only if the upstream
 595                 // keepstore server checks for valid signatures.
 596                 // Without knowing the blob signing key, there is no
 597                 // way for keepproxy to know whether a given token is
 598                 // permitted to read a block.  So these tests fail:
 599                 if false {
 600                         _, _, err = kc.Ask(hash)
 601                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 602                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 603                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 604
 605                         _, _, _, err = kc.Get(hash)
 606                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 607                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 608                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 609                 }
 610
 611                 _, _, err = kc.PutB([]byte("foo"))
 612                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 613         }
 614 }
 615
 616 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 617         kc := runProxy(c, nil, false)
 618         defer closeListener()
 619
 620         // Point keepproxy to a non-existant keepstore
 621         locals := map[string]string{
 622                 TestProxyUUID: "http://localhost:12345",
 623         }
 624         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
 625
 626         // Ask should result in temporary bad gateway error
 627         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 628         _, _, err := kc.Ask(hash)
 629         c.Check(err, NotNil)
 630         errNotFound, _ := err.(*keepclient.ErrNotFound)
 631         c.Check(errNotFound.Temporary(), Equals, true)
 632         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 633
 634         // Get should result in temporary bad gateway error
 635         _, _, _, err = kc.Get(hash)
 636         c.Check(err, NotNil)
 637         errNotFound, _ = err.(*keepclient.ErrNotFound)
 638         c.Check(errNotFound.Temporary(), Equals, true)
 639         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 640 }
 641
 642 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 643         kc := runProxy(c, nil, false)
 644         defer closeListener()
 645
 646         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 647         for _, f := range []func() error{
 648                 func() error {
 649                         _, _, err := kc.Ask(hash)
 650                         return err
 651                 },
 652                 func() error {
 653                         _, _, _, err := kc.Get(hash)
 654                         return err
 655                 },
 656         } {
 657                 err := f()
 658                 c.Assert(err, NotNil)
 659                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 660                 c.Check(errNotFound.Temporary(), Equals, true)
 661                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 662         }
 663 }
 664
 665 func (s *ServerRequiredSuite) TestPing(c *C) {
 666         kc := runProxy(c, nil, false)
 667         defer closeListener()
 668
 669         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
 670
 671         req, err := http.NewRequest("GET",
 672                 "http://"+listener.Addr().String()+"/_health/ping",
 673                 nil)
 674         c.Assert(err, IsNil)
 675         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 676
 677         resp := httptest.NewRecorder()
 678         rtr.ServeHTTP(resp, req)
 679         c.Check(resp.Code, Equals, 200)
 680         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 681 }