services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "os"
  17         "strings"
  18         "sync"
  19         "testing"
  20         "time"
  21
  22         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25
  26         . "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 func Test(t *testing.T) {
  31         TestingT(t)
  32 }
  33
  34 // Gocheck boilerplate
  35 var _ = Suite(&ServerRequiredSuite{})
  36
  37 // Tests that require the Keep server running
  38 type ServerRequiredSuite struct{}
  39
  40 // Gocheck boilerplate
  41 var _ = Suite(&NoKeepServerSuite{})
  42
  43 // Test with no keepserver to simulate errors
  44 type NoKeepServerSuite struct{}
  45
  46 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  47
  48 // Wait (up to 1 second) for keepproxy to listen on a port. This
  49 // avoids a race condition where we hit a "connection refused" error
  50 // because we start testing the proxy too soon.
  51 func waitForListener() {
  52         const (
  53                 ms = 5
  54         )
  55         for i := 0; listener == nil && i < 10000; i += ms {
  56                 time.Sleep(ms * time.Millisecond)
  57         }
  58         if listener == nil {
  59                 panic("Timed out waiting for listener to start")
  60         }
  61 }
  62
  63 func closeListener() {
  64         if listener != nil {
  65                 listener.Close()
  66         }
  67 }
  68
  69 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  70         arvadostest.StartAPI()
  71         arvadostest.StartKeep(2, false)
  72 }
  73
  74 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  75         arvadostest.ResetEnv()
  76 }
  77
  78 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  79         arvadostest.StopKeep(2)
  80         arvadostest.StopAPI()
  81 }
  82
  83 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  84         arvadostest.StartAPI()
  85         // We need API to have some keep services listed, but the
  86         // services themselves should be unresponsive.
  87         arvadostest.StartKeep(2, false)
  88         arvadostest.StopKeep(2)
  89 }
  90
  91 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  92         arvadostest.ResetEnv()
  93 }
  94
  95 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  96         arvadostest.StopAPI()
  97 }
  98
  99 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
 100         args = append([]string{"keepproxy"}, args...)
 101         os.Args = append(args, "-listen=:0")
 102         listener = nil
 103         go main()
 104         waitForListener()
 105
 106         arv, err := arvadosclient.MakeArvadosClient()
 107         c.Assert(err, Equals, nil)
 108         if bogusClientToken {
 109                 arv.ApiToken = "bogus-token"
 110         }
 111         kc := keepclient.New(arv)
 112         sr := map[string]string{
 113                 TestProxyUUID: "http://" + listener.Addr().String(),
 114         }
 115         kc.SetServiceRoots(sr, sr, sr)
 116         kc.Arvados.External = true
 117
 118         return kc
 119 }
 120
 121 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 122         runProxy(c, nil, false)
 123         defer closeListener()
 124
 125         req, err := http.NewRequest("POST",
 126                 "http://"+listener.Addr().String()+"/",
 127                 strings.NewReader("TestViaHeader"))
 128         c.Assert(err, Equals, nil)
 129         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 130         resp, err := (&http.Client{}).Do(req)
 131         c.Assert(err, Equals, nil)
 132         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 133         locator, err := ioutil.ReadAll(resp.Body)
 134         c.Assert(err, Equals, nil)
 135         resp.Body.Close()
 136
 137         req, err = http.NewRequest("GET",
 138                 "http://"+listener.Addr().String()+"/"+string(locator),
 139                 nil)
 140         c.Assert(err, Equals, nil)
 141         resp, err = (&http.Client{}).Do(req)
 142         c.Assert(err, Equals, nil)
 143         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 144         resp.Body.Close()
 145 }
 146
 147 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 148         kc := runProxy(c, nil, false)
 149         defer closeListener()
 150
 151         sr := map[string]string{
 152                 TestProxyUUID: "http://" + listener.Addr().String(),
 153         }
 154         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 155
 156         content := []byte("TestLoopDetection")
 157         _, _, err := kc.PutB(content)
 158         c.Check(err, ErrorMatches, `.*loop detected.*`)
 159
 160         hash := fmt.Sprintf("%x", md5.Sum(content))
 161         _, _, _, err = kc.Get(hash)
 162         c.Check(err, ErrorMatches, `.*loop detected.*`)
 163 }
 164
 165 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 166         kc := runProxy(c, nil, false)
 167         defer closeListener()
 168
 169         // Set up fake keepstore to record request headers
 170         var hdr http.Header
 171         ts := httptest.NewServer(http.HandlerFunc(
 172                 func(w http.ResponseWriter, r *http.Request) {
 173                         hdr = r.Header
 174                         http.Error(w, "Error", http.StatusInternalServerError)
 175                 }))
 176         defer ts.Close()
 177
 178         // Point keepproxy router's keepclient to the fake keepstore
 179         sr := map[string]string{
 180                 TestProxyUUID: ts.URL,
 181         }
 182         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 183
 184         // Set up client to ask for storage classes to keepproxy
 185         kc.StorageClasses = []string{"secure"}
 186         content := []byte("Very important data")
 187         _, _, err := kc.PutB(content)
 188         c.Check(err, NotNil)
 189         // errNotFound, _ := err.(*keepclient.ErrNotFound)
 190         // c.Check(errNotFound.Temporary(), Equals, true)
 191         // c.Assert(err, ErrorMatches, ".*connection refused.*")
 192         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 193 }
 194
 195 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 196         kc := runProxy(c, nil, false)
 197         defer closeListener()
 198
 199         content := []byte("TestDesiredReplicas")
 200         hash := fmt.Sprintf("%x", md5.Sum(content))
 201
 202         for _, kc.Want_replicas = range []int{0, 1, 2} {
 203                 locator, rep, err := kc.PutB(content)
 204                 c.Check(err, Equals, nil)
 205                 c.Check(rep, Equals, kc.Want_replicas)
 206                 if rep > 0 {
 207                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 208                 }
 209         }
 210 }
 211
 212 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 213         kc := runProxy(c, nil, false)
 214         defer closeListener()
 215
 216         content := []byte("TestPutWrongContentLength")
 217         hash := fmt.Sprintf("%x", md5.Sum(content))
 218
 219         // If we use http.Client to send these requests to the network
 220         // server we just started, the Go http library automatically
 221         // fixes the invalid Content-Length header. In order to test
 222         // our server behavior, we have to call the handler directly
 223         // using an httptest.ResponseRecorder.
 224         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
 225
 226         type testcase struct {
 227                 sendLength   string
 228                 expectStatus int
 229         }
 230
 231         for _, t := range []testcase{
 232                 {"1", http.StatusBadRequest},
 233                 {"", http.StatusLengthRequired},
 234                 {"-1", http.StatusLengthRequired},
 235                 {"abcdef", http.StatusLengthRequired},
 236         } {
 237                 req, err := http.NewRequest("PUT",
 238                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 239                         bytes.NewReader(content))
 240                 c.Assert(err, IsNil)
 241                 req.Header.Set("Content-Length", t.sendLength)
 242                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 243                 req.Header.Set("Content-Type", "application/octet-stream")
 244
 245                 resp := httptest.NewRecorder()
 246                 rtr.ServeHTTP(resp, req)
 247                 c.Check(resp.Code, Equals, t.expectStatus)
 248         }
 249 }
 250
 251 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 252         kc := runProxy(c, nil, false)
 253         defer closeListener()
 254         router.(*proxyHandler).timeout = time.Nanosecond
 255
 256         buf := make([]byte, 1<<20)
 257         rand.Read(buf)
 258         var wg sync.WaitGroup
 259         for i := 0; i < 128; i++ {
 260                 wg.Add(1)
 261                 go func() {
 262                         defer wg.Done()
 263                         kc.PutB(buf)
 264                 }()
 265         }
 266         done := make(chan bool)
 267         go func() {
 268                 wg.Wait()
 269                 close(done)
 270         }()
 271         select {
 272         case <-done:
 273         case <-time.After(10 * time.Second):
 274                 c.Error("timeout")
 275         }
 276 }
 277
 278 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 279         kc := runProxy(c, nil, false)
 280         defer closeListener()
 281
 282         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 283         var hash2 string
 284
 285         {
 286                 _, _, err := kc.Ask(hash)
 287                 c.Check(err, Equals, keepclient.BlockNotFound)
 288                 c.Log("Finished Ask (expected BlockNotFound)")
 289         }
 290
 291         {
 292                 reader, _, _, err := kc.Get(hash)
 293                 c.Check(reader, Equals, nil)
 294                 c.Check(err, Equals, keepclient.BlockNotFound)
 295                 c.Log("Finished Get (expected BlockNotFound)")
 296         }
 297
 298         // Note in bug #5309 among other errors keepproxy would set
 299         // Content-Length incorrectly on the 404 BlockNotFound response, this
 300         // would result in a protocol violation that would prevent reuse of the
 301         // connection, which would manifest by the next attempt to use the
 302         // connection (in this case the PutB below) failing.  So to test for
 303         // that bug it's necessary to trigger an error response (such as
 304         // BlockNotFound) and then do something else with the same httpClient
 305         // connection.
 306
 307         {
 308                 var rep int
 309                 var err error
 310                 hash2, rep, err = kc.PutB([]byte("foo"))
 311                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 312                 c.Check(rep, Equals, 2)
 313                 c.Check(err, Equals, nil)
 314                 c.Log("Finished PutB (expected success)")
 315         }
 316
 317         {
 318                 blocklen, _, err := kc.Ask(hash2)
 319                 c.Assert(err, Equals, nil)
 320                 c.Check(blocklen, Equals, int64(3))
 321                 c.Log("Finished Ask (expected success)")
 322         }
 323
 324         {
 325                 reader, blocklen, _, err := kc.Get(hash2)
 326                 c.Assert(err, Equals, nil)
 327                 all, err := ioutil.ReadAll(reader)
 328                 c.Check(err, IsNil)
 329                 c.Check(all, DeepEquals, []byte("foo"))
 330                 c.Check(blocklen, Equals, int64(3))
 331                 c.Log("Finished Get (expected success)")
 332         }
 333
 334         {
 335                 var rep int
 336                 var err error
 337                 hash2, rep, err = kc.PutB([]byte(""))
 338                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 339                 c.Check(rep, Equals, 2)
 340                 c.Check(err, Equals, nil)
 341                 c.Log("Finished PutB zero block")
 342         }
 343
 344         {
 345                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 346                 c.Assert(err, Equals, nil)
 347                 all, err := ioutil.ReadAll(reader)
 348                 c.Check(err, IsNil)
 349                 c.Check(all, DeepEquals, []byte(""))
 350                 c.Check(blocklen, Equals, int64(0))
 351                 c.Log("Finished Get zero block")
 352         }
 353 }
 354
 355 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 356         kc := runProxy(c, nil, true)
 357         defer closeListener()
 358
 359         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 360
 361         _, _, err := kc.Ask(hash)
 362         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 363
 364         hash2, rep, err := kc.PutB([]byte("bar"))
 365         c.Check(hash2, Equals, "")
 366         c.Check(rep, Equals, 0)
 367         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 368
 369         blocklen, _, err := kc.Ask(hash)
 370         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 371         c.Check(err, ErrorMatches, ".*not found.*")
 372         c.Check(blocklen, Equals, int64(0))
 373
 374         _, blocklen, _, err = kc.Get(hash)
 375         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 376         c.Check(err, ErrorMatches, ".*not found.*")
 377         c.Check(blocklen, Equals, int64(0))
 378
 379 }
 380
 381 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 382         kc := runProxy(c, []string{"-no-get"}, false)
 383         defer closeListener()
 384
 385         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 386
 387         {
 388                 _, _, err := kc.Ask(hash)
 389                 errNotFound, _ := err.(keepclient.ErrNotFound)
 390                 c.Check(errNotFound, NotNil)
 391                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 392                 c.Log("Ask 1")
 393         }
 394
 395         {
 396                 hash2, rep, err := kc.PutB([]byte("baz"))
 397                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 398                 c.Check(rep, Equals, 2)
 399                 c.Check(err, Equals, nil)
 400                 c.Log("PutB")
 401         }
 402
 403         {
 404                 blocklen, _, err := kc.Ask(hash)
 405                 errNotFound, _ := err.(keepclient.ErrNotFound)
 406                 c.Check(errNotFound, NotNil)
 407                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 408                 c.Check(blocklen, Equals, int64(0))
 409                 c.Log("Ask 2")
 410         }
 411
 412         {
 413                 _, blocklen, _, err := kc.Get(hash)
 414                 errNotFound, _ := err.(keepclient.ErrNotFound)
 415                 c.Check(errNotFound, NotNil)
 416                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 417                 c.Check(blocklen, Equals, int64(0))
 418                 c.Log("Get")
 419         }
 420 }
 421
 422 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 423         kc := runProxy(c, []string{"-no-put"}, false)
 424         defer closeListener()
 425
 426         hash2, rep, err := kc.PutB([]byte("quux"))
 427         c.Check(hash2, Equals, "")
 428         c.Check(rep, Equals, 0)
 429         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 430 }
 431
 432 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 433         runProxy(c, nil, false)
 434         defer closeListener()
 435
 436         {
 437                 client := http.Client{}
 438                 req, err := http.NewRequest("OPTIONS",
 439                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 440                         nil)
 441                 c.Assert(err, IsNil)
 442                 req.Header.Add("Access-Control-Request-Method", "PUT")
 443                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 444                 resp, err := client.Do(req)
 445                 c.Check(err, Equals, nil)
 446                 c.Check(resp.StatusCode, Equals, 200)
 447                 body, err := ioutil.ReadAll(resp.Body)
 448                 c.Check(err, IsNil)
 449                 c.Check(string(body), Equals, "")
 450                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 451                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 452         }
 453
 454         {
 455                 resp, err := http.Get(
 456                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 457                 c.Check(err, Equals, nil)
 458                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 459                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 460         }
 461 }
 462
 463 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 464         runProxy(c, nil, false)
 465         defer closeListener()
 466
 467         {
 468                 client := http.Client{}
 469                 req, err := http.NewRequest("POST",
 470                         "http://"+listener.Addr().String()+"/",
 471                         strings.NewReader("qux"))
 472                 c.Check(err, IsNil)
 473                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 474                 req.Header.Add("Content-Type", "application/octet-stream")
 475                 resp, err := client.Do(req)
 476                 c.Check(err, Equals, nil)
 477                 body, err := ioutil.ReadAll(resp.Body)
 478                 c.Check(err, Equals, nil)
 479                 c.Check(string(body), Matches,
 480                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 481         }
 482 }
 483
 484 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 485         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 486                 Equals,
 487                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 488         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 489                 Equals,
 490                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 491         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 492                 Equals,
 493                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 494         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 495                 Equals,
 496                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 497
 498 }
 499
 500 // Test GetIndex
 501 //   Put one block, with 2 replicas
 502 //   With no prefix (expect the block locator, twice)
 503 //   With an existing prefix (expect the block locator, twice)
 504 //   With a valid but non-existing prefix (expect "\n")
 505 //   With an invalid prefix (expect error)
 506 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 507         kc := runProxy(c, nil, false)
 508         defer closeListener()
 509
 510         // Put "index-data" blocks
 511         data := []byte("index-data")
 512         hash := fmt.Sprintf("%x", md5.Sum(data))
 513
 514         hash2, rep, err := kc.PutB(data)
 515         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 516         c.Check(rep, Equals, 2)
 517         c.Check(err, Equals, nil)
 518
 519         reader, blocklen, _, err := kc.Get(hash)
 520         c.Assert(err, IsNil)
 521         c.Check(blocklen, Equals, int64(10))
 522         all, err := ioutil.ReadAll(reader)
 523         c.Assert(err, IsNil)
 524         c.Check(all, DeepEquals, data)
 525
 526         // Put some more blocks
 527         _, _, err = kc.PutB([]byte("some-more-index-data"))
 528         c.Check(err, IsNil)
 529
 530         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 531
 532         // Invoke GetIndex
 533         for _, spec := range []struct {
 534                 prefix         string
 535                 expectTestHash bool
 536                 expectOther    bool
 537         }{
 538                 {"", true, true},         // with no prefix
 539                 {hash[:3], true, false},  // with matching prefix
 540                 {"abcdef", false, false}, // with no such prefix
 541         } {
 542                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 543                 c.Assert(err, Equals, nil)
 544                 indexResp, err := ioutil.ReadAll(indexReader)
 545                 c.Assert(err, Equals, nil)
 546                 locators := strings.Split(string(indexResp), "\n")
 547                 gotTestHash := 0
 548                 gotOther := 0
 549                 for _, locator := range locators {
 550                         if locator == "" {
 551                                 continue
 552                         }
 553                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 554                         if locator[:32] == hash {
 555                                 gotTestHash++
 556                         } else {
 557                                 gotOther++
 558                         }
 559                 }
 560                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 561                 c.Check(gotOther > 0, Equals, spec.expectOther)
 562         }
 563
 564         // GetIndex with invalid prefix
 565         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 566         c.Assert((err != nil), Equals, true)
 567 }
 568
 569 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 570         kc := runProxy(c, nil, false)
 571         defer closeListener()
 572         hash, _, err := kc.PutB([]byte("shareddata"))
 573         c.Check(err, IsNil)
 574         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 575         rdr, _, _, err := kc.Get(hash)
 576         c.Assert(err, IsNil)
 577         data, err := ioutil.ReadAll(rdr)
 578         c.Check(err, IsNil)
 579         c.Check(data, DeepEquals, []byte("shareddata"))
 580 }
 581
 582 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 583         kc := runProxy(c, nil, false)
 584         defer closeListener()
 585
 586         // Put a test block
 587         hash, rep, err := kc.PutB([]byte("foo"))
 588         c.Check(err, IsNil)
 589         c.Check(rep, Equals, 2)
 590
 591         for _, badToken := range []string{
 592                 "nosuchtoken",
 593                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 594         } {
 595                 kc.Arvados.ApiToken = badToken
 596
 597                 // Ask and Get will fail only if the upstream
 598                 // keepstore server checks for valid signatures.
 599                 // Without knowing the blob signing key, there is no
 600                 // way for keepproxy to know whether a given token is
 601                 // permitted to read a block.  So these tests fail:
 602                 if false {
 603                         _, _, err = kc.Ask(hash)
 604                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 605                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 606                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 607
 608                         _, _, _, err = kc.Get(hash)
 609                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 610                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 611                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 612                 }
 613
 614                 _, _, err = kc.PutB([]byte("foo"))
 615                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 616         }
 617 }
 618
 619 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 620         arv, err := arvadosclient.MakeArvadosClient()
 621         c.Assert(err, Equals, nil)
 622
 623         // keepclient with no such keep server
 624         kc := keepclient.New(arv)
 625         locals := map[string]string{
 626                 TestProxyUUID: "http://localhost:12345",
 627         }
 628         kc.SetServiceRoots(locals, nil, nil)
 629
 630         // Ask should result in temporary connection refused error
 631         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 632         _, _, err = kc.Ask(hash)
 633         c.Check(err, NotNil)
 634         errNotFound, _ := err.(*keepclient.ErrNotFound)
 635         c.Check(errNotFound.Temporary(), Equals, true)
 636         c.Assert(err, ErrorMatches, ".*connection refused.*")
 637
 638         // Get should result in temporary connection refused error
 639         _, _, _, err = kc.Get(hash)
 640         c.Check(err, NotNil)
 641         errNotFound, _ = err.(*keepclient.ErrNotFound)
 642         c.Check(errNotFound.Temporary(), Equals, true)
 643         c.Assert(err, ErrorMatches, ".*connection refused.*")
 644 }
 645
 646 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 647         kc := runProxy(c, nil, false)
 648         defer closeListener()
 649
 650         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 651         for _, f := range []func() error{
 652                 func() error {
 653                         _, _, err := kc.Ask(hash)
 654                         return err
 655                 },
 656                 func() error {
 657                         _, _, _, err := kc.Get(hash)
 658                         return err
 659                 },
 660         } {
 661                 err := f()
 662                 c.Assert(err, NotNil)
 663                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 664                 c.Check(errNotFound.Temporary(), Equals, true)
 665                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 666         }
 667 }
 668
 669 func (s *ServerRequiredSuite) TestPing(c *C) {
 670         kc := runProxy(c, nil, false)
 671         defer closeListener()
 672
 673         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
 674
 675         req, err := http.NewRequest("GET",
 676                 "http://"+listener.Addr().String()+"/_health/ping",
 677                 nil)
 678         c.Assert(err, IsNil)
 679         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 680
 681         resp := httptest.NewRecorder()
 682         rtr.ServeHTTP(resp, req)
 683         c.Check(resp.Code, Equals, 200)
 684         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 685 }