services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "os"
  17         "strings"
  18         "sync"
  19         "testing"
  20         "time"
  21
  22         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25
  26         . "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 func Test(t *testing.T) {
  31         TestingT(t)
  32 }
  33
  34 // Gocheck boilerplate
  35 var _ = Suite(&ServerRequiredSuite{})
  36
  37 // Tests that require the Keep server running
  38 type ServerRequiredSuite struct{}
  39
  40 // Gocheck boilerplate
  41 var _ = Suite(&NoKeepServerSuite{})
  42
  43 // Test with no keepserver to simulate errors
  44 type NoKeepServerSuite struct{}
  45
  46 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  47
  48 // Wait (up to 1 second) for keepproxy to listen on a port. This
  49 // avoids a race condition where we hit a "connection refused" error
  50 // because we start testing the proxy too soon.
  51 func waitForListener() {
  52         const (
  53                 ms = 5
  54         )
  55         for i := 0; listener == nil && i < 10000; i += ms {
  56                 time.Sleep(ms * time.Millisecond)
  57         }
  58         if listener == nil {
  59                 panic("Timed out waiting for listener to start")
  60         }
  61 }
  62
  63 func closeListener() {
  64         if listener != nil {
  65                 listener.Close()
  66         }
  67 }
  68
  69 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  70         arvadostest.StartAPI()
  71         arvadostest.StartKeep(2, false)
  72 }
  73
  74 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  75         arvadostest.ResetEnv()
  76 }
  77
  78 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  79         arvadostest.StopKeep(2)
  80         arvadostest.StopAPI()
  81 }
  82
  83 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  84         arvadostest.StartAPI()
  85         // We need API to have some keep services listed, but the
  86         // services themselves should be unresponsive.
  87         arvadostest.StartKeep(2, false)
  88         arvadostest.StopKeep(2)
  89 }
  90
  91 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  92         arvadostest.ResetEnv()
  93 }
  94
  95 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  96         arvadostest.StopAPI()
  97 }
  98
  99 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
 100         args = append([]string{"keepproxy"}, args...)
 101         os.Args = append(args, "-listen=:0")
 102         listener = nil
 103         go main()
 104         waitForListener()
 105
 106         arv, err := arvadosclient.MakeArvadosClient()
 107         c.Assert(err, Equals, nil)
 108         if bogusClientToken {
 109                 arv.ApiToken = "bogus-token"
 110         }
 111         kc := keepclient.New(arv)
 112         sr := map[string]string{
 113                 TestProxyUUID: "http://" + listener.Addr().String(),
 114         }
 115         kc.SetServiceRoots(sr, sr, sr)
 116         kc.Arvados.External = true
 117
 118         return kc
 119 }
 120
 121 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 122         runProxy(c, nil, false)
 123         defer closeListener()
 124
 125         req, err := http.NewRequest("POST",
 126                 "http://"+listener.Addr().String()+"/",
 127                 strings.NewReader("TestViaHeader"))
 128         c.Assert(err, Equals, nil)
 129         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 130         resp, err := (&http.Client{}).Do(req)
 131         c.Assert(err, Equals, nil)
 132         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 133         locator, err := ioutil.ReadAll(resp.Body)
 134         c.Assert(err, Equals, nil)
 135         resp.Body.Close()
 136
 137         req, err = http.NewRequest("GET",
 138                 "http://"+listener.Addr().String()+"/"+string(locator),
 139                 nil)
 140         c.Assert(err, Equals, nil)
 141         resp, err = (&http.Client{}).Do(req)
 142         c.Assert(err, Equals, nil)
 143         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 144         resp.Body.Close()
 145 }
 146
 147 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 148         kc := runProxy(c, nil, false)
 149         defer closeListener()
 150
 151         sr := map[string]string{
 152                 TestProxyUUID: "http://" + listener.Addr().String(),
 153         }
 154         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 155
 156         content := []byte("TestLoopDetection")
 157         _, _, err := kc.PutB(content)
 158         c.Check(err, ErrorMatches, `.*loop detected.*`)
 159
 160         hash := fmt.Sprintf("%x", md5.Sum(content))
 161         _, _, _, err = kc.Get(hash)
 162         c.Check(err, ErrorMatches, `.*loop detected.*`)
 163 }
 164
 165 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 166         kc := runProxy(c, nil, false)
 167         defer closeListener()
 168
 169         content := []byte("TestDesiredReplicas")
 170         hash := fmt.Sprintf("%x", md5.Sum(content))
 171
 172         for _, kc.Want_replicas = range []int{0, 1, 2} {
 173                 locator, rep, err := kc.PutB(content)
 174                 c.Check(err, Equals, nil)
 175                 c.Check(rep, Equals, kc.Want_replicas)
 176                 if rep > 0 {
 177                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 178                 }
 179         }
 180 }
 181
 182 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 183         kc := runProxy(c, nil, false)
 184         defer closeListener()
 185
 186         content := []byte("TestPutWrongContentLength")
 187         hash := fmt.Sprintf("%x", md5.Sum(content))
 188
 189         // If we use http.Client to send these requests to the network
 190         // server we just started, the Go http library automatically
 191         // fixes the invalid Content-Length header. In order to test
 192         // our server behavior, we have to call the handler directly
 193         // using an httptest.ResponseRecorder.
 194         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
 195
 196         type testcase struct {
 197                 sendLength   string
 198                 expectStatus int
 199         }
 200
 201         for _, t := range []testcase{
 202                 {"1", http.StatusBadRequest},
 203                 {"", http.StatusLengthRequired},
 204                 {"-1", http.StatusLengthRequired},
 205                 {"abcdef", http.StatusLengthRequired},
 206         } {
 207                 req, err := http.NewRequest("PUT",
 208                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 209                         bytes.NewReader(content))
 210                 c.Assert(err, IsNil)
 211                 req.Header.Set("Content-Length", t.sendLength)
 212                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 213                 req.Header.Set("Content-Type", "application/octet-stream")
 214
 215                 resp := httptest.NewRecorder()
 216                 rtr.ServeHTTP(resp, req)
 217                 c.Check(resp.Code, Equals, t.expectStatus)
 218         }
 219 }
 220
 221 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 222         kc := runProxy(c, nil, false)
 223         defer closeListener()
 224         router.(*proxyHandler).timeout = time.Nanosecond
 225
 226         buf := make([]byte, 1<<20)
 227         rand.Read(buf)
 228         var wg sync.WaitGroup
 229         for i := 0; i < 128; i++ {
 230                 wg.Add(1)
 231                 go func() {
 232                         defer wg.Done()
 233                         kc.PutB(buf)
 234                 }()
 235         }
 236         done := make(chan bool)
 237         go func() {
 238                 wg.Wait()
 239                 close(done)
 240         }()
 241         select {
 242         case <-done:
 243         case <-time.After(10 * time.Second):
 244                 c.Error("timeout")
 245         }
 246 }
 247
 248 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 249         kc := runProxy(c, nil, false)
 250         defer closeListener()
 251
 252         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 253         var hash2 string
 254
 255         {
 256                 _, _, err := kc.Ask(hash)
 257                 c.Check(err, Equals, keepclient.BlockNotFound)
 258                 c.Log("Finished Ask (expected BlockNotFound)")
 259         }
 260
 261         {
 262                 reader, _, _, err := kc.Get(hash)
 263                 c.Check(reader, Equals, nil)
 264                 c.Check(err, Equals, keepclient.BlockNotFound)
 265                 c.Log("Finished Get (expected BlockNotFound)")
 266         }
 267
 268         // Note in bug #5309 among other errors keepproxy would set
 269         // Content-Length incorrectly on the 404 BlockNotFound response, this
 270         // would result in a protocol violation that would prevent reuse of the
 271         // connection, which would manifest by the next attempt to use the
 272         // connection (in this case the PutB below) failing.  So to test for
 273         // that bug it's necessary to trigger an error response (such as
 274         // BlockNotFound) and then do something else with the same httpClient
 275         // connection.
 276
 277         {
 278                 var rep int
 279                 var err error
 280                 hash2, rep, err = kc.PutB([]byte("foo"))
 281                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 282                 c.Check(rep, Equals, 2)
 283                 c.Check(err, Equals, nil)
 284                 c.Log("Finished PutB (expected success)")
 285         }
 286
 287         {
 288                 blocklen, _, err := kc.Ask(hash2)
 289                 c.Assert(err, Equals, nil)
 290                 c.Check(blocklen, Equals, int64(3))
 291                 c.Log("Finished Ask (expected success)")
 292         }
 293
 294         {
 295                 reader, blocklen, _, err := kc.Get(hash2)
 296                 c.Assert(err, Equals, nil)
 297                 all, err := ioutil.ReadAll(reader)
 298                 c.Check(err, IsNil)
 299                 c.Check(all, DeepEquals, []byte("foo"))
 300                 c.Check(blocklen, Equals, int64(3))
 301                 c.Log("Finished Get (expected success)")
 302         }
 303
 304         {
 305                 var rep int
 306                 var err error
 307                 hash2, rep, err = kc.PutB([]byte(""))
 308                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 309                 c.Check(rep, Equals, 2)
 310                 c.Check(err, Equals, nil)
 311                 c.Log("Finished PutB zero block")
 312         }
 313
 314         {
 315                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 316                 c.Assert(err, Equals, nil)
 317                 all, err := ioutil.ReadAll(reader)
 318                 c.Check(err, IsNil)
 319                 c.Check(all, DeepEquals, []byte(""))
 320                 c.Check(blocklen, Equals, int64(0))
 321                 c.Log("Finished Get zero block")
 322         }
 323 }
 324
 325 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 326         kc := runProxy(c, nil, true)
 327         defer closeListener()
 328
 329         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 330
 331         _, _, err := kc.Ask(hash)
 332         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 333
 334         hash2, rep, err := kc.PutB([]byte("bar"))
 335         c.Check(hash2, Equals, "")
 336         c.Check(rep, Equals, 0)
 337         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 338
 339         blocklen, _, err := kc.Ask(hash)
 340         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 341         c.Check(err, ErrorMatches, ".*not found.*")
 342         c.Check(blocklen, Equals, int64(0))
 343
 344         _, blocklen, _, err = kc.Get(hash)
 345         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 346         c.Check(err, ErrorMatches, ".*not found.*")
 347         c.Check(blocklen, Equals, int64(0))
 348
 349 }
 350
 351 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 352         kc := runProxy(c, []string{"-no-get"}, false)
 353         defer closeListener()
 354
 355         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 356
 357         {
 358                 _, _, err := kc.Ask(hash)
 359                 errNotFound, _ := err.(keepclient.ErrNotFound)
 360                 c.Check(errNotFound, NotNil)
 361                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 362                 c.Log("Ask 1")
 363         }
 364
 365         {
 366                 hash2, rep, err := kc.PutB([]byte("baz"))
 367                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 368                 c.Check(rep, Equals, 2)
 369                 c.Check(err, Equals, nil)
 370                 c.Log("PutB")
 371         }
 372
 373         {
 374                 blocklen, _, err := kc.Ask(hash)
 375                 errNotFound, _ := err.(keepclient.ErrNotFound)
 376                 c.Check(errNotFound, NotNil)
 377                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 378                 c.Check(blocklen, Equals, int64(0))
 379                 c.Log("Ask 2")
 380         }
 381
 382         {
 383                 _, blocklen, _, err := kc.Get(hash)
 384                 errNotFound, _ := err.(keepclient.ErrNotFound)
 385                 c.Check(errNotFound, NotNil)
 386                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 387                 c.Check(blocklen, Equals, int64(0))
 388                 c.Log("Get")
 389         }
 390 }
 391
 392 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 393         kc := runProxy(c, []string{"-no-put"}, false)
 394         defer closeListener()
 395
 396         hash2, rep, err := kc.PutB([]byte("quux"))
 397         c.Check(hash2, Equals, "")
 398         c.Check(rep, Equals, 0)
 399         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 400 }
 401
 402 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 403         runProxy(c, nil, false)
 404         defer closeListener()
 405
 406         {
 407                 client := http.Client{}
 408                 req, err := http.NewRequest("OPTIONS",
 409                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 410                         nil)
 411                 c.Assert(err, IsNil)
 412                 req.Header.Add("Access-Control-Request-Method", "PUT")
 413                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 414                 resp, err := client.Do(req)
 415                 c.Check(err, Equals, nil)
 416                 c.Check(resp.StatusCode, Equals, 200)
 417                 body, err := ioutil.ReadAll(resp.Body)
 418                 c.Check(err, IsNil)
 419                 c.Check(string(body), Equals, "")
 420                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 421                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 422         }
 423
 424         {
 425                 resp, err := http.Get(
 426                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 427                 c.Check(err, Equals, nil)
 428                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 429                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 430         }
 431 }
 432
 433 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 434         runProxy(c, nil, false)
 435         defer closeListener()
 436
 437         {
 438                 client := http.Client{}
 439                 req, err := http.NewRequest("POST",
 440                         "http://"+listener.Addr().String()+"/",
 441                         strings.NewReader("qux"))
 442                 c.Check(err, IsNil)
 443                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 444                 req.Header.Add("Content-Type", "application/octet-stream")
 445                 resp, err := client.Do(req)
 446                 c.Check(err, Equals, nil)
 447                 body, err := ioutil.ReadAll(resp.Body)
 448                 c.Check(err, Equals, nil)
 449                 c.Check(string(body), Matches,
 450                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 451         }
 452 }
 453
 454 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 455         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 456                 Equals,
 457                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 458         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 459                 Equals,
 460                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 461         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 462                 Equals,
 463                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 464         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 465                 Equals,
 466                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 467
 468 }
 469
 470 // Test GetIndex
 471 //   Put one block, with 2 replicas
 472 //   With no prefix (expect the block locator, twice)
 473 //   With an existing prefix (expect the block locator, twice)
 474 //   With a valid but non-existing prefix (expect "\n")
 475 //   With an invalid prefix (expect error)
 476 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 477         kc := runProxy(c, nil, false)
 478         defer closeListener()
 479
 480         // Put "index-data" blocks
 481         data := []byte("index-data")
 482         hash := fmt.Sprintf("%x", md5.Sum(data))
 483
 484         hash2, rep, err := kc.PutB(data)
 485         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 486         c.Check(rep, Equals, 2)
 487         c.Check(err, Equals, nil)
 488
 489         reader, blocklen, _, err := kc.Get(hash)
 490         c.Assert(err, IsNil)
 491         c.Check(blocklen, Equals, int64(10))
 492         all, err := ioutil.ReadAll(reader)
 493         c.Assert(err, IsNil)
 494         c.Check(all, DeepEquals, data)
 495
 496         // Put some more blocks
 497         _, _, err = kc.PutB([]byte("some-more-index-data"))
 498         c.Check(err, IsNil)
 499
 500         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 501
 502         // Invoke GetIndex
 503         for _, spec := range []struct {
 504                 prefix         string
 505                 expectTestHash bool
 506                 expectOther    bool
 507         }{
 508                 {"", true, true},         // with no prefix
 509                 {hash[:3], true, false},  // with matching prefix
 510                 {"abcdef", false, false}, // with no such prefix
 511         } {
 512                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 513                 c.Assert(err, Equals, nil)
 514                 indexResp, err := ioutil.ReadAll(indexReader)
 515                 c.Assert(err, Equals, nil)
 516                 locators := strings.Split(string(indexResp), "\n")
 517                 gotTestHash := 0
 518                 gotOther := 0
 519                 for _, locator := range locators {
 520                         if locator == "" {
 521                                 continue
 522                         }
 523                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 524                         if locator[:32] == hash {
 525                                 gotTestHash++
 526                         } else {
 527                                 gotOther++
 528                         }
 529                 }
 530                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 531                 c.Check(gotOther > 0, Equals, spec.expectOther)
 532         }
 533
 534         // GetIndex with invalid prefix
 535         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 536         c.Assert((err != nil), Equals, true)
 537 }
 538
 539 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 540         kc := runProxy(c, nil, false)
 541         defer closeListener()
 542         hash, _, err := kc.PutB([]byte("shareddata"))
 543         c.Check(err, IsNil)
 544         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 545         rdr, _, _, err := kc.Get(hash)
 546         c.Assert(err, IsNil)
 547         data, err := ioutil.ReadAll(rdr)
 548         c.Check(err, IsNil)
 549         c.Check(data, DeepEquals, []byte("shareddata"))
 550 }
 551
 552 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 553         kc := runProxy(c, nil, false)
 554         defer closeListener()
 555
 556         // Put a test block
 557         hash, rep, err := kc.PutB([]byte("foo"))
 558         c.Check(err, IsNil)
 559         c.Check(rep, Equals, 2)
 560
 561         for _, badToken := range []string{
 562                 "nosuchtoken",
 563                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 564         } {
 565                 kc.Arvados.ApiToken = badToken
 566
 567                 // Ask and Get will fail only if the upstream
 568                 // keepstore server checks for valid signatures.
 569                 // Without knowing the blob signing key, there is no
 570                 // way for keepproxy to know whether a given token is
 571                 // permitted to read a block.  So these tests fail:
 572                 if false {
 573                         _, _, err = kc.Ask(hash)
 574                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 575                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 576                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 577
 578                         _, _, _, err = kc.Get(hash)
 579                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 580                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 581                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
 582                 }
 583
 584                 _, _, err = kc.PutB([]byte("foo"))
 585                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
 586         }
 587 }
 588
 589 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 590         arv, err := arvadosclient.MakeArvadosClient()
 591         c.Assert(err, Equals, nil)
 592
 593         // keepclient with no such keep server
 594         kc := keepclient.New(arv)
 595         locals := map[string]string{
 596                 TestProxyUUID: "http://localhost:12345",
 597         }
 598         kc.SetServiceRoots(locals, nil, nil)
 599
 600         // Ask should result in temporary connection refused error
 601         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 602         _, _, err = kc.Ask(hash)
 603         c.Check(err, NotNil)
 604         errNotFound, _ := err.(*keepclient.ErrNotFound)
 605         c.Check(errNotFound.Temporary(), Equals, true)
 606         c.Assert(err, ErrorMatches, ".*connection refused.*")
 607
 608         // Get should result in temporary connection refused error
 609         _, _, _, err = kc.Get(hash)
 610         c.Check(err, NotNil)
 611         errNotFound, _ = err.(*keepclient.ErrNotFound)
 612         c.Check(errNotFound.Temporary(), Equals, true)
 613         c.Assert(err, ErrorMatches, ".*connection refused.*")
 614 }
 615
 616 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 617         kc := runProxy(c, nil, false)
 618         defer closeListener()
 619
 620         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 621         for _, f := range []func() error{
 622                 func() error {
 623                         _, _, err := kc.Ask(hash)
 624                         return err
 625                 },
 626                 func() error {
 627                         _, _, _, err := kc.Get(hash)
 628                         return err
 629                 },
 630         } {
 631                 err := f()
 632                 c.Assert(err, NotNil)
 633                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 634                 c.Check(errNotFound.Temporary(), Equals, true)
 635                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 636         }
 637 }
 638
 639 func (s *ServerRequiredSuite) TestPing(c *C) {
 640         kc := runProxy(c, nil, false)
 641         defer closeListener()
 642
 643         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
 644
 645         req, err := http.NewRequest("GET",
 646                 "http://"+listener.Addr().String()+"/_health/ping",
 647                 nil)
 648         c.Assert(err, IsNil)
 649         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 650
 651         resp := httptest.NewRecorder()
 652         rtr.ServeHTTP(resp, req)
 653         c.Check(resp.Code, Equals, 200)
 654         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 655 }