services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "strings"
  17         "sync"
  18         "testing"
  19         "time"
  20
  21         "git.arvados.org/arvados.git/lib/config"
  22         "git.arvados.org/arvados.git/sdk/go/arvados"
  23         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  24         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         log "github.com/sirupsen/logrus"
  28
  29         "gopkg.in/check.v1"
  30         . "gopkg.in/check.v1"
  31 )
  32
  33 // Gocheck boilerplate
  34 func Test(t *testing.T) {
  35         TestingT(t)
  36 }
  37
  38 // Gocheck boilerplate
  39 var _ = Suite(&ServerRequiredSuite{})
  40
  41 // Tests that require the Keep server running
  42 type ServerRequiredSuite struct{}
  43
  44 // Gocheck boilerplate
  45 var _ = Suite(&ServerRequiredConfigYmlSuite{})
  46
  47 // Tests that require the Keep servers running as defined in config.yml
  48 type ServerRequiredConfigYmlSuite struct{}
  49
  50 // Gocheck boilerplate
  51 var _ = Suite(&NoKeepServerSuite{})
  52
  53 // Test with no keepserver to simulate errors
  54 type NoKeepServerSuite struct{}
  55
  56 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  57
  58 // Wait (up to 1 second) for keepproxy to listen on a port. This
  59 // avoids a race condition where we hit a "connection refused" error
  60 // because we start testing the proxy too soon.
  61 func waitForListener() {
  62         const (
  63                 ms = 5
  64         )
  65         for i := 0; listener == nil && i < 10000; i += ms {
  66                 time.Sleep(ms * time.Millisecond)
  67         }
  68         if listener == nil {
  69                 panic("Timed out waiting for listener to start")
  70         }
  71 }
  72
  73 func closeListener() {
  74         if listener != nil {
  75                 listener.Close()
  76         }
  77 }
  78
  79 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  80         arvadostest.StartAPI()
  81         arvadostest.StartKeep(2, false)
  82 }
  83
  84 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  85         arvadostest.ResetEnv()
  86 }
  87
  88 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  89         arvadostest.StopKeep(2)
  90         arvadostest.StopAPI()
  91 }
  92
  93 func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
  94         arvadostest.StartAPI()
  95         // config.yml defines 4 keepstores
  96         arvadostest.StartKeep(4, false)
  97 }
  98
  99 func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
 100         arvadostest.ResetEnv()
 101 }
 102
 103 func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
 104         arvadostest.StopKeep(4)
 105         arvadostest.StopAPI()
 106 }
 107
 108 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
 109         arvadostest.StartAPI()
 110         // We need API to have some keep services listed, but the
 111         // services themselves should be unresponsive.
 112         arvadostest.StartKeep(2, false)
 113         arvadostest.StopKeep(2)
 114 }
 115
 116 func (s *NoKeepServerSuite) SetUpTest(c *C) {
 117         arvadostest.ResetEnv()
 118 }
 119
 120 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
 121         arvadostest.StopAPI()
 122 }
 123
 124 func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool, kp *arvados.UploadDownloadRolePermissions) (*keepclient.KeepClient, *bytes.Buffer) {
 125         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
 126         c.Assert(err, Equals, nil)
 127         cluster, err := cfg.GetCluster("")
 128         c.Assert(err, Equals, nil)
 129
 130         if !loadKeepstoresFromConfig {
 131                 // Do not load Keepstore InternalURLs from the config file
 132                 cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
 133         }
 134
 135         cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: ":0"}: {}}
 136
 137         if kp != nil {
 138                 cluster.Collections.KeepproxyPermission = *kp
 139         }
 140
 141         listener = nil
 142         logbuf := &bytes.Buffer{}
 143         logger := log.New()
 144         logger.Out = logbuf
 145         go func() {
 146                 run(logger, cluster)
 147                 defer closeListener()
 148         }()
 149         waitForListener()
 150
 151         client := arvados.NewClientFromEnv()
 152         arv, err := arvadosclient.New(client)
 153         c.Assert(err, Equals, nil)
 154         if bogusClientToken {
 155                 arv.ApiToken = "bogus-token"
 156         }
 157         kc := keepclient.New(arv)
 158         sr := map[string]string{
 159                 TestProxyUUID: "http://" + listener.Addr().String(),
 160         }
 161         kc.SetServiceRoots(sr, sr, sr)
 162         kc.Arvados.External = true
 163
 164         return kc, logbuf
 165 }
 166
 167 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 168         runProxy(c, false, false, nil)
 169         defer closeListener()
 170
 171         req, err := http.NewRequest("POST",
 172                 "http://"+listener.Addr().String()+"/",
 173                 strings.NewReader("TestViaHeader"))
 174         c.Assert(err, Equals, nil)
 175         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 176         resp, err := (&http.Client{}).Do(req)
 177         c.Assert(err, Equals, nil)
 178         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 179         c.Assert(resp.StatusCode, Equals, http.StatusOK)
 180         locator, err := ioutil.ReadAll(resp.Body)
 181         c.Assert(err, Equals, nil)
 182         resp.Body.Close()
 183
 184         req, err = http.NewRequest("GET",
 185                 "http://"+listener.Addr().String()+"/"+string(locator),
 186                 nil)
 187         c.Assert(err, Equals, nil)
 188         resp, err = (&http.Client{}).Do(req)
 189         c.Assert(err, Equals, nil)
 190         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 191         resp.Body.Close()
 192 }
 193
 194 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 195         kc, _ := runProxy(c, false, false, nil)
 196         defer closeListener()
 197
 198         sr := map[string]string{
 199                 TestProxyUUID: "http://" + listener.Addr().String(),
 200         }
 201         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 202
 203         content := []byte("TestLoopDetection")
 204         _, _, err := kc.PutB(content)
 205         c.Check(err, ErrorMatches, `.*loop detected.*`)
 206
 207         hash := fmt.Sprintf("%x", md5.Sum(content))
 208         _, _, _, err = kc.Get(hash)
 209         c.Check(err, ErrorMatches, `.*loop detected.*`)
 210 }
 211
 212 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 213         kc, _ := runProxy(c, false, false, nil)
 214         defer closeListener()
 215
 216         // Set up fake keepstore to record request headers
 217         var hdr http.Header
 218         ts := httptest.NewServer(http.HandlerFunc(
 219                 func(w http.ResponseWriter, r *http.Request) {
 220                         hdr = r.Header
 221                         http.Error(w, "Error", http.StatusInternalServerError)
 222                 }))
 223         defer ts.Close()
 224
 225         // Point keepproxy router's keepclient to the fake keepstore
 226         sr := map[string]string{
 227                 TestProxyUUID: ts.URL,
 228         }
 229         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 230
 231         // Set up client to ask for storage classes to keepproxy
 232         kc.StorageClasses = []string{"secure"}
 233         content := []byte("Very important data")
 234         _, _, err := kc.PutB(content)
 235         c.Check(err, NotNil)
 236         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 237 }
 238
 239 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 240         kc, _ := runProxy(c, false, false, nil)
 241         defer closeListener()
 242
 243         content := []byte("TestDesiredReplicas")
 244         hash := fmt.Sprintf("%x", md5.Sum(content))
 245
 246         for _, kc.Want_replicas = range []int{0, 1, 2} {
 247                 locator, rep, err := kc.PutB(content)
 248                 c.Check(err, Equals, nil)
 249                 c.Check(rep, Equals, kc.Want_replicas)
 250                 if rep > 0 {
 251                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 252                 }
 253         }
 254 }
 255
 256 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 257         kc, _ := runProxy(c, false, false, nil)
 258         defer closeListener()
 259
 260         content := []byte("TestPutWrongContentLength")
 261         hash := fmt.Sprintf("%x", md5.Sum(content))
 262
 263         // If we use http.Client to send these requests to the network
 264         // server we just started, the Go http library automatically
 265         // fixes the invalid Content-Length header. In order to test
 266         // our server behavior, we have to call the handler directly
 267         // using an httptest.ResponseRecorder.
 268         rtr, err := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{}, log.New())
 269         c.Assert(err, check.IsNil)
 270
 271         type testcase struct {
 272                 sendLength   string
 273                 expectStatus int
 274         }
 275
 276         for _, t := range []testcase{
 277                 {"1", http.StatusBadRequest},
 278                 {"", http.StatusLengthRequired},
 279                 {"-1", http.StatusLengthRequired},
 280                 {"abcdef", http.StatusLengthRequired},
 281         } {
 282                 req, err := http.NewRequest("PUT",
 283                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 284                         bytes.NewReader(content))
 285                 c.Assert(err, IsNil)
 286                 req.Header.Set("Content-Length", t.sendLength)
 287                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 288                 req.Header.Set("Content-Type", "application/octet-stream")
 289
 290                 resp := httptest.NewRecorder()
 291                 rtr.ServeHTTP(resp, req)
 292                 c.Check(resp.Code, Equals, t.expectStatus)
 293         }
 294 }
 295
 296 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 297         kc, _ := runProxy(c, false, false, nil)
 298         defer closeListener()
 299         router.(*proxyHandler).timeout = time.Nanosecond
 300
 301         buf := make([]byte, 1<<20)
 302         rand.Read(buf)
 303         var wg sync.WaitGroup
 304         for i := 0; i < 128; i++ {
 305                 wg.Add(1)
 306                 go func() {
 307                         defer wg.Done()
 308                         kc.PutB(buf)
 309                 }()
 310         }
 311         done := make(chan bool)
 312         go func() {
 313                 wg.Wait()
 314                 close(done)
 315         }()
 316         select {
 317         case <-done:
 318         case <-time.After(10 * time.Second):
 319                 c.Error("timeout")
 320         }
 321 }
 322
 323 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 324         kc, logbuf := runProxy(c, false, false, nil)
 325         defer closeListener()
 326
 327         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 328         var hash2 string
 329
 330         {
 331                 _, _, err := kc.Ask(hash)
 332                 c.Check(err, Equals, keepclient.BlockNotFound)
 333                 c.Log("Finished Ask (expected BlockNotFound)")
 334         }
 335
 336         {
 337                 reader, _, _, err := kc.Get(hash)
 338                 c.Check(reader, Equals, nil)
 339                 c.Check(err, Equals, keepclient.BlockNotFound)
 340                 c.Log("Finished Get (expected BlockNotFound)")
 341         }
 342
 343         // Note in bug #5309 among other errors keepproxy would set
 344         // Content-Length incorrectly on the 404 BlockNotFound response, this
 345         // would result in a protocol violation that would prevent reuse of the
 346         // connection, which would manifest by the next attempt to use the
 347         // connection (in this case the PutB below) failing.  So to test for
 348         // that bug it's necessary to trigger an error response (such as
 349         // BlockNotFound) and then do something else with the same httpClient
 350         // connection.
 351
 352         {
 353                 var rep int
 354                 var err error
 355                 hash2, rep, err = kc.PutB([]byte("foo"))
 356                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 357                 c.Check(rep, Equals, 2)
 358                 c.Check(err, Equals, nil)
 359                 c.Log("Finished PutB (expected success)")
 360
 361                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 362                 logbuf.Reset()
 363         }
 364
 365         {
 366                 blocklen, _, err := kc.Ask(hash2)
 367                 c.Assert(err, Equals, nil)
 368                 c.Check(blocklen, Equals, int64(3))
 369                 c.Log("Finished Ask (expected success)")
 370                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 371                 logbuf.Reset()
 372         }
 373
 374         {
 375                 reader, blocklen, _, err := kc.Get(hash2)
 376                 c.Assert(err, Equals, nil)
 377                 all, err := ioutil.ReadAll(reader)
 378                 c.Check(err, IsNil)
 379                 c.Check(all, DeepEquals, []byte("foo"))
 380                 c.Check(blocklen, Equals, int64(3))
 381                 c.Log("Finished Get (expected success)")
 382                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 383                 logbuf.Reset()
 384         }
 385
 386         {
 387                 var rep int
 388                 var err error
 389                 hash2, rep, err = kc.PutB([]byte(""))
 390                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 391                 c.Check(rep, Equals, 2)
 392                 c.Check(err, Equals, nil)
 393                 c.Log("Finished PutB zero block")
 394         }
 395
 396         {
 397                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 398                 c.Assert(err, Equals, nil)
 399                 all, err := ioutil.ReadAll(reader)
 400                 c.Check(err, IsNil)
 401                 c.Check(all, DeepEquals, []byte(""))
 402                 c.Check(blocklen, Equals, int64(0))
 403                 c.Log("Finished Get zero block")
 404         }
 405 }
 406
 407 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 408         kc, _ := runProxy(c, true, false, nil)
 409         defer closeListener()
 410
 411         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 412
 413         _, _, err := kc.Ask(hash)
 414         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 415
 416         hash2, rep, err := kc.PutB([]byte("bar"))
 417         c.Check(hash2, Equals, "")
 418         c.Check(rep, Equals, 0)
 419         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 420
 421         blocklen, _, err := kc.Ask(hash)
 422         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 423         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 424         c.Check(blocklen, Equals, int64(0))
 425
 426         _, blocklen, _, err = kc.Get(hash)
 427         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 428         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 429         c.Check(blocklen, Equals, int64(0))
 430 }
 431
 432 func testPermission(c *C, admin bool, perm arvados.UploadDownloadPermission) {
 433         kp := arvados.UploadDownloadRolePermissions{}
 434         if admin {
 435                 kp.Admin = perm
 436                 kp.User = arvados.UploadDownloadPermission{Upload: true, Download: true}
 437         } else {
 438                 kp.Admin = arvados.UploadDownloadPermission{Upload: true, Download: true}
 439                 kp.User = perm
 440         }
 441
 442         kc, logbuf := runProxy(c, false, false, &kp)
 443         defer closeListener()
 444         if admin {
 445                 kc.Arvados.ApiToken = arvadostest.AdminToken
 446         } else {
 447                 kc.Arvados.ApiToken = arvadostest.ActiveToken
 448         }
 449
 450         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 451         var hash2 string
 452
 453         {
 454                 var rep int
 455                 var err error
 456                 hash2, rep, err = kc.PutB([]byte("foo"))
 457
 458                 if perm.Upload {
 459                         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 460                         c.Check(rep, Equals, 2)
 461                         c.Check(err, Equals, nil)
 462                         c.Log("Finished PutB (expected success)")
 463                         if admin {
 464                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 465                         } else {
 466
 467                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
 468                         }
 469                 } else {
 470                         c.Check(hash2, Equals, "")
 471                         c.Check(rep, Equals, 0)
 472                         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 473                 }
 474                 logbuf.Reset()
 475         }
 476         if perm.Upload {
 477                 // can't test download without upload.
 478
 479                 reader, blocklen, _, err := kc.Get(hash2)
 480                 if perm.Download {
 481                         c.Assert(err, Equals, nil)
 482                         all, err := ioutil.ReadAll(reader)
 483                         c.Check(err, IsNil)
 484                         c.Check(all, DeepEquals, []byte("foo"))
 485                         c.Check(blocklen, Equals, int64(3))
 486                         c.Log("Finished Get (expected success)")
 487                         if admin {
 488                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 489                         } else {
 490                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
 491                         }
 492                 } else {
 493                         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 494                         c.Check(err, ErrorMatches, ".*Missing or invalid Authorization header, or method not allowed.*")
 495                         c.Check(blocklen, Equals, int64(0))
 496                 }
 497                 logbuf.Reset()
 498         }
 499
 500 }
 501
 502 func (s *ServerRequiredSuite) TestPutGetPermission(c *C) {
 503
 504         for _, adminperm := range []bool{true, false} {
 505                 for _, userperm := range []bool{true, false} {
 506
 507                         testPermission(c, true,
 508                                 arvados.UploadDownloadPermission{
 509                                         Upload:   adminperm,
 510                                         Download: true,
 511                                 })
 512                         testPermission(c, true,
 513                                 arvados.UploadDownloadPermission{
 514                                         Upload:   true,
 515                                         Download: adminperm,
 516                                 })
 517                         testPermission(c, false,
 518                                 arvados.UploadDownloadPermission{
 519                                         Upload:   true,
 520                                         Download: userperm,
 521                                 })
 522                         testPermission(c, false,
 523                                 arvados.UploadDownloadPermission{
 524                                         Upload:   true,
 525                                         Download: userperm,
 526                                 })
 527                 }
 528         }
 529 }
 530
 531 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 532         runProxy(c, false, false, nil)
 533         defer closeListener()
 534
 535         {
 536                 client := http.Client{}
 537                 req, err := http.NewRequest("OPTIONS",
 538                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 539                         nil)
 540                 c.Assert(err, IsNil)
 541                 req.Header.Add("Access-Control-Request-Method", "PUT")
 542                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 543                 resp, err := client.Do(req)
 544                 c.Check(err, Equals, nil)
 545                 c.Check(resp.StatusCode, Equals, 200)
 546                 body, err := ioutil.ReadAll(resp.Body)
 547                 c.Check(err, IsNil)
 548                 c.Check(string(body), Equals, "")
 549                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 550                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 551         }
 552
 553         {
 554                 resp, err := http.Get(
 555                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 556                 c.Check(err, Equals, nil)
 557                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 558                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 559         }
 560 }
 561
 562 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 563         runProxy(c, false, false, nil)
 564         defer closeListener()
 565
 566         {
 567                 client := http.Client{}
 568                 req, err := http.NewRequest("POST",
 569                         "http://"+listener.Addr().String()+"/",
 570                         strings.NewReader("qux"))
 571                 c.Check(err, IsNil)
 572                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 573                 req.Header.Add("Content-Type", "application/octet-stream")
 574                 resp, err := client.Do(req)
 575                 c.Check(err, Equals, nil)
 576                 body, err := ioutil.ReadAll(resp.Body)
 577                 c.Check(err, Equals, nil)
 578                 c.Check(string(body), Matches,
 579                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 580         }
 581 }
 582
 583 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 584         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 585                 Equals,
 586                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 587         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 588                 Equals,
 589                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 590         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 591                 Equals,
 592                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 593         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 594                 Equals,
 595                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 596
 597 }
 598
 599 // Test GetIndex
 600 //   Put one block, with 2 replicas
 601 //   With no prefix (expect the block locator, twice)
 602 //   With an existing prefix (expect the block locator, twice)
 603 //   With a valid but non-existing prefix (expect "\n")
 604 //   With an invalid prefix (expect error)
 605 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 606         getIndexWorker(c, false)
 607 }
 608
 609 // Test GetIndex
 610 //   Uses config.yml
 611 //   Put one block, with 2 replicas
 612 //   With no prefix (expect the block locator, twice)
 613 //   With an existing prefix (expect the block locator, twice)
 614 //   With a valid but non-existing prefix (expect "\n")
 615 //   With an invalid prefix (expect error)
 616 func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
 617         getIndexWorker(c, true)
 618 }
 619
 620 func getIndexWorker(c *C, useConfig bool) {
 621         kc, _ := runProxy(c, false, useConfig, nil)
 622         defer closeListener()
 623
 624         // Put "index-data" blocks
 625         data := []byte("index-data")
 626         hash := fmt.Sprintf("%x", md5.Sum(data))
 627
 628         hash2, rep, err := kc.PutB(data)
 629         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 630         c.Check(rep, Equals, 2)
 631         c.Check(err, Equals, nil)
 632
 633         reader, blocklen, _, err := kc.Get(hash)
 634         c.Assert(err, IsNil)
 635         c.Check(blocklen, Equals, int64(10))
 636         all, err := ioutil.ReadAll(reader)
 637         c.Assert(err, IsNil)
 638         c.Check(all, DeepEquals, data)
 639
 640         // Put some more blocks
 641         _, _, err = kc.PutB([]byte("some-more-index-data"))
 642         c.Check(err, IsNil)
 643
 644         kc.Arvados.ApiToken = arvadostest.SystemRootToken
 645
 646         // Invoke GetIndex
 647         for _, spec := range []struct {
 648                 prefix         string
 649                 expectTestHash bool
 650                 expectOther    bool
 651         }{
 652                 {"", true, true},         // with no prefix
 653                 {hash[:3], true, false},  // with matching prefix
 654                 {"abcdef", false, false}, // with no such prefix
 655         } {
 656                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 657                 c.Assert(err, Equals, nil)
 658                 indexResp, err := ioutil.ReadAll(indexReader)
 659                 c.Assert(err, Equals, nil)
 660                 locators := strings.Split(string(indexResp), "\n")
 661                 gotTestHash := 0
 662                 gotOther := 0
 663                 for _, locator := range locators {
 664                         if locator == "" {
 665                                 continue
 666                         }
 667                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 668                         if locator[:32] == hash {
 669                                 gotTestHash++
 670                         } else {
 671                                 gotOther++
 672                         }
 673                 }
 674                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 675                 c.Check(gotOther > 0, Equals, spec.expectOther)
 676         }
 677
 678         // GetIndex with invalid prefix
 679         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 680         c.Assert((err != nil), Equals, true)
 681 }
 682
 683 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 684         kc, _ := runProxy(c, false, false, nil)
 685         defer closeListener()
 686         hash, _, err := kc.PutB([]byte("shareddata"))
 687         c.Check(err, IsNil)
 688         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 689         rdr, _, _, err := kc.Get(hash)
 690         c.Assert(err, IsNil)
 691         data, err := ioutil.ReadAll(rdr)
 692         c.Check(err, IsNil)
 693         c.Check(data, DeepEquals, []byte("shareddata"))
 694 }
 695
 696 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 697         kc, _ := runProxy(c, false, false, nil)
 698         defer closeListener()
 699
 700         // Put a test block
 701         hash, rep, err := kc.PutB([]byte("foo"))
 702         c.Check(err, IsNil)
 703         c.Check(rep, Equals, 2)
 704
 705         for _, badToken := range []string{
 706                 "nosuchtoken",
 707                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 708         } {
 709                 kc.Arvados.ApiToken = badToken
 710
 711                 // Ask and Get will fail only if the upstream
 712                 // keepstore server checks for valid signatures.
 713                 // Without knowing the blob signing key, there is no
 714                 // way for keepproxy to know whether a given token is
 715                 // permitted to read a block.  So these tests fail:
 716                 if false {
 717                         _, _, err = kc.Ask(hash)
 718                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 719                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 720                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 721
 722                         _, _, _, err = kc.Get(hash)
 723                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 724                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 725                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header, or method not allowed\".*")
 726                 }
 727
 728                 _, _, err = kc.PutB([]byte("foo"))
 729                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header, or method not allowed")
 730         }
 731 }
 732
 733 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 734         kc, _ := runProxy(c, false, false, nil)
 735         defer closeListener()
 736
 737         // Point keepproxy at a non-existent keepstore
 738         locals := map[string]string{
 739                 TestProxyUUID: "http://localhost:12345",
 740         }
 741         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
 742
 743         // Ask should result in temporary bad gateway error
 744         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 745         _, _, err := kc.Ask(hash)
 746         c.Check(err, NotNil)
 747         errNotFound, _ := err.(*keepclient.ErrNotFound)
 748         c.Check(errNotFound.Temporary(), Equals, true)
 749         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 750
 751         // Get should result in temporary bad gateway error
 752         _, _, _, err = kc.Get(hash)
 753         c.Check(err, NotNil)
 754         errNotFound, _ = err.(*keepclient.ErrNotFound)
 755         c.Check(errNotFound.Temporary(), Equals, true)
 756         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 757 }
 758
 759 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 760         kc, _ := runProxy(c, false, false, nil)
 761         defer closeListener()
 762
 763         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 764         for _, f := range []func() error{
 765                 func() error {
 766                         _, _, err := kc.Ask(hash)
 767                         return err
 768                 },
 769                 func() error {
 770                         _, _, _, err := kc.Get(hash)
 771                         return err
 772                 },
 773         } {
 774                 err := f()
 775                 c.Assert(err, NotNil)
 776                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 777                 c.Check(errNotFound.Temporary(), Equals, true)
 778                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 779         }
 780 }
 781
 782 func (s *ServerRequiredSuite) TestPing(c *C) {
 783         kc, _ := runProxy(c, false, false, nil)
 784         defer closeListener()
 785
 786         rtr, err := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{ManagementToken: arvadostest.ManagementToken}, log.New())
 787         c.Assert(err, check.IsNil)
 788
 789         req, err := http.NewRequest("GET",
 790                 "http://"+listener.Addr().String()+"/_health/ping",
 791                 nil)
 792         c.Assert(err, IsNil)
 793         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 794
 795         resp := httptest.NewRecorder()
 796         rtr.ServeHTTP(resp, req)
 797         c.Check(resp.Code, Equals, 200)
 798         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 799 }