services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io/ioutil"
  13         "math/rand"
  14         "net/http"
  15         "net/http/httptest"
  16         "strings"
  17         "sync"
  18         "testing"
  19         "time"
  20
  21         "git.arvados.org/arvados.git/lib/config"
  22         "git.arvados.org/arvados.git/sdk/go/arvados"
  23         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  24         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         log "github.com/sirupsen/logrus"
  28
  29         . "gopkg.in/check.v1"
  30 )
  31
  32 // Gocheck boilerplate
  33 func Test(t *testing.T) {
  34         TestingT(t)
  35 }
  36
  37 // Gocheck boilerplate
  38 var _ = Suite(&ServerRequiredSuite{})
  39
  40 // Tests that require the Keep server running
  41 type ServerRequiredSuite struct{}
  42
  43 // Gocheck boilerplate
  44 var _ = Suite(&ServerRequiredConfigYmlSuite{})
  45
  46 // Tests that require the Keep servers running as defined in config.yml
  47 type ServerRequiredConfigYmlSuite struct{}
  48
  49 // Gocheck boilerplate
  50 var _ = Suite(&NoKeepServerSuite{})
  51
  52 // Test with no keepserver to simulate errors
  53 type NoKeepServerSuite struct{}
  54
  55 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  56
  57 // Wait (up to 1 second) for keepproxy to listen on a port. This
  58 // avoids a race condition where we hit a "connection refused" error
  59 // because we start testing the proxy too soon.
  60 func waitForListener() {
  61         const (
  62                 ms = 5
  63         )
  64         for i := 0; listener == nil && i < 10000; i += ms {
  65                 time.Sleep(ms * time.Millisecond)
  66         }
  67         if listener == nil {
  68                 panic("Timed out waiting for listener to start")
  69         }
  70 }
  71
  72 func closeListener() {
  73         if listener != nil {
  74                 listener.Close()
  75         }
  76 }
  77
  78 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  79         arvadostest.StartAPI()
  80         arvadostest.StartKeep(2, false)
  81 }
  82
  83 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  84         arvadostest.ResetEnv()
  85 }
  86
  87 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  88         arvadostest.StopKeep(2)
  89         arvadostest.StopAPI()
  90 }
  91
  92 func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
  93         arvadostest.StartAPI()
  94         // config.yml defines 4 keepstores
  95         arvadostest.StartKeep(4, false)
  96 }
  97
  98 func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
  99         arvadostest.ResetEnv()
 100 }
 101
 102 func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
 103         arvadostest.StopKeep(4)
 104         arvadostest.StopAPI()
 105 }
 106
 107 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
 108         arvadostest.StartAPI()
 109         // We need API to have some keep services listed, but the
 110         // services themselves should be unresponsive.
 111         arvadostest.StartKeep(2, false)
 112         arvadostest.StopKeep(2)
 113 }
 114
 115 func (s *NoKeepServerSuite) SetUpTest(c *C) {
 116         arvadostest.ResetEnv()
 117 }
 118
 119 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
 120         arvadostest.StopAPI()
 121 }
 122
 123 func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool, kp *arvados.UploadDownloadRolePermissions) (*keepclient.KeepClient, *bytes.Buffer) {
 124         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
 125         c.Assert(err, Equals, nil)
 126         cluster, err := cfg.GetCluster("")
 127         c.Assert(err, Equals, nil)
 128
 129         if !loadKeepstoresFromConfig {
 130                 // Do not load Keepstore InternalURLs from the config file
 131                 cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
 132         }
 133
 134         cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: ":0"}: {}}
 135
 136         if kp != nil {
 137                 cluster.Collections.KeepproxyPermission = *kp
 138         }
 139
 140         listener = nil
 141         logbuf := &bytes.Buffer{}
 142         logger := log.New()
 143         logger.Out = logbuf
 144         go func() {
 145                 run(logger, cluster)
 146                 defer closeListener()
 147         }()
 148         waitForListener()
 149
 150         client := arvados.NewClientFromEnv()
 151         arv, err := arvadosclient.New(client)
 152         c.Assert(err, Equals, nil)
 153         if bogusClientToken {
 154                 arv.ApiToken = "bogus-token"
 155         }
 156         kc := keepclient.New(arv)
 157         sr := map[string]string{
 158                 TestProxyUUID: "http://" + listener.Addr().String(),
 159         }
 160         kc.SetServiceRoots(sr, sr, sr)
 161         kc.Arvados.External = true
 162
 163         return kc, logbuf
 164 }
 165
 166 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 167         runProxy(c, false, false, nil)
 168         defer closeListener()
 169
 170         req, err := http.NewRequest("POST",
 171                 "http://"+listener.Addr().String()+"/",
 172                 strings.NewReader("TestViaHeader"))
 173         c.Assert(err, Equals, nil)
 174         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 175         resp, err := (&http.Client{}).Do(req)
 176         c.Assert(err, Equals, nil)
 177         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 178         c.Assert(resp.StatusCode, Equals, http.StatusOK)
 179         locator, err := ioutil.ReadAll(resp.Body)
 180         c.Assert(err, Equals, nil)
 181         resp.Body.Close()
 182
 183         req, err = http.NewRequest("GET",
 184                 "http://"+listener.Addr().String()+"/"+string(locator),
 185                 nil)
 186         c.Assert(err, Equals, nil)
 187         resp, err = (&http.Client{}).Do(req)
 188         c.Assert(err, Equals, nil)
 189         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 190         resp.Body.Close()
 191 }
 192
 193 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 194         kc, _ := runProxy(c, false, false, nil)
 195         defer closeListener()
 196
 197         sr := map[string]string{
 198                 TestProxyUUID: "http://" + listener.Addr().String(),
 199         }
 200         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 201
 202         content := []byte("TestLoopDetection")
 203         _, _, err := kc.PutB(content)
 204         c.Check(err, ErrorMatches, `.*loop detected.*`)
 205
 206         hash := fmt.Sprintf("%x", md5.Sum(content))
 207         _, _, _, err = kc.Get(hash)
 208         c.Check(err, ErrorMatches, `.*loop detected.*`)
 209 }
 210
 211 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 212         kc, _ := runProxy(c, false, false, nil)
 213         defer closeListener()
 214
 215         // Set up fake keepstore to record request headers
 216         var hdr http.Header
 217         ts := httptest.NewServer(http.HandlerFunc(
 218                 func(w http.ResponseWriter, r *http.Request) {
 219                         hdr = r.Header
 220                         http.Error(w, "Error", http.StatusInternalServerError)
 221                 }))
 222         defer ts.Close()
 223
 224         // Point keepproxy router's keepclient to the fake keepstore
 225         sr := map[string]string{
 226                 TestProxyUUID: ts.URL,
 227         }
 228         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 229
 230         // Set up client to ask for storage classes to keepproxy
 231         kc.StorageClasses = []string{"secure"}
 232         content := []byte("Very important data")
 233         _, _, err := kc.PutB(content)
 234         c.Check(err, NotNil)
 235         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 236 }
 237
 238 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 239         kc, _ := runProxy(c, false, false, nil)
 240         defer closeListener()
 241
 242         content := []byte("TestDesiredReplicas")
 243         hash := fmt.Sprintf("%x", md5.Sum(content))
 244
 245         for _, kc.Want_replicas = range []int{0, 1, 2} {
 246                 locator, rep, err := kc.PutB(content)
 247                 c.Check(err, Equals, nil)
 248                 c.Check(rep, Equals, kc.Want_replicas)
 249                 if rep > 0 {
 250                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 251                 }
 252         }
 253 }
 254
 255 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 256         kc, _ := runProxy(c, false, false, nil)
 257         defer closeListener()
 258
 259         content := []byte("TestPutWrongContentLength")
 260         hash := fmt.Sprintf("%x", md5.Sum(content))
 261
 262         // If we use http.Client to send these requests to the network
 263         // server we just started, the Go http library automatically
 264         // fixes the invalid Content-Length header. In order to test
 265         // our server behavior, we have to call the handler directly
 266         // using an httptest.ResponseRecorder.
 267         rtr := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{}, log.New())
 268
 269         type testcase struct {
 270                 sendLength   string
 271                 expectStatus int
 272         }
 273
 274         for _, t := range []testcase{
 275                 {"1", http.StatusBadRequest},
 276                 {"", http.StatusLengthRequired},
 277                 {"-1", http.StatusLengthRequired},
 278                 {"abcdef", http.StatusLengthRequired},
 279         } {
 280                 req, err := http.NewRequest("PUT",
 281                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 282                         bytes.NewReader(content))
 283                 c.Assert(err, IsNil)
 284                 req.Header.Set("Content-Length", t.sendLength)
 285                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 286                 req.Header.Set("Content-Type", "application/octet-stream")
 287
 288                 resp := httptest.NewRecorder()
 289                 rtr.ServeHTTP(resp, req)
 290                 c.Check(resp.Code, Equals, t.expectStatus)
 291         }
 292 }
 293
 294 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 295         kc, _ := runProxy(c, false, false, nil)
 296         defer closeListener()
 297         router.(*proxyHandler).timeout = time.Nanosecond
 298
 299         buf := make([]byte, 1<<20)
 300         rand.Read(buf)
 301         var wg sync.WaitGroup
 302         for i := 0; i < 128; i++ {
 303                 wg.Add(1)
 304                 go func() {
 305                         defer wg.Done()
 306                         kc.PutB(buf)
 307                 }()
 308         }
 309         done := make(chan bool)
 310         go func() {
 311                 wg.Wait()
 312                 close(done)
 313         }()
 314         select {
 315         case <-done:
 316         case <-time.After(10 * time.Second):
 317                 c.Error("timeout")
 318         }
 319 }
 320
 321 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 322         kc, logbuf := runProxy(c, false, false, nil)
 323         defer closeListener()
 324
 325         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 326         var hash2 string
 327
 328         {
 329                 _, _, err := kc.Ask(hash)
 330                 c.Check(err, Equals, keepclient.BlockNotFound)
 331                 c.Log("Finished Ask (expected BlockNotFound)")
 332         }
 333
 334         {
 335                 reader, _, _, err := kc.Get(hash)
 336                 c.Check(reader, Equals, nil)
 337                 c.Check(err, Equals, keepclient.BlockNotFound)
 338                 c.Log("Finished Get (expected BlockNotFound)")
 339         }
 340
 341         // Note in bug #5309 among other errors keepproxy would set
 342         // Content-Length incorrectly on the 404 BlockNotFound response, this
 343         // would result in a protocol violation that would prevent reuse of the
 344         // connection, which would manifest by the next attempt to use the
 345         // connection (in this case the PutB below) failing.  So to test for
 346         // that bug it's necessary to trigger an error response (such as
 347         // BlockNotFound) and then do something else with the same httpClient
 348         // connection.
 349
 350         {
 351                 var rep int
 352                 var err error
 353                 hash2, rep, err = kc.PutB([]byte("foo"))
 354                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 355                 c.Check(rep, Equals, 2)
 356                 c.Check(err, Equals, nil)
 357                 c.Log("Finished PutB (expected success)")
 358
 359                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 360                 logbuf.Reset()
 361         }
 362
 363         {
 364                 blocklen, _, err := kc.Ask(hash2)
 365                 c.Assert(err, Equals, nil)
 366                 c.Check(blocklen, Equals, int64(3))
 367                 c.Log("Finished Ask (expected success)")
 368                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 369                 logbuf.Reset()
 370         }
 371
 372         {
 373                 reader, blocklen, _, err := kc.Get(hash2)
 374                 c.Assert(err, Equals, nil)
 375                 all, err := ioutil.ReadAll(reader)
 376                 c.Check(err, IsNil)
 377                 c.Check(all, DeepEquals, []byte("foo"))
 378                 c.Check(blocklen, Equals, int64(3))
 379                 c.Log("Finished Get (expected success)")
 380                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 381                 logbuf.Reset()
 382         }
 383
 384         {
 385                 var rep int
 386                 var err error
 387                 hash2, rep, err = kc.PutB([]byte(""))
 388                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 389                 c.Check(rep, Equals, 2)
 390                 c.Check(err, Equals, nil)
 391                 c.Log("Finished PutB zero block")
 392         }
 393
 394         {
 395                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 396                 c.Assert(err, Equals, nil)
 397                 all, err := ioutil.ReadAll(reader)
 398                 c.Check(err, IsNil)
 399                 c.Check(all, DeepEquals, []byte(""))
 400                 c.Check(blocklen, Equals, int64(0))
 401                 c.Log("Finished Get zero block")
 402         }
 403 }
 404
 405 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 406         kc, _ := runProxy(c, true, false, nil)
 407         defer closeListener()
 408
 409         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 410
 411         _, _, err := kc.Ask(hash)
 412         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 413
 414         hash2, rep, err := kc.PutB([]byte("bar"))
 415         c.Check(hash2, Equals, "")
 416         c.Check(rep, Equals, 0)
 417         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 418
 419         blocklen, _, err := kc.Ask(hash)
 420         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 421         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 422         c.Check(blocklen, Equals, int64(0))
 423
 424         _, blocklen, _, err = kc.Get(hash)
 425         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 426         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 427         c.Check(blocklen, Equals, int64(0))
 428 }
 429
 430 func testPermission(c *C, admin bool, perm arvados.UploadDownloadPermission) {
 431         kp := arvados.UploadDownloadRolePermissions{}
 432         if admin {
 433                 kp.Admin = perm
 434                 kp.User = arvados.UploadDownloadPermission{Upload: true, Download: true}
 435         } else {
 436                 kp.Admin = arvados.UploadDownloadPermission{Upload: true, Download: true}
 437                 kp.User = perm
 438         }
 439
 440         kc, logbuf := runProxy(c, false, false, &kp)
 441         defer closeListener()
 442         if admin {
 443                 kc.Arvados.ApiToken = arvadostest.AdminToken
 444         } else {
 445                 kc.Arvados.ApiToken = arvadostest.ActiveToken
 446         }
 447
 448         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 449         var hash2 string
 450
 451         {
 452                 var rep int
 453                 var err error
 454                 hash2, rep, err = kc.PutB([]byte("foo"))
 455
 456                 if perm.Upload {
 457                         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 458                         c.Check(rep, Equals, 2)
 459                         c.Check(err, Equals, nil)
 460                         c.Log("Finished PutB (expected success)")
 461                         if admin {
 462                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 463                         } else {
 464
 465                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
 466                         }
 467                 } else {
 468                         c.Check(hash2, Equals, "")
 469                         c.Check(rep, Equals, 0)
 470                         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 471                 }
 472                 logbuf.Reset()
 473         }
 474         if perm.Upload {
 475                 // can't test download without upload.
 476
 477                 reader, blocklen, _, err := kc.Get(hash2)
 478                 if perm.Download {
 479                         c.Assert(err, Equals, nil)
 480                         all, err := ioutil.ReadAll(reader)
 481                         c.Check(err, IsNil)
 482                         c.Check(all, DeepEquals, []byte("foo"))
 483                         c.Check(blocklen, Equals, int64(3))
 484                         c.Log("Finished Get (expected success)")
 485                         if admin {
 486                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 487                         } else {
 488                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
 489                         }
 490                 } else {
 491                         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 492                         c.Check(err, ErrorMatches, ".*Missing or invalid Authorization header, or method not allowed.*")
 493                         c.Check(blocklen, Equals, int64(0))
 494                 }
 495                 logbuf.Reset()
 496         }
 497
 498 }
 499
 500 func (s *ServerRequiredSuite) TestPutGetPermission(c *C) {
 501
 502         for _, adminperm := range []bool{true, false} {
 503                 for _, userperm := range []bool{true, false} {
 504
 505                         testPermission(c, true,
 506                                 arvados.UploadDownloadPermission{
 507                                         Upload:   adminperm,
 508                                         Download: true,
 509                                 })
 510                         testPermission(c, true,
 511                                 arvados.UploadDownloadPermission{
 512                                         Upload:   true,
 513                                         Download: adminperm,
 514                                 })
 515                         testPermission(c, false,
 516                                 arvados.UploadDownloadPermission{
 517                                         Upload:   true,
 518                                         Download: userperm,
 519                                 })
 520                         testPermission(c, false,
 521                                 arvados.UploadDownloadPermission{
 522                                         Upload:   true,
 523                                         Download: userperm,
 524                                 })
 525                 }
 526         }
 527 }
 528
 529 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 530         runProxy(c, false, false, nil)
 531         defer closeListener()
 532
 533         {
 534                 client := http.Client{}
 535                 req, err := http.NewRequest("OPTIONS",
 536                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 537                         nil)
 538                 c.Assert(err, IsNil)
 539                 req.Header.Add("Access-Control-Request-Method", "PUT")
 540                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 541                 resp, err := client.Do(req)
 542                 c.Check(err, Equals, nil)
 543                 c.Check(resp.StatusCode, Equals, 200)
 544                 body, err := ioutil.ReadAll(resp.Body)
 545                 c.Check(err, IsNil)
 546                 c.Check(string(body), Equals, "")
 547                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 548                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 549         }
 550
 551         {
 552                 resp, err := http.Get(
 553                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 554                 c.Check(err, Equals, nil)
 555                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 556                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 557         }
 558 }
 559
 560 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 561         runProxy(c, false, false, nil)
 562         defer closeListener()
 563
 564         {
 565                 client := http.Client{}
 566                 req, err := http.NewRequest("POST",
 567                         "http://"+listener.Addr().String()+"/",
 568                         strings.NewReader("qux"))
 569                 c.Check(err, IsNil)
 570                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 571                 req.Header.Add("Content-Type", "application/octet-stream")
 572                 resp, err := client.Do(req)
 573                 c.Check(err, Equals, nil)
 574                 body, err := ioutil.ReadAll(resp.Body)
 575                 c.Check(err, Equals, nil)
 576                 c.Check(string(body), Matches,
 577                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 578         }
 579 }
 580
 581 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 582         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 583                 Equals,
 584                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 585         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 586                 Equals,
 587                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 588         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 589                 Equals,
 590                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 591         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 592                 Equals,
 593                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 594
 595 }
 596
 597 // Test GetIndex
 598 //   Put one block, with 2 replicas
 599 //   With no prefix (expect the block locator, twice)
 600 //   With an existing prefix (expect the block locator, twice)
 601 //   With a valid but non-existing prefix (expect "\n")
 602 //   With an invalid prefix (expect error)
 603 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 604         getIndexWorker(c, false)
 605 }
 606
 607 // Test GetIndex
 608 //   Uses config.yml
 609 //   Put one block, with 2 replicas
 610 //   With no prefix (expect the block locator, twice)
 611 //   With an existing prefix (expect the block locator, twice)
 612 //   With a valid but non-existing prefix (expect "\n")
 613 //   With an invalid prefix (expect error)
 614 func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
 615         getIndexWorker(c, true)
 616 }
 617
 618 func getIndexWorker(c *C, useConfig bool) {
 619         kc, _ := runProxy(c, false, useConfig, nil)
 620         defer closeListener()
 621
 622         // Put "index-data" blocks
 623         data := []byte("index-data")
 624         hash := fmt.Sprintf("%x", md5.Sum(data))
 625
 626         hash2, rep, err := kc.PutB(data)
 627         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 628         c.Check(rep, Equals, 2)
 629         c.Check(err, Equals, nil)
 630
 631         reader, blocklen, _, err := kc.Get(hash)
 632         c.Assert(err, IsNil)
 633         c.Check(blocklen, Equals, int64(10))
 634         all, err := ioutil.ReadAll(reader)
 635         c.Assert(err, IsNil)
 636         c.Check(all, DeepEquals, data)
 637
 638         // Put some more blocks
 639         _, _, err = kc.PutB([]byte("some-more-index-data"))
 640         c.Check(err, IsNil)
 641
 642         kc.Arvados.ApiToken = arvadostest.SystemRootToken
 643
 644         // Invoke GetIndex
 645         for _, spec := range []struct {
 646                 prefix         string
 647                 expectTestHash bool
 648                 expectOther    bool
 649         }{
 650                 {"", true, true},         // with no prefix
 651                 {hash[:3], true, false},  // with matching prefix
 652                 {"abcdef", false, false}, // with no such prefix
 653         } {
 654                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 655                 c.Assert(err, Equals, nil)
 656                 indexResp, err := ioutil.ReadAll(indexReader)
 657                 c.Assert(err, Equals, nil)
 658                 locators := strings.Split(string(indexResp), "\n")
 659                 gotTestHash := 0
 660                 gotOther := 0
 661                 for _, locator := range locators {
 662                         if locator == "" {
 663                                 continue
 664                         }
 665                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 666                         if locator[:32] == hash {
 667                                 gotTestHash++
 668                         } else {
 669                                 gotOther++
 670                         }
 671                 }
 672                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 673                 c.Check(gotOther > 0, Equals, spec.expectOther)
 674         }
 675
 676         // GetIndex with invalid prefix
 677         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 678         c.Assert((err != nil), Equals, true)
 679 }
 680
 681 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 682         kc, _ := runProxy(c, false, false, nil)
 683         defer closeListener()
 684         hash, _, err := kc.PutB([]byte("shareddata"))
 685         c.Check(err, IsNil)
 686         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 687         rdr, _, _, err := kc.Get(hash)
 688         c.Assert(err, IsNil)
 689         data, err := ioutil.ReadAll(rdr)
 690         c.Check(err, IsNil)
 691         c.Check(data, DeepEquals, []byte("shareddata"))
 692 }
 693
 694 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 695         kc, _ := runProxy(c, false, false, nil)
 696         defer closeListener()
 697
 698         // Put a test block
 699         hash, rep, err := kc.PutB([]byte("foo"))
 700         c.Check(err, IsNil)
 701         c.Check(rep, Equals, 2)
 702
 703         for _, badToken := range []string{
 704                 "nosuchtoken",
 705                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 706         } {
 707                 kc.Arvados.ApiToken = badToken
 708
 709                 // Ask and Get will fail only if the upstream
 710                 // keepstore server checks for valid signatures.
 711                 // Without knowing the blob signing key, there is no
 712                 // way for keepproxy to know whether a given token is
 713                 // permitted to read a block.  So these tests fail:
 714                 if false {
 715                         _, _, err = kc.Ask(hash)
 716                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 717                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 718                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 719
 720                         _, _, _, err = kc.Get(hash)
 721                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 722                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 723                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header, or method not allowed\".*")
 724                 }
 725
 726                 _, _, err = kc.PutB([]byte("foo"))
 727                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header, or method not allowed")
 728         }
 729 }
 730
 731 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 732         kc, _ := runProxy(c, false, false, nil)
 733         defer closeListener()
 734
 735         // Point keepproxy at a non-existent keepstore
 736         locals := map[string]string{
 737                 TestProxyUUID: "http://localhost:12345",
 738         }
 739         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
 740
 741         // Ask should result in temporary bad gateway error
 742         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 743         _, _, err := kc.Ask(hash)
 744         c.Check(err, NotNil)
 745         errNotFound, _ := err.(*keepclient.ErrNotFound)
 746         c.Check(errNotFound.Temporary(), Equals, true)
 747         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 748
 749         // Get should result in temporary bad gateway error
 750         _, _, _, err = kc.Get(hash)
 751         c.Check(err, NotNil)
 752         errNotFound, _ = err.(*keepclient.ErrNotFound)
 753         c.Check(errNotFound.Temporary(), Equals, true)
 754         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 755 }
 756
 757 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 758         kc, _ := runProxy(c, false, false, nil)
 759         defer closeListener()
 760
 761         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 762         for _, f := range []func() error{
 763                 func() error {
 764                         _, _, err := kc.Ask(hash)
 765                         return err
 766                 },
 767                 func() error {
 768                         _, _, _, err := kc.Get(hash)
 769                         return err
 770                 },
 771         } {
 772                 err := f()
 773                 c.Assert(err, NotNil)
 774                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 775                 c.Check(errNotFound.Temporary(), Equals, true)
 776                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 777         }
 778 }
 779
 780 func (s *ServerRequiredSuite) TestPing(c *C) {
 781         kc, _ := runProxy(c, false, false, nil)
 782         defer closeListener()
 783
 784         rtr := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{ManagementToken: arvadostest.ManagementToken}, log.New())
 785
 786         req, err := http.NewRequest("GET",
 787                 "http://"+listener.Addr().String()+"/_health/ping",
 788                 nil)
 789         c.Assert(err, IsNil)
 790         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 791
 792         resp := httptest.NewRecorder()
 793         rtr.ServeHTTP(resp, req)
 794         c.Check(resp.Code, Equals, 200)
 795         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 796 }