services/keepproxy/keepproxy_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "fmt"
  11         "io/ioutil"
  12         "math/rand"
  13         "net/http"
  14         "net/http/httptest"
  15         "strings"
  16         "sync"
  17         "testing"
  18         "time"
  19
  20         "git.arvados.org/arvados.git/lib/config"
  21         "git.arvados.org/arvados.git/sdk/go/arvados"
  22         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  23         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  24         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  25         "git.arvados.org/arvados.git/sdk/go/keepclient"
  26         log "github.com/sirupsen/logrus"
  27
  28         "gopkg.in/check.v1"
  29         . "gopkg.in/check.v1"
  30 )
  31
  32 // Gocheck boilerplate
  33 func Test(t *testing.T) {
  34         TestingT(t)
  35 }
  36
  37 // Gocheck boilerplate
  38 var _ = Suite(&ServerRequiredSuite{})
  39
  40 // Tests that require the Keep server running
  41 type ServerRequiredSuite struct{}
  42
  43 // Gocheck boilerplate
  44 var _ = Suite(&ServerRequiredConfigYmlSuite{})
  45
  46 // Tests that require the Keep servers running as defined in config.yml
  47 type ServerRequiredConfigYmlSuite struct{}
  48
  49 // Gocheck boilerplate
  50 var _ = Suite(&NoKeepServerSuite{})
  51
  52 // Test with no keepserver to simulate errors
  53 type NoKeepServerSuite struct{}
  54
  55 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  56
  57 // Wait (up to 1 second) for keepproxy to listen on a port. This
  58 // avoids a race condition where we hit a "connection refused" error
  59 // because we start testing the proxy too soon.
  60 func waitForListener() {
  61         const (
  62                 ms = 5
  63         )
  64         for i := 0; listener == nil && i < 10000; i += ms {
  65                 time.Sleep(ms * time.Millisecond)
  66         }
  67         if listener == nil {
  68                 panic("Timed out waiting for listener to start")
  69         }
  70 }
  71
  72 func closeListener() {
  73         if listener != nil {
  74                 listener.Close()
  75         }
  76 }
  77
  78 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  79         arvadostest.StartKeep(2, false)
  80 }
  81
  82 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  83         arvadostest.ResetEnv()
  84 }
  85
  86 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  87         arvadostest.StopKeep(2)
  88 }
  89
  90 func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
  91         // config.yml defines 4 keepstores
  92         arvadostest.StartKeep(4, false)
  93 }
  94
  95 func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
  96         arvadostest.ResetEnv()
  97 }
  98
  99 func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
 100         arvadostest.StopKeep(4)
 101 }
 102
 103 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
 104         // We need API to have some keep services listed, but the
 105         // services themselves should be unresponsive.
 106         arvadostest.StartKeep(2, false)
 107         arvadostest.StopKeep(2)
 108 }
 109
 110 func (s *NoKeepServerSuite) SetUpTest(c *C) {
 111         arvadostest.ResetEnv()
 112 }
 113
 114 func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool, kp *arvados.UploadDownloadRolePermissions) (*keepclient.KeepClient, *bytes.Buffer) {
 115         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
 116         c.Assert(err, Equals, nil)
 117         cluster, err := cfg.GetCluster("")
 118         c.Assert(err, Equals, nil)
 119
 120         if !loadKeepstoresFromConfig {
 121                 // Do not load Keepstore InternalURLs from the config file
 122                 cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
 123         }
 124
 125         cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: ":0"}: {}}
 126
 127         if kp != nil {
 128                 cluster.Collections.KeepproxyPermission = *kp
 129         }
 130
 131         listener = nil
 132         logbuf := &bytes.Buffer{}
 133         logger := log.New()
 134         logger.Out = logbuf
 135         go func() {
 136                 run(logger, cluster)
 137                 defer closeListener()
 138         }()
 139         waitForListener()
 140
 141         client := arvados.NewClientFromEnv()
 142         arv, err := arvadosclient.New(client)
 143         c.Assert(err, Equals, nil)
 144         if bogusClientToken {
 145                 arv.ApiToken = "bogus-token"
 146         }
 147         kc := keepclient.New(arv)
 148         sr := map[string]string{
 149                 TestProxyUUID: "http://" + listener.Addr().String(),
 150         }
 151         kc.SetServiceRoots(sr, sr, sr)
 152         kc.Arvados.External = true
 153
 154         return kc, logbuf
 155 }
 156
 157 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 158         runProxy(c, false, false, nil)
 159         defer closeListener()
 160
 161         req, err := http.NewRequest("POST",
 162                 "http://"+listener.Addr().String()+"/",
 163                 strings.NewReader("TestViaHeader"))
 164         c.Assert(err, Equals, nil)
 165         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 166         resp, err := (&http.Client{}).Do(req)
 167         c.Assert(err, Equals, nil)
 168         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 169         c.Assert(resp.StatusCode, Equals, http.StatusOK)
 170         locator, err := ioutil.ReadAll(resp.Body)
 171         c.Assert(err, Equals, nil)
 172         resp.Body.Close()
 173
 174         req, err = http.NewRequest("GET",
 175                 "http://"+listener.Addr().String()+"/"+string(locator),
 176                 nil)
 177         c.Assert(err, Equals, nil)
 178         resp, err = (&http.Client{}).Do(req)
 179         c.Assert(err, Equals, nil)
 180         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
 181         resp.Body.Close()
 182 }
 183
 184 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 185         kc, _ := runProxy(c, false, false, nil)
 186         defer closeListener()
 187
 188         sr := map[string]string{
 189                 TestProxyUUID: "http://" + listener.Addr().String(),
 190         }
 191         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 192
 193         content := []byte("TestLoopDetection")
 194         _, _, err := kc.PutB(content)
 195         c.Check(err, ErrorMatches, `.*loop detected.*`)
 196
 197         hash := fmt.Sprintf("%x", md5.Sum(content))
 198         _, _, _, err = kc.Get(hash)
 199         c.Check(err, ErrorMatches, `.*loop detected.*`)
 200 }
 201
 202 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 203         kc, _ := runProxy(c, false, false, nil)
 204         defer closeListener()
 205
 206         // Set up fake keepstore to record request headers
 207         var hdr http.Header
 208         ts := httptest.NewServer(http.HandlerFunc(
 209                 func(w http.ResponseWriter, r *http.Request) {
 210                         hdr = r.Header
 211                         http.Error(w, "Error", http.StatusInternalServerError)
 212                 }))
 213         defer ts.Close()
 214
 215         // Point keepproxy router's keepclient to the fake keepstore
 216         sr := map[string]string{
 217                 TestProxyUUID: ts.URL,
 218         }
 219         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
 220
 221         // Set up client to ask for storage classes to keepproxy
 222         kc.StorageClasses = []string{"secure"}
 223         content := []byte("Very important data")
 224         _, _, err := kc.PutB(content)
 225         c.Check(err, NotNil)
 226         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 227 }
 228
 229 func (s *ServerRequiredSuite) TestStorageClassesConfirmedHeader(c *C) {
 230         runProxy(c, false, false, nil)
 231         defer closeListener()
 232
 233         content := []byte("foo")
 234         hash := fmt.Sprintf("%x", md5.Sum(content))
 235         client := &http.Client{}
 236
 237         req, err := http.NewRequest("PUT",
 238                 fmt.Sprintf("http://%s/%s", listener.Addr().String(), hash),
 239                 bytes.NewReader(content))
 240         c.Assert(err, IsNil)
 241         req.Header.Set("X-Keep-Storage-Classes", "default")
 242         req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 243         req.Header.Set("Content-Type", "application/octet-stream")
 244
 245         resp, err := client.Do(req)
 246         c.Assert(err, IsNil)
 247         c.Assert(resp.StatusCode, Equals, http.StatusOK)
 248         c.Assert(resp.Header.Get("X-Keep-Storage-Classes-Confirmed"), Equals, "default=2")
 249 }
 250
 251 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 252         kc, _ := runProxy(c, false, false, nil)
 253         defer closeListener()
 254
 255         content := []byte("TestDesiredReplicas")
 256         hash := fmt.Sprintf("%x", md5.Sum(content))
 257
 258         for _, kc.Want_replicas = range []int{0, 1, 2, 3} {
 259                 locator, rep, err := kc.PutB(content)
 260                 if kc.Want_replicas < 3 {
 261                         c.Check(err, Equals, nil)
 262                         c.Check(rep, Equals, kc.Want_replicas)
 263                         if rep > 0 {
 264                                 c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
 265                         }
 266                 } else {
 267                         c.Check(err, ErrorMatches, ".*503.*")
 268                 }
 269         }
 270 }
 271
 272 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 273         kc, _ := runProxy(c, false, false, nil)
 274         defer closeListener()
 275
 276         content := []byte("TestPutWrongContentLength")
 277         hash := fmt.Sprintf("%x", md5.Sum(content))
 278
 279         // If we use http.Client to send these requests to the network
 280         // server we just started, the Go http library automatically
 281         // fixes the invalid Content-Length header. In order to test
 282         // our server behavior, we have to call the handler directly
 283         // using an httptest.ResponseRecorder.
 284         rtr, err := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{}, log.New())
 285         c.Assert(err, check.IsNil)
 286
 287         type testcase struct {
 288                 sendLength   string
 289                 expectStatus int
 290         }
 291
 292         for _, t := range []testcase{
 293                 {"1", http.StatusBadRequest},
 294                 {"", http.StatusLengthRequired},
 295                 {"-1", http.StatusLengthRequired},
 296                 {"abcdef", http.StatusLengthRequired},
 297         } {
 298                 req, err := http.NewRequest("PUT",
 299                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
 300                         bytes.NewReader(content))
 301                 c.Assert(err, IsNil)
 302                 req.Header.Set("Content-Length", t.sendLength)
 303                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 304                 req.Header.Set("Content-Type", "application/octet-stream")
 305
 306                 resp := httptest.NewRecorder()
 307                 rtr.ServeHTTP(resp, req)
 308                 c.Check(resp.Code, Equals, t.expectStatus)
 309         }
 310 }
 311
 312 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 313         kc, _ := runProxy(c, false, false, nil)
 314         defer closeListener()
 315         router.(*proxyHandler).timeout = time.Nanosecond
 316
 317         buf := make([]byte, 1<<20)
 318         rand.Read(buf)
 319         var wg sync.WaitGroup
 320         for i := 0; i < 128; i++ {
 321                 wg.Add(1)
 322                 go func() {
 323                         defer wg.Done()
 324                         kc.PutB(buf)
 325                 }()
 326         }
 327         done := make(chan bool)
 328         go func() {
 329                 wg.Wait()
 330                 close(done)
 331         }()
 332         select {
 333         case <-done:
 334         case <-time.After(10 * time.Second):
 335                 c.Error("timeout")
 336         }
 337 }
 338
 339 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 340         kc, logbuf := runProxy(c, false, false, nil)
 341         defer closeListener()
 342
 343         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 344         var hash2 string
 345
 346         {
 347                 _, _, err := kc.Ask(hash)
 348                 c.Check(err, Equals, keepclient.BlockNotFound)
 349                 c.Log("Finished Ask (expected BlockNotFound)")
 350         }
 351
 352         {
 353                 reader, _, _, err := kc.Get(hash)
 354                 c.Check(reader, Equals, nil)
 355                 c.Check(err, Equals, keepclient.BlockNotFound)
 356                 c.Log("Finished Get (expected BlockNotFound)")
 357         }
 358
 359         // Note in bug #5309 among other errors keepproxy would set
 360         // Content-Length incorrectly on the 404 BlockNotFound response, this
 361         // would result in a protocol violation that would prevent reuse of the
 362         // connection, which would manifest by the next attempt to use the
 363         // connection (in this case the PutB below) failing.  So to test for
 364         // that bug it's necessary to trigger an error response (such as
 365         // BlockNotFound) and then do something else with the same httpClient
 366         // connection.
 367
 368         {
 369                 var rep int
 370                 var err error
 371                 hash2, rep, err = kc.PutB([]byte("foo"))
 372                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 373                 c.Check(rep, Equals, 2)
 374                 c.Check(err, Equals, nil)
 375                 c.Log("Finished PutB (expected success)")
 376
 377                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 378                 logbuf.Reset()
 379         }
 380
 381         {
 382                 blocklen, _, err := kc.Ask(hash2)
 383                 c.Assert(err, Equals, nil)
 384                 c.Check(blocklen, Equals, int64(3))
 385                 c.Log("Finished Ask (expected success)")
 386                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 387                 logbuf.Reset()
 388         }
 389
 390         {
 391                 reader, blocklen, _, err := kc.Get(hash2)
 392                 c.Assert(err, Equals, nil)
 393                 all, err := ioutil.ReadAll(reader)
 394                 c.Check(err, IsNil)
 395                 c.Check(all, DeepEquals, []byte("foo"))
 396                 c.Check(blocklen, Equals, int64(3))
 397                 c.Log("Finished Get (expected success)")
 398                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 399                 logbuf.Reset()
 400         }
 401
 402         {
 403                 var rep int
 404                 var err error
 405                 hash2, rep, err = kc.PutB([]byte(""))
 406                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 407                 c.Check(rep, Equals, 2)
 408                 c.Check(err, Equals, nil)
 409                 c.Log("Finished PutB zero block")
 410         }
 411
 412         {
 413                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 414                 c.Assert(err, Equals, nil)
 415                 all, err := ioutil.ReadAll(reader)
 416                 c.Check(err, IsNil)
 417                 c.Check(all, DeepEquals, []byte(""))
 418                 c.Check(blocklen, Equals, int64(0))
 419                 c.Log("Finished Get zero block")
 420         }
 421 }
 422
 423 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 424         kc, _ := runProxy(c, true, false, nil)
 425         defer closeListener()
 426
 427         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 428
 429         _, _, err := kc.Ask(hash)
 430         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 431
 432         hash2, rep, err := kc.PutB([]byte("bar"))
 433         c.Check(hash2, Equals, "")
 434         c.Check(rep, Equals, 0)
 435         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError{})
 436
 437         blocklen, _, err := kc.Ask(hash)
 438         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 439         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 440         c.Check(blocklen, Equals, int64(0))
 441
 442         _, blocklen, _, err = kc.Get(hash)
 443         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 444         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 445         c.Check(blocklen, Equals, int64(0))
 446 }
 447
 448 func testPermission(c *C, admin bool, perm arvados.UploadDownloadPermission) {
 449         kp := arvados.UploadDownloadRolePermissions{}
 450         if admin {
 451                 kp.Admin = perm
 452                 kp.User = arvados.UploadDownloadPermission{Upload: true, Download: true}
 453         } else {
 454                 kp.Admin = arvados.UploadDownloadPermission{Upload: true, Download: true}
 455                 kp.User = perm
 456         }
 457
 458         kc, logbuf := runProxy(c, false, false, &kp)
 459         defer closeListener()
 460         if admin {
 461                 kc.Arvados.ApiToken = arvadostest.AdminToken
 462         } else {
 463                 kc.Arvados.ApiToken = arvadostest.ActiveToken
 464         }
 465
 466         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 467         var hash2 string
 468
 469         {
 470                 var rep int
 471                 var err error
 472                 hash2, rep, err = kc.PutB([]byte("foo"))
 473
 474                 if perm.Upload {
 475                         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 476                         c.Check(rep, Equals, 2)
 477                         c.Check(err, Equals, nil)
 478                         c.Log("Finished PutB (expected success)")
 479                         if admin {
 480                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 481                         } else {
 482
 483                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
 484                         }
 485                 } else {
 486                         c.Check(hash2, Equals, "")
 487                         c.Check(rep, Equals, 0)
 488                         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError{})
 489                 }
 490                 logbuf.Reset()
 491         }
 492         if perm.Upload {
 493                 // can't test download without upload.
 494
 495                 reader, blocklen, _, err := kc.Get(hash2)
 496                 if perm.Download {
 497                         c.Assert(err, Equals, nil)
 498                         all, err := ioutil.ReadAll(reader)
 499                         c.Check(err, IsNil)
 500                         c.Check(all, DeepEquals, []byte("foo"))
 501                         c.Check(blocklen, Equals, int64(3))
 502                         c.Log("Finished Get (expected success)")
 503                         if admin {
 504                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
 505                         } else {
 506                                 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
 507                         }
 508                 } else {
 509                         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 510                         c.Check(err, ErrorMatches, ".*Missing or invalid Authorization header, or method not allowed.*")
 511                         c.Check(blocklen, Equals, int64(0))
 512                 }
 513                 logbuf.Reset()
 514         }
 515
 516 }
 517
 518 func (s *ServerRequiredSuite) TestPutGetPermission(c *C) {
 519
 520         for _, adminperm := range []bool{true, false} {
 521                 for _, userperm := range []bool{true, false} {
 522
 523                         testPermission(c, true,
 524                                 arvados.UploadDownloadPermission{
 525                                         Upload:   adminperm,
 526                                         Download: true,
 527                                 })
 528                         testPermission(c, true,
 529                                 arvados.UploadDownloadPermission{
 530                                         Upload:   true,
 531                                         Download: adminperm,
 532                                 })
 533                         testPermission(c, false,
 534                                 arvados.UploadDownloadPermission{
 535                                         Upload:   true,
 536                                         Download: userperm,
 537                                 })
 538                         testPermission(c, false,
 539                                 arvados.UploadDownloadPermission{
 540                                         Upload:   true,
 541                                         Download: userperm,
 542                                 })
 543                 }
 544         }
 545 }
 546
 547 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 548         runProxy(c, false, false, nil)
 549         defer closeListener()
 550
 551         {
 552                 client := http.Client{}
 553                 req, err := http.NewRequest("OPTIONS",
 554                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 555                         nil)
 556                 c.Assert(err, IsNil)
 557                 req.Header.Add("Access-Control-Request-Method", "PUT")
 558                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 559                 resp, err := client.Do(req)
 560                 c.Check(err, Equals, nil)
 561                 c.Check(resp.StatusCode, Equals, 200)
 562                 body, err := ioutil.ReadAll(resp.Body)
 563                 c.Check(err, IsNil)
 564                 c.Check(string(body), Equals, "")
 565                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 566                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 567         }
 568
 569         {
 570                 resp, err := http.Get(
 571                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 572                 c.Check(err, Equals, nil)
 573                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 574                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 575         }
 576 }
 577
 578 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 579         runProxy(c, false, false, nil)
 580         defer closeListener()
 581
 582         {
 583                 client := http.Client{}
 584                 req, err := http.NewRequest("POST",
 585                         "http://"+listener.Addr().String()+"/",
 586                         strings.NewReader("qux"))
 587                 c.Check(err, IsNil)
 588                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 589                 req.Header.Add("Content-Type", "application/octet-stream")
 590                 resp, err := client.Do(req)
 591                 c.Check(err, Equals, nil)
 592                 body, err := ioutil.ReadAll(resp.Body)
 593                 c.Check(err, Equals, nil)
 594                 c.Check(string(body), Matches,
 595                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 596         }
 597 }
 598
 599 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 600         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 601                 Equals,
 602                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 603         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 604                 Equals,
 605                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 606         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 607                 Equals,
 608                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 609         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 610                 Equals,
 611                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 612
 613 }
 614
 615 // Test GetIndex
 616 //   Put one block, with 2 replicas
 617 //   With no prefix (expect the block locator, twice)
 618 //   With an existing prefix (expect the block locator, twice)
 619 //   With a valid but non-existing prefix (expect "\n")
 620 //   With an invalid prefix (expect error)
 621 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 622         getIndexWorker(c, false)
 623 }
 624
 625 // Test GetIndex
 626 //   Uses config.yml
 627 //   Put one block, with 2 replicas
 628 //   With no prefix (expect the block locator, twice)
 629 //   With an existing prefix (expect the block locator, twice)
 630 //   With a valid but non-existing prefix (expect "\n")
 631 //   With an invalid prefix (expect error)
 632 func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
 633         getIndexWorker(c, true)
 634 }
 635
 636 func getIndexWorker(c *C, useConfig bool) {
 637         kc, _ := runProxy(c, false, useConfig, nil)
 638         defer closeListener()
 639
 640         // Put "index-data" blocks
 641         data := []byte("index-data")
 642         hash := fmt.Sprintf("%x", md5.Sum(data))
 643
 644         hash2, rep, err := kc.PutB(data)
 645         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 646         c.Check(rep, Equals, 2)
 647         c.Check(err, Equals, nil)
 648
 649         reader, blocklen, _, err := kc.Get(hash)
 650         c.Assert(err, IsNil)
 651         c.Check(blocklen, Equals, int64(10))
 652         all, err := ioutil.ReadAll(reader)
 653         c.Assert(err, IsNil)
 654         c.Check(all, DeepEquals, data)
 655
 656         // Put some more blocks
 657         _, _, err = kc.PutB([]byte("some-more-index-data"))
 658         c.Check(err, IsNil)
 659
 660         kc.Arvados.ApiToken = arvadostest.SystemRootToken
 661
 662         // Invoke GetIndex
 663         for _, spec := range []struct {
 664                 prefix         string
 665                 expectTestHash bool
 666                 expectOther    bool
 667         }{
 668                 {"", true, true},         // with no prefix
 669                 {hash[:3], true, false},  // with matching prefix
 670                 {"abcdef", false, false}, // with no such prefix
 671         } {
 672                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 673                 c.Assert(err, Equals, nil)
 674                 indexResp, err := ioutil.ReadAll(indexReader)
 675                 c.Assert(err, Equals, nil)
 676                 locators := strings.Split(string(indexResp), "\n")
 677                 gotTestHash := 0
 678                 gotOther := 0
 679                 for _, locator := range locators {
 680                         if locator == "" {
 681                                 continue
 682                         }
 683                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 684                         if locator[:32] == hash {
 685                                 gotTestHash++
 686                         } else {
 687                                 gotOther++
 688                         }
 689                 }
 690                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 691                 c.Check(gotOther > 0, Equals, spec.expectOther)
 692         }
 693
 694         // GetIndex with invalid prefix
 695         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 696         c.Assert((err != nil), Equals, true)
 697 }
 698
 699 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 700         kc, _ := runProxy(c, false, false, nil)
 701         defer closeListener()
 702         hash, _, err := kc.PutB([]byte("shareddata"))
 703         c.Check(err, IsNil)
 704         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
 705         rdr, _, _, err := kc.Get(hash)
 706         c.Assert(err, IsNil)
 707         data, err := ioutil.ReadAll(rdr)
 708         c.Check(err, IsNil)
 709         c.Check(data, DeepEquals, []byte("shareddata"))
 710 }
 711
 712 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 713         kc, _ := runProxy(c, false, false, nil)
 714         defer closeListener()
 715
 716         // Put a test block
 717         hash, rep, err := kc.PutB([]byte("foo"))
 718         c.Check(err, IsNil)
 719         c.Check(rep, Equals, 2)
 720
 721         for _, badToken := range []string{
 722                 "nosuchtoken",
 723                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 724         } {
 725                 kc.Arvados.ApiToken = badToken
 726
 727                 // Ask and Get will fail only if the upstream
 728                 // keepstore server checks for valid signatures.
 729                 // Without knowing the blob signing key, there is no
 730                 // way for keepproxy to know whether a given token is
 731                 // permitted to read a block.  So these tests fail:
 732                 if false {
 733                         _, _, err = kc.Ask(hash)
 734                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 735                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 736                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
 737
 738                         _, _, _, err = kc.Get(hash)
 739                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
 740                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
 741                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header, or method not allowed\".*")
 742                 }
 743
 744                 _, _, err = kc.PutB([]byte("foo"))
 745                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header, or method not allowed")
 746         }
 747 }
 748
 749 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 750         kc, _ := runProxy(c, false, false, nil)
 751         defer closeListener()
 752
 753         // Point keepproxy at a non-existent keepstore
 754         locals := map[string]string{
 755                 TestProxyUUID: "http://localhost:12345",
 756         }
 757         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
 758
 759         // Ask should result in temporary bad gateway error
 760         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 761         _, _, err := kc.Ask(hash)
 762         c.Check(err, NotNil)
 763         errNotFound, _ := err.(*keepclient.ErrNotFound)
 764         c.Check(errNotFound.Temporary(), Equals, true)
 765         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 766
 767         // Get should result in temporary bad gateway error
 768         _, _, _, err = kc.Get(hash)
 769         c.Check(err, NotNil)
 770         errNotFound, _ = err.(*keepclient.ErrNotFound)
 771         c.Check(errNotFound.Temporary(), Equals, true)
 772         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
 773 }
 774
 775 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 776         kc, _ := runProxy(c, false, false, nil)
 777         defer closeListener()
 778
 779         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 780         for _, f := range []func() error{
 781                 func() error {
 782                         _, _, err := kc.Ask(hash)
 783                         return err
 784                 },
 785                 func() error {
 786                         _, _, _, err := kc.Get(hash)
 787                         return err
 788                 },
 789         } {
 790                 err := f()
 791                 c.Assert(err, NotNil)
 792                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 793                 c.Check(errNotFound.Temporary(), Equals, true)
 794                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 795         }
 796 }
 797
 798 func (s *ServerRequiredSuite) TestPing(c *C) {
 799         kc, _ := runProxy(c, false, false, nil)
 800         defer closeListener()
 801
 802         rtr, err := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{ManagementToken: arvadostest.ManagementToken}, log.New())
 803         c.Assert(err, check.IsNil)
 804
 805         req, err := http.NewRequest("GET",
 806                 "http://"+listener.Addr().String()+"/_health/ping",
 807                 nil)
 808         c.Assert(err, IsNil)
 809         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 810
 811         resp := httptest.NewRecorder()
 812         rtr.ServeHTTP(resp, req)
 813         c.Check(resp.Code, Equals, 200)
 814         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 815 }