services/keepproxy/keepproxy_test.go

   1 package main
   2
   3 import (
   4         "crypto/md5"
   5         "crypto/tls"
   6         "fmt"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   8         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   9         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  10         . "gopkg.in/check.v1"
  11         "io"
  12         "io/ioutil"
  13         "log"
  14         "net/http"
  15         "net/url"
  16         "os"
  17         "strings"
  18         "testing"
  19         "time"
  20 )
  21
  22 // Gocheck boilerplate
  23 func Test(t *testing.T) {
  24         TestingT(t)
  25 }
  26
  27 // Gocheck boilerplate
  28 var _ = Suite(&ServerRequiredSuite{})
  29
  30 // Tests that require the Keep server running
  31 type ServerRequiredSuite struct{}
  32
  33 // Wait (up to 1 second) for keepproxy to listen on a port. This
  34 // avoids a race condition where we hit a "connection refused" error
  35 // because we start testing the proxy too soon.
  36 func waitForListener() {
  37         const (
  38                 ms = 5
  39         )
  40         for i := 0; listener == nil && i < 1000; i += ms {
  41                 time.Sleep(ms * time.Millisecond)
  42         }
  43         if listener == nil {
  44                 log.Fatalf("Timed out waiting for listener to start")
  45         }
  46 }
  47
  48 func closeListener() {
  49         if listener != nil {
  50                 listener.Close()
  51         }
  52 }
  53
  54 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  55         arvadostest.StartAPI()
  56         arvadostest.StartKeep(2, false)
  57 }
  58
  59 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  60         arvadostest.ResetEnv()
  61 }
  62
  63 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  64         arvadostest.StopKeep(2)
  65         arvadostest.StopAPI()
  66 }
  67
  68 func setupProxyService() {
  69
  70         client := &http.Client{Transport: &http.Transport{
  71                 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
  72
  73         var req *http.Request
  74         var err error
  75         if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil {
  76                 panic(err.Error())
  77         }
  78         req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN")))
  79
  80         reader, writer := io.Pipe()
  81
  82         req.Body = reader
  83
  84         go func() {
  85                 data := url.Values{}
  86                 data.Set("keep_service", `{
  87   "service_host": "localhost",
  88   "service_port": 29950,
  89   "service_ssl_flag": false,
  90   "service_type": "proxy"
  91 }`)
  92
  93                 writer.Write([]byte(data.Encode()))
  94                 writer.Close()
  95         }()
  96
  97         var resp *http.Response
  98         if resp, err = client.Do(req); err != nil {
  99                 panic(err.Error())
 100         }
 101         if resp.StatusCode != 200 {
 102                 panic(resp.Status)
 103         }
 104 }
 105
 106 func runProxy(c *C, args []string, port int, bogusClientToken bool) keepclient.KeepClient {
 107         if bogusClientToken {
 108                 os.Setenv("ARVADOS_API_TOKEN", "bogus-token")
 109         }
 110         arv, err := arvadosclient.MakeArvadosClient()
 111         c.Assert(err, Equals, nil)
 112         kc := keepclient.KeepClient{
 113                 Arvados:       &arv,
 114                 Want_replicas: 2,
 115                 Using_proxy:   true,
 116                 Client:        &http.Client{},
 117         }
 118         locals := map[string]string{
 119                 "proxy": fmt.Sprintf("http://localhost:%v", port),
 120         }
 121         writableLocals := map[string]string{
 122                 "proxy": fmt.Sprintf("http://localhost:%v", port),
 123         }
 124         kc.SetServiceRoots(locals, writableLocals, nil)
 125         c.Check(kc.Using_proxy, Equals, true)
 126         c.Check(len(kc.LocalRoots()), Equals, 1)
 127         for _, root := range kc.LocalRoots() {
 128                 c.Check(root, Equals, fmt.Sprintf("http://localhost:%v", port))
 129         }
 130         log.Print("keepclient created")
 131         if bogusClientToken {
 132                 arvadostest.ResetEnv()
 133         }
 134
 135         {
 136                 os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
 137                 listener = nil
 138                 go main()
 139         }
 140
 141         return kc
 142 }
 143
 144 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 145         log.Print("TestPutAndGet start")
 146
 147         os.Args = []string{"keepproxy", "-listen=:29950"}
 148         listener = nil
 149         go main()
 150         time.Sleep(100 * time.Millisecond)
 151
 152         setupProxyService()
 153
 154         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
 155         arv, err := arvadosclient.MakeArvadosClient()
 156         c.Assert(err, Equals, nil)
 157         kc, err := keepclient.MakeKeepClient(&arv)
 158         c.Assert(err, Equals, nil)
 159         c.Check(kc.Arvados.External, Equals, true)
 160         c.Check(kc.Using_proxy, Equals, true)
 161         c.Check(len(kc.LocalRoots()), Equals, 1)
 162         for _, root := range kc.LocalRoots() {
 163                 c.Check(root, Equals, "http://localhost:29950")
 164         }
 165         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
 166
 167         waitForListener()
 168         defer closeListener()
 169
 170         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 171         var hash2 string
 172
 173         {
 174                 _, _, err := kc.Ask(hash)
 175                 c.Check(err, Equals, keepclient.BlockNotFound)
 176                 log.Print("Finished Ask (expected BlockNotFound)")
 177         }
 178
 179         {
 180                 reader, _, _, err := kc.Get(hash)
 181                 c.Check(reader, Equals, nil)
 182                 c.Check(err, Equals, keepclient.BlockNotFound)
 183                 log.Print("Finished Get (expected BlockNotFound)")
 184         }
 185
 186         // Note in bug #5309 among other errors keepproxy would set
 187         // Content-Length incorrectly on the 404 BlockNotFound response, this
 188         // would result in a protocol violation that would prevent reuse of the
 189         // connection, which would manifest by the next attempt to use the
 190         // connection (in this case the PutB below) failing.  So to test for
 191         // that bug it's necessary to trigger an error response (such as
 192         // BlockNotFound) and then do something else with the same httpClient
 193         // connection.
 194
 195         {
 196                 var rep int
 197                 var err error
 198                 hash2, rep, err = kc.PutB([]byte("foo"))
 199                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 200                 c.Check(rep, Equals, 2)
 201                 c.Check(err, Equals, nil)
 202                 log.Print("Finished PutB (expected success)")
 203         }
 204
 205         {
 206                 blocklen, _, err := kc.Ask(hash2)
 207                 c.Assert(err, Equals, nil)
 208                 c.Check(blocklen, Equals, int64(3))
 209                 log.Print("Finished Ask (expected success)")
 210         }
 211
 212         {
 213                 reader, blocklen, _, err := kc.Get(hash2)
 214                 c.Assert(err, Equals, nil)
 215                 all, err := ioutil.ReadAll(reader)
 216                 c.Check(all, DeepEquals, []byte("foo"))
 217                 c.Check(blocklen, Equals, int64(3))
 218                 log.Print("Finished Get (expected success)")
 219         }
 220
 221         {
 222                 var rep int
 223                 var err error
 224                 hash2, rep, err = kc.PutB([]byte(""))
 225                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 226                 c.Check(rep, Equals, 2)
 227                 c.Check(err, Equals, nil)
 228                 log.Print("Finished PutB zero block")
 229         }
 230
 231         {
 232                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 233                 c.Assert(err, Equals, nil)
 234                 all, err := ioutil.ReadAll(reader)
 235                 c.Check(all, DeepEquals, []byte(""))
 236                 c.Check(blocklen, Equals, int64(0))
 237                 log.Print("Finished Get zero block")
 238         }
 239
 240         log.Print("TestPutAndGet done")
 241 }
 242
 243 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 244         log.Print("TestPutAskGetForbidden start")
 245
 246         kc := runProxy(c, []string{"keepproxy"}, 29951, true)
 247         waitForListener()
 248         defer closeListener()
 249
 250         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 251
 252         {
 253                 _, _, err := kc.Ask(hash)
 254                 errNotFound, _ := err.(keepclient.ErrNotFound)
 255                 c.Check(errNotFound, NotNil)
 256                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 257                 log.Print("Ask 1")
 258         }
 259
 260         {
 261                 hash2, rep, err := kc.PutB([]byte("bar"))
 262                 c.Check(hash2, Equals, "")
 263                 c.Check(rep, Equals, 0)
 264                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 265                 log.Print("PutB")
 266         }
 267
 268         {
 269                 blocklen, _, err := kc.Ask(hash)
 270                 errNotFound, _ := err.(keepclient.ErrNotFound)
 271                 c.Check(errNotFound, NotNil)
 272                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 273                 c.Check(blocklen, Equals, int64(0))
 274                 log.Print("Ask 2")
 275         }
 276
 277         {
 278                 _, blocklen, _, err := kc.Get(hash)
 279                 errNotFound, _ := err.(keepclient.ErrNotFound)
 280                 c.Check(errNotFound, NotNil)
 281                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 282                 c.Check(blocklen, Equals, int64(0))
 283                 log.Print("Get")
 284         }
 285
 286         log.Print("TestPutAskGetForbidden done")
 287 }
 288
 289 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 290         log.Print("TestGetDisabled start")
 291
 292         kc := runProxy(c, []string{"keepproxy", "-no-get"}, 29952, false)
 293         waitForListener()
 294         defer closeListener()
 295
 296         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 297
 298         {
 299                 _, _, err := kc.Ask(hash)
 300                 errNotFound, _ := err.(keepclient.ErrNotFound)
 301                 c.Check(errNotFound, NotNil)
 302                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 303                 log.Print("Ask 1")
 304         }
 305
 306         {
 307                 hash2, rep, err := kc.PutB([]byte("baz"))
 308                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 309                 c.Check(rep, Equals, 2)
 310                 c.Check(err, Equals, nil)
 311                 log.Print("PutB")
 312         }
 313
 314         {
 315                 blocklen, _, err := kc.Ask(hash)
 316                 errNotFound, _ := err.(keepclient.ErrNotFound)
 317                 c.Check(errNotFound, NotNil)
 318                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 319                 c.Check(blocklen, Equals, int64(0))
 320                 log.Print("Ask 2")
 321         }
 322
 323         {
 324                 _, blocklen, _, err := kc.Get(hash)
 325                 errNotFound, _ := err.(keepclient.ErrNotFound)
 326                 c.Check(errNotFound, NotNil)
 327                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 328                 c.Check(blocklen, Equals, int64(0))
 329                 log.Print("Get")
 330         }
 331
 332         log.Print("TestGetDisabled done")
 333 }
 334
 335 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 336         log.Print("TestPutDisabled start")
 337
 338         kc := runProxy(c, []string{"keepproxy", "-no-put"}, 29953, false)
 339         waitForListener()
 340         defer closeListener()
 341
 342         {
 343                 hash2, rep, err := kc.PutB([]byte("quux"))
 344                 c.Check(hash2, Equals, "")
 345                 c.Check(rep, Equals, 0)
 346                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 347                 log.Print("PutB")
 348         }
 349
 350         log.Print("TestPutDisabled done")
 351 }
 352
 353 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 354         runProxy(c, []string{"keepproxy"}, 29954, false)
 355         waitForListener()
 356         defer closeListener()
 357
 358         {
 359                 client := http.Client{}
 360                 req, err := http.NewRequest("OPTIONS",
 361                         fmt.Sprintf("http://localhost:29954/%x+3",
 362                                 md5.Sum([]byte("foo"))),
 363                         nil)
 364                 req.Header.Add("Access-Control-Request-Method", "PUT")
 365                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 366                 resp, err := client.Do(req)
 367                 c.Check(err, Equals, nil)
 368                 c.Check(resp.StatusCode, Equals, 200)
 369                 body, err := ioutil.ReadAll(resp.Body)
 370                 c.Check(string(body), Equals, "")
 371                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 372                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 373         }
 374
 375         {
 376                 resp, err := http.Get(
 377                         fmt.Sprintf("http://localhost:29954/%x+3",
 378                                 md5.Sum([]byte("foo"))))
 379                 c.Check(err, Equals, nil)
 380                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 381                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 382         }
 383 }
 384
 385 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 386         runProxy(c, []string{"keepproxy"}, 29955, false)
 387         waitForListener()
 388         defer closeListener()
 389
 390         {
 391                 client := http.Client{}
 392                 req, err := http.NewRequest("POST",
 393                         "http://localhost:29955/",
 394                         strings.NewReader("qux"))
 395                 req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 396                 req.Header.Add("Content-Type", "application/octet-stream")
 397                 resp, err := client.Do(req)
 398                 c.Check(err, Equals, nil)
 399                 body, err := ioutil.ReadAll(resp.Body)
 400                 c.Check(err, Equals, nil)
 401                 c.Check(string(body), Matches,
 402                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 403         }
 404 }
 405
 406 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 407         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 408                 Equals,
 409                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 410         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 411                 Equals,
 412                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 413         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 414                 Equals,
 415                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 416         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 417                 Equals,
 418                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 419
 420 }
 421
 422 // Test GetIndex
 423 //   Put one block, with 2 replicas
 424 //   With no prefix (expect the block locator, twice)
 425 //   With an existing prefix (expect the block locator, twice)
 426 //   With a valid but non-existing prefix (expect "\n")
 427 //   With an invalid prefix (expect error)
 428 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 429         kc := runProxy(c, []string{"keepproxy"}, 28852, false)
 430         waitForListener()
 431         defer closeListener()
 432
 433         // Put "index-data" blocks
 434         data := []byte("index-data")
 435         hash := fmt.Sprintf("%x", md5.Sum(data))
 436
 437         hash2, rep, err := kc.PutB(data)
 438         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 439         c.Check(rep, Equals, 2)
 440         c.Check(err, Equals, nil)
 441
 442         reader, blocklen, _, err := kc.Get(hash)
 443         c.Assert(err, Equals, nil)
 444         c.Check(blocklen, Equals, int64(10))
 445         all, err := ioutil.ReadAll(reader)
 446         c.Check(all, DeepEquals, data)
 447
 448         // Put some more blocks
 449         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 450         c.Check(err, Equals, nil)
 451
 452         // Invoke GetIndex
 453         for _, spec := range []struct {
 454                 prefix         string
 455                 expectTestHash bool
 456                 expectOther    bool
 457         }{
 458                 {"", true, true},         // with no prefix
 459                 {hash[:3], true, false},  // with matching prefix
 460                 {"abcdef", false, false}, // with no such prefix
 461         } {
 462                 indexReader, err := kc.GetIndex("proxy", spec.prefix)
 463                 c.Assert(err, Equals, nil)
 464                 indexResp, err := ioutil.ReadAll(indexReader)
 465                 c.Assert(err, Equals, nil)
 466                 locators := strings.Split(string(indexResp), "\n")
 467                 gotTestHash := 0
 468                 gotOther := 0
 469                 for _, locator := range locators {
 470                         if locator == "" {
 471                                 continue
 472                         }
 473                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 474                         if locator[:32] == hash {
 475                                 gotTestHash++
 476                         } else {
 477                                 gotOther++
 478                         }
 479                 }
 480                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 481                 c.Check(gotOther > 0, Equals, spec.expectOther)
 482         }
 483
 484         // GetIndex with invalid prefix
 485         _, err = kc.GetIndex("proxy", "xyz")
 486         c.Assert((err != nil), Equals, true)
 487 }
 488
 489 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 490         kc := runProxy(c, []string{"keepproxy"}, 28852, false)
 491         waitForListener()
 492         defer closeListener()
 493
 494         // Put a test block
 495         hash, rep, err := kc.PutB([]byte("foo"))
 496         c.Check(err, Equals, nil)
 497         c.Check(rep, Equals, 2)
 498
 499         for _, token := range []string{
 500                 "nosuchtoken",
 501                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 502         } {
 503                 // Change token to given bad token
 504                 kc.Arvados.ApiToken = token
 505
 506                 // Ask should result in error
 507                 _, _, err = kc.Ask(hash)
 508                 c.Check(err, NotNil)
 509                 errNotFound, _ := err.(keepclient.ErrNotFound)
 510                 c.Check(errNotFound.Temporary(), Equals, false)
 511                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 512
 513                 // Get should result in error
 514                 _, _, _, err = kc.Get(hash)
 515                 c.Check(err, NotNil)
 516                 errNotFound, _ = err.(keepclient.ErrNotFound)
 517                 c.Check(errNotFound.Temporary(), Equals, false)
 518                 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
 519         }
 520 }
 521
 522 func (s *ServerRequiredSuite) TestPutAskGetConnectionError(c *C) {
 523         arv, err := arvadosclient.MakeArvadosClient()
 524         c.Assert(err, Equals, nil)
 525
 526         // keepclient with no such keep server
 527         kc := keepclient.New(&arv)
 528         locals := map[string]string{
 529                 "proxy": "http://localhost:12345",
 530         }
 531         kc.SetServiceRoots(locals, nil, nil)
 532
 533         // Ask should result in temporary connection refused error
 534         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 535         _, _, err = kc.Ask(hash)
 536         c.Check(err, NotNil)
 537         errNotFound, _ := err.(*keepclient.ErrNotFound)
 538         c.Check(errNotFound.Temporary(), Equals, true)
 539         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 540
 541         // Get should result in temporary connection refused error
 542         _, _, _, err = kc.Get(hash)
 543         c.Check(err, NotNil)
 544         errNotFound, _ = err.(*keepclient.ErrNotFound)
 545         c.Check(errNotFound.Temporary(), Equals, true)
 546         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 547 }