services/keepproxy/keepproxy_test.go

   1 package main
   2
   3 import (
   4         "crypto/md5"
   5         "fmt"
   6         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   8         "git.curoverse.com/arvados.git/sdk/go/keepclient"
   9         "io/ioutil"
  10         "log"
  11         "net/http"
  12         "os"
  13         "strings"
  14         "testing"
  15         "time"
  16
  17         . "gopkg.in/check.v1"
  18 )
  19
  20 // Gocheck boilerplate
  21 func Test(t *testing.T) {
  22         TestingT(t)
  23 }
  24
  25 // Gocheck boilerplate
  26 var _ = Suite(&ServerRequiredSuite{})
  27
  28 // Tests that require the Keep server running
  29 type ServerRequiredSuite struct{}
  30
  31 // Gocheck boilerplate
  32 var _ = Suite(&NoKeepServerSuite{})
  33
  34 // Test with no keepserver to simulate errors
  35 type NoKeepServerSuite struct{}
  36
  37 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  38
  39 // Wait (up to 1 second) for keepproxy to listen on a port. This
  40 // avoids a race condition where we hit a "connection refused" error
  41 // because we start testing the proxy too soon.
  42 func waitForListener() {
  43         const (
  44                 ms = 5
  45         )
  46         for i := 0; listener == nil && i < 1000; i += ms {
  47                 time.Sleep(ms * time.Millisecond)
  48         }
  49         if listener == nil {
  50                 log.Fatalf("Timed out waiting for listener to start")
  51         }
  52 }
  53
  54 func closeListener() {
  55         if listener != nil {
  56                 listener.Close()
  57         }
  58 }
  59
  60 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  61         arvadostest.StartAPI()
  62         arvadostest.StartKeep(2, false)
  63 }
  64
  65 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  66         arvadostest.ResetEnv()
  67 }
  68
  69 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  70         arvadostest.StopKeep(2)
  71         arvadostest.StopAPI()
  72 }
  73
  74 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  75         arvadostest.StartAPI()
  76         // We need API to have some keep services listed, but the
  77         // services themselves should be unresponsive.
  78         arvadostest.StartKeep(2, false)
  79         arvadostest.StopKeep(2)
  80 }
  81
  82 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  83         arvadostest.ResetEnv()
  84 }
  85
  86 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  87         arvadostest.StopAPI()
  88 }
  89
  90 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
  91         args = append([]string{"keepproxy"}, args...)
  92         os.Args = append(args, "-listen=:0")
  93         listener = nil
  94         go main()
  95         waitForListener()
  96
  97         arv, err := arvadosclient.MakeArvadosClient()
  98         c.Assert(err, Equals, nil)
  99         if bogusClientToken {
 100                 arv.ApiToken = "bogus-token"
 101         }
 102         kc := keepclient.New(&arv)
 103         sr := map[string]string{
 104                 TestProxyUUID: "http://" + listener.Addr().String(),
 105         }
 106         kc.SetServiceRoots(sr, sr, sr)
 107         kc.Arvados.External = true
 108         kc.Using_proxy = true
 109
 110         return kc
 111 }
 112
 113 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 114         kc := runProxy(c, nil, false)
 115         defer closeListener()
 116
 117         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 118         var hash2 string
 119
 120         {
 121                 _, _, err := kc.Ask(hash)
 122                 c.Check(err, Equals, keepclient.BlockNotFound)
 123                 log.Print("Finished Ask (expected BlockNotFound)")
 124         }
 125
 126         {
 127                 reader, _, _, err := kc.Get(hash)
 128                 c.Check(reader, Equals, nil)
 129                 c.Check(err, Equals, keepclient.BlockNotFound)
 130                 log.Print("Finished Get (expected BlockNotFound)")
 131         }
 132
 133         // Note in bug #5309 among other errors keepproxy would set
 134         // Content-Length incorrectly on the 404 BlockNotFound response, this
 135         // would result in a protocol violation that would prevent reuse of the
 136         // connection, which would manifest by the next attempt to use the
 137         // connection (in this case the PutB below) failing.  So to test for
 138         // that bug it's necessary to trigger an error response (such as
 139         // BlockNotFound) and then do something else with the same httpClient
 140         // connection.
 141
 142         {
 143                 var rep int
 144                 var err error
 145                 hash2, rep, err = kc.PutB([]byte("foo"))
 146                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 147                 c.Check(rep, Equals, 2)
 148                 c.Check(err, Equals, nil)
 149                 log.Print("Finished PutB (expected success)")
 150         }
 151
 152         {
 153                 blocklen, _, err := kc.Ask(hash2)
 154                 c.Assert(err, Equals, nil)
 155                 c.Check(blocklen, Equals, int64(3))
 156                 log.Print("Finished Ask (expected success)")
 157         }
 158
 159         {
 160                 reader, blocklen, _, err := kc.Get(hash2)
 161                 c.Assert(err, Equals, nil)
 162                 all, err := ioutil.ReadAll(reader)
 163                 c.Check(all, DeepEquals, []byte("foo"))
 164                 c.Check(blocklen, Equals, int64(3))
 165                 log.Print("Finished Get (expected success)")
 166         }
 167
 168         {
 169                 var rep int
 170                 var err error
 171                 hash2, rep, err = kc.PutB([]byte(""))
 172                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 173                 c.Check(rep, Equals, 2)
 174                 c.Check(err, Equals, nil)
 175                 log.Print("Finished PutB zero block")
 176         }
 177
 178         {
 179                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 180                 c.Assert(err, Equals, nil)
 181                 all, err := ioutil.ReadAll(reader)
 182                 c.Check(all, DeepEquals, []byte(""))
 183                 c.Check(blocklen, Equals, int64(0))
 184                 log.Print("Finished Get zero block")
 185         }
 186 }
 187
 188 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 189         kc := runProxy(c, nil, true)
 190         defer closeListener()
 191
 192         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 193
 194         {
 195                 _, _, err := kc.Ask(hash)
 196                 errNotFound, _ := err.(keepclient.ErrNotFound)
 197                 c.Check(errNotFound, NotNil)
 198                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 199                 log.Print("Ask 1")
 200         }
 201
 202         {
 203                 hash2, rep, err := kc.PutB([]byte("bar"))
 204                 c.Check(hash2, Equals, "")
 205                 c.Check(rep, Equals, 0)
 206                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 207                 log.Print("PutB")
 208         }
 209
 210         {
 211                 blocklen, _, err := kc.Ask(hash)
 212                 errNotFound, _ := err.(keepclient.ErrNotFound)
 213                 c.Check(errNotFound, NotNil)
 214                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 215                 c.Check(blocklen, Equals, int64(0))
 216                 log.Print("Ask 2")
 217         }
 218
 219         {
 220                 _, blocklen, _, err := kc.Get(hash)
 221                 errNotFound, _ := err.(keepclient.ErrNotFound)
 222                 c.Check(errNotFound, NotNil)
 223                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 224                 c.Check(blocklen, Equals, int64(0))
 225                 log.Print("Get")
 226         }
 227 }
 228
 229 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 230         kc := runProxy(c, []string{"-no-get"}, false)
 231         defer closeListener()
 232
 233         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 234
 235         {
 236                 _, _, err := kc.Ask(hash)
 237                 errNotFound, _ := err.(keepclient.ErrNotFound)
 238                 c.Check(errNotFound, NotNil)
 239                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 240                 log.Print("Ask 1")
 241         }
 242
 243         {
 244                 hash2, rep, err := kc.PutB([]byte("baz"))
 245                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 246                 c.Check(rep, Equals, 2)
 247                 c.Check(err, Equals, nil)
 248                 log.Print("PutB")
 249         }
 250
 251         {
 252                 blocklen, _, err := kc.Ask(hash)
 253                 errNotFound, _ := err.(keepclient.ErrNotFound)
 254                 c.Check(errNotFound, NotNil)
 255                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 256                 c.Check(blocklen, Equals, int64(0))
 257                 log.Print("Ask 2")
 258         }
 259
 260         {
 261                 _, blocklen, _, err := kc.Get(hash)
 262                 errNotFound, _ := err.(keepclient.ErrNotFound)
 263                 c.Check(errNotFound, NotNil)
 264                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 265                 c.Check(blocklen, Equals, int64(0))
 266                 log.Print("Get")
 267         }
 268 }
 269
 270 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 271         kc := runProxy(c, []string{"-no-put"}, false)
 272         defer closeListener()
 273
 274         hash2, rep, err := kc.PutB([]byte("quux"))
 275         c.Check(hash2, Equals, "")
 276         c.Check(rep, Equals, 0)
 277         c.Check(err, Equals, keepclient.InsufficientReplicasError)
 278 }
 279
 280 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 281         runProxy(c, nil, false)
 282         defer closeListener()
 283
 284         {
 285                 client := http.Client{}
 286                 req, err := http.NewRequest("OPTIONS",
 287                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 288                         nil)
 289                 req.Header.Add("Access-Control-Request-Method", "PUT")
 290                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 291                 resp, err := client.Do(req)
 292                 c.Check(err, Equals, nil)
 293                 c.Check(resp.StatusCode, Equals, 200)
 294                 body, err := ioutil.ReadAll(resp.Body)
 295                 c.Check(string(body), Equals, "")
 296                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 297                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 298         }
 299
 300         {
 301                 resp, err := http.Get(
 302                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 303                 c.Check(err, Equals, nil)
 304                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 305                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 306         }
 307 }
 308
 309 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 310         runProxy(c, nil, false)
 311         defer closeListener()
 312
 313         {
 314                 client := http.Client{}
 315                 req, err := http.NewRequest("POST",
 316                         "http://"+listener.Addr().String()+"/",
 317                         strings.NewReader("qux"))
 318                 req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 319                 req.Header.Add("Content-Type", "application/octet-stream")
 320                 resp, err := client.Do(req)
 321                 c.Check(err, Equals, nil)
 322                 body, err := ioutil.ReadAll(resp.Body)
 323                 c.Check(err, Equals, nil)
 324                 c.Check(string(body), Matches,
 325                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 326         }
 327 }
 328
 329 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 330         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 331                 Equals,
 332                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 333         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 334                 Equals,
 335                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 336         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 337                 Equals,
 338                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 339         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 340                 Equals,
 341                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 342
 343 }
 344
 345 // Test GetIndex
 346 //   Put one block, with 2 replicas
 347 //   With no prefix (expect the block locator, twice)
 348 //   With an existing prefix (expect the block locator, twice)
 349 //   With a valid but non-existing prefix (expect "\n")
 350 //   With an invalid prefix (expect error)
 351 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 352         kc := runProxy(c, nil, false)
 353         defer closeListener()
 354
 355         // Put "index-data" blocks
 356         data := []byte("index-data")
 357         hash := fmt.Sprintf("%x", md5.Sum(data))
 358
 359         hash2, rep, err := kc.PutB(data)
 360         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 361         c.Check(rep, Equals, 2)
 362         c.Check(err, Equals, nil)
 363
 364         reader, blocklen, _, err := kc.Get(hash)
 365         c.Assert(err, Equals, nil)
 366         c.Check(blocklen, Equals, int64(10))
 367         all, err := ioutil.ReadAll(reader)
 368         c.Check(all, DeepEquals, data)
 369
 370         // Put some more blocks
 371         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 372         c.Check(err, Equals, nil)
 373
 374         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 375
 376         // Invoke GetIndex
 377         for _, spec := range []struct {
 378                 prefix         string
 379                 expectTestHash bool
 380                 expectOther    bool
 381         }{
 382                 {"", true, true},         // with no prefix
 383                 {hash[:3], true, false},  // with matching prefix
 384                 {"abcdef", false, false}, // with no such prefix
 385         } {
 386                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 387                 c.Assert(err, Equals, nil)
 388                 indexResp, err := ioutil.ReadAll(indexReader)
 389                 c.Assert(err, Equals, nil)
 390                 locators := strings.Split(string(indexResp), "\n")
 391                 gotTestHash := 0
 392                 gotOther := 0
 393                 for _, locator := range locators {
 394                         if locator == "" {
 395                                 continue
 396                         }
 397                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 398                         if locator[:32] == hash {
 399                                 gotTestHash++
 400                         } else {
 401                                 gotOther++
 402                         }
 403                 }
 404                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 405                 c.Check(gotOther > 0, Equals, spec.expectOther)
 406         }
 407
 408         // GetIndex with invalid prefix
 409         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 410         c.Assert((err != nil), Equals, true)
 411 }
 412
 413 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 414         kc := runProxy(c, nil, false)
 415         defer closeListener()
 416
 417         // Put a test block
 418         hash, rep, err := kc.PutB([]byte("foo"))
 419         c.Check(err, Equals, nil)
 420         c.Check(rep, Equals, 2)
 421
 422         for _, token := range []string{
 423                 "nosuchtoken",
 424                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 425         } {
 426                 // Change token to given bad token
 427                 kc.Arvados.ApiToken = token
 428
 429                 // Ask should result in error
 430                 _, _, err = kc.Ask(hash)
 431                 c.Check(err, NotNil)
 432                 errNotFound, _ := err.(keepclient.ErrNotFound)
 433                 c.Check(errNotFound.Temporary(), Equals, false)
 434                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 435
 436                 // Get should result in error
 437                 _, _, _, err = kc.Get(hash)
 438                 c.Check(err, NotNil)
 439                 errNotFound, _ = err.(keepclient.ErrNotFound)
 440                 c.Check(errNotFound.Temporary(), Equals, false)
 441                 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
 442         }
 443 }
 444
 445 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 446         arv, err := arvadosclient.MakeArvadosClient()
 447         c.Assert(err, Equals, nil)
 448
 449         // keepclient with no such keep server
 450         kc := keepclient.New(&arv)
 451         locals := map[string]string{
 452                 TestProxyUUID: "http://localhost:12345",
 453         }
 454         kc.SetServiceRoots(locals, nil, nil)
 455
 456         // Ask should result in temporary connection refused error
 457         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 458         _, _, err = kc.Ask(hash)
 459         c.Check(err, NotNil)
 460         errNotFound, _ := err.(*keepclient.ErrNotFound)
 461         c.Check(errNotFound.Temporary(), Equals, true)
 462         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 463
 464         // Get should result in temporary connection refused error
 465         _, _, _, err = kc.Get(hash)
 466         c.Check(err, NotNil)
 467         errNotFound, _ = err.(*keepclient.ErrNotFound)
 468         c.Check(errNotFound.Temporary(), Equals, true)
 469         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 470 }
 471
 472 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 473         kc := runProxy(c, nil, false)
 474         defer closeListener()
 475
 476         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 477         for _, f := range []func()error {
 478                 func() error {
 479                         _, _, err := kc.Ask(hash)
 480                         return err
 481                 },
 482                 func() error {
 483                         _, _, _, err := kc.Get(hash)
 484                         return err
 485                 },
 486         } {
 487                 err := f()
 488                 c.Assert(err, NotNil)
 489                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 490                 c.Check(errNotFound.Temporary(), Equals, true)
 491                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 492         }
 493 }