services/keepproxy/keepproxy_test.go

   1 package main
   2
   3 import (
   4         "crypto/md5"
   5         "fmt"
   6         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   8         "git.curoverse.com/arvados.git/sdk/go/keepclient"
   9         "io/ioutil"
  10         "log"
  11         "net/http"
  12         "os"
  13         "strings"
  14         "testing"
  15         "time"
  16
  17         . "gopkg.in/check.v1"
  18 )
  19
  20 // Gocheck boilerplate
  21 func Test(t *testing.T) {
  22         TestingT(t)
  23 }
  24
  25 // Gocheck boilerplate
  26 var _ = Suite(&ServerRequiredSuite{})
  27
  28 // Tests that require the Keep server running
  29 type ServerRequiredSuite struct{}
  30
  31 // Gocheck boilerplate
  32 var _ = Suite(&NoKeepServerSuite{})
  33
  34 // Test with no keepserver to simulate errors
  35 type NoKeepServerSuite struct{}
  36
  37 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  38
  39 // Wait (up to 1 second) for keepproxy to listen on a port. This
  40 // avoids a race condition where we hit a "connection refused" error
  41 // because we start testing the proxy too soon.
  42 func waitForListener() {
  43         const (
  44                 ms = 5
  45         )
  46         for i := 0; listener == nil && i < 10000; i += ms {
  47                 time.Sleep(ms * time.Millisecond)
  48         }
  49         if listener == nil {
  50                 log.Fatalf("Timed out waiting for listener to start")
  51         }
  52 }
  53
  54 func closeListener() {
  55         if listener != nil {
  56                 listener.Close()
  57         }
  58 }
  59
  60 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  61         arvadostest.StartAPI()
  62         arvadostest.StartKeep(2, false)
  63 }
  64
  65 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  66         arvadostest.ResetEnv()
  67 }
  68
  69 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  70         arvadostest.StopKeep(2)
  71         arvadostest.StopAPI()
  72 }
  73
  74 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  75         arvadostest.StartAPI()
  76         // We need API to have some keep services listed, but the
  77         // services themselves should be unresponsive.
  78         arvadostest.StartKeep(2, false)
  79         arvadostest.StopKeep(2)
  80 }
  81
  82 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  83         arvadostest.ResetEnv()
  84 }
  85
  86 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  87         arvadostest.StopAPI()
  88 }
  89
  90 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
  91         args = append([]string{"keepproxy"}, args...)
  92         os.Args = append(args, "-listen=:0")
  93         listener = nil
  94         go main()
  95         waitForListener()
  96
  97         arv, err := arvadosclient.MakeArvadosClient()
  98         c.Assert(err, Equals, nil)
  99         if bogusClientToken {
 100                 arv.ApiToken = "bogus-token"
 101         }
 102         kc := keepclient.New(&arv)
 103         sr := map[string]string{
 104                 TestProxyUUID: "http://" + listener.Addr().String(),
 105         }
 106         kc.SetServiceRoots(sr, sr, sr)
 107         kc.Arvados.External = true
 108
 109         return kc
 110 }
 111
 112 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 113         kc := runProxy(c, nil, false)
 114         defer closeListener()
 115
 116         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 117         var hash2 string
 118
 119         {
 120                 _, _, err := kc.Ask(hash)
 121                 c.Check(err, Equals, keepclient.BlockNotFound)
 122                 log.Print("Finished Ask (expected BlockNotFound)")
 123         }
 124
 125         {
 126                 reader, _, _, err := kc.Get(hash)
 127                 c.Check(reader, Equals, nil)
 128                 c.Check(err, Equals, keepclient.BlockNotFound)
 129                 log.Print("Finished Get (expected BlockNotFound)")
 130         }
 131
 132         // Note in bug #5309 among other errors keepproxy would set
 133         // Content-Length incorrectly on the 404 BlockNotFound response, this
 134         // would result in a protocol violation that would prevent reuse of the
 135         // connection, which would manifest by the next attempt to use the
 136         // connection (in this case the PutB below) failing.  So to test for
 137         // that bug it's necessary to trigger an error response (such as
 138         // BlockNotFound) and then do something else with the same httpClient
 139         // connection.
 140
 141         {
 142                 var rep int
 143                 var err error
 144                 hash2, rep, err = kc.PutB([]byte("foo"))
 145                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 146                 c.Check(rep, Equals, 2)
 147                 c.Check(err, Equals, nil)
 148                 log.Print("Finished PutB (expected success)")
 149         }
 150
 151         {
 152                 blocklen, _, err := kc.Ask(hash2)
 153                 c.Assert(err, Equals, nil)
 154                 c.Check(blocklen, Equals, int64(3))
 155                 log.Print("Finished Ask (expected success)")
 156         }
 157
 158         {
 159                 reader, blocklen, _, err := kc.Get(hash2)
 160                 c.Assert(err, Equals, nil)
 161                 all, err := ioutil.ReadAll(reader)
 162                 c.Check(all, DeepEquals, []byte("foo"))
 163                 c.Check(blocklen, Equals, int64(3))
 164                 log.Print("Finished Get (expected success)")
 165         }
 166
 167         {
 168                 var rep int
 169                 var err error
 170                 hash2, rep, err = kc.PutB([]byte(""))
 171                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 172                 c.Check(rep, Equals, 2)
 173                 c.Check(err, Equals, nil)
 174                 log.Print("Finished PutB zero block")
 175         }
 176
 177         {
 178                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 179                 c.Assert(err, Equals, nil)
 180                 all, err := ioutil.ReadAll(reader)
 181                 c.Check(all, DeepEquals, []byte(""))
 182                 c.Check(blocklen, Equals, int64(0))
 183                 log.Print("Finished Get zero block")
 184         }
 185 }
 186
 187 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 188         kc := runProxy(c, nil, true)
 189         defer closeListener()
 190
 191         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 192
 193         {
 194                 _, _, err := kc.Ask(hash)
 195                 errNotFound, _ := err.(keepclient.ErrNotFound)
 196                 c.Check(errNotFound, NotNil)
 197                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 198                 log.Print("Ask 1")
 199         }
 200
 201         {
 202                 hash2, rep, err := kc.PutB([]byte("bar"))
 203                 c.Check(hash2, Equals, "")
 204                 c.Check(rep, Equals, 0)
 205                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 206                 log.Print("PutB")
 207         }
 208
 209         {
 210                 blocklen, _, err := kc.Ask(hash)
 211                 errNotFound, _ := err.(keepclient.ErrNotFound)
 212                 c.Check(errNotFound, NotNil)
 213                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 214                 c.Check(blocklen, Equals, int64(0))
 215                 log.Print("Ask 2")
 216         }
 217
 218         {
 219                 _, blocklen, _, err := kc.Get(hash)
 220                 errNotFound, _ := err.(keepclient.ErrNotFound)
 221                 c.Check(errNotFound, NotNil)
 222                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 223                 c.Check(blocklen, Equals, int64(0))
 224                 log.Print("Get")
 225         }
 226 }
 227
 228 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 229         kc := runProxy(c, []string{"-no-get"}, false)
 230         defer closeListener()
 231
 232         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 233
 234         {
 235                 _, _, err := kc.Ask(hash)
 236                 errNotFound, _ := err.(keepclient.ErrNotFound)
 237                 c.Check(errNotFound, NotNil)
 238                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 239                 log.Print("Ask 1")
 240         }
 241
 242         {
 243                 hash2, rep, err := kc.PutB([]byte("baz"))
 244                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 245                 c.Check(rep, Equals, 2)
 246                 c.Check(err, Equals, nil)
 247                 log.Print("PutB")
 248         }
 249
 250         {
 251                 blocklen, _, err := kc.Ask(hash)
 252                 errNotFound, _ := err.(keepclient.ErrNotFound)
 253                 c.Check(errNotFound, NotNil)
 254                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 255                 c.Check(blocklen, Equals, int64(0))
 256                 log.Print("Ask 2")
 257         }
 258
 259         {
 260                 _, blocklen, _, err := kc.Get(hash)
 261                 errNotFound, _ := err.(keepclient.ErrNotFound)
 262                 c.Check(errNotFound, NotNil)
 263                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 264                 c.Check(blocklen, Equals, int64(0))
 265                 log.Print("Get")
 266         }
 267 }
 268
 269 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 270         kc := runProxy(c, []string{"-no-put"}, false)
 271         defer closeListener()
 272
 273         hash2, rep, err := kc.PutB([]byte("quux"))
 274         c.Check(hash2, Equals, "")
 275         c.Check(rep, Equals, 0)
 276         c.Check(err, Equals, keepclient.InsufficientReplicasError)
 277 }
 278
 279 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 280         runProxy(c, nil, false)
 281         defer closeListener()
 282
 283         {
 284                 client := http.Client{}
 285                 req, err := http.NewRequest("OPTIONS",
 286                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 287                         nil)
 288                 req.Header.Add("Access-Control-Request-Method", "PUT")
 289                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 290                 resp, err := client.Do(req)
 291                 c.Check(err, Equals, nil)
 292                 c.Check(resp.StatusCode, Equals, 200)
 293                 body, err := ioutil.ReadAll(resp.Body)
 294                 c.Check(string(body), Equals, "")
 295                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 296                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 297         }
 298
 299         {
 300                 resp, err := http.Get(
 301                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 302                 c.Check(err, Equals, nil)
 303                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 304                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 305         }
 306 }
 307
 308 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 309         runProxy(c, nil, false)
 310         defer closeListener()
 311
 312         {
 313                 client := http.Client{}
 314                 req, err := http.NewRequest("POST",
 315                         "http://"+listener.Addr().String()+"/",
 316                         strings.NewReader("qux"))
 317                 req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 318                 req.Header.Add("Content-Type", "application/octet-stream")
 319                 resp, err := client.Do(req)
 320                 c.Check(err, Equals, nil)
 321                 body, err := ioutil.ReadAll(resp.Body)
 322                 c.Check(err, Equals, nil)
 323                 c.Check(string(body), Matches,
 324                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 325         }
 326 }
 327
 328 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 329         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 330                 Equals,
 331                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 332         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 333                 Equals,
 334                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 335         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 336                 Equals,
 337                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 338         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 339                 Equals,
 340                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 341
 342 }
 343
 344 // Test GetIndex
 345 //   Put one block, with 2 replicas
 346 //   With no prefix (expect the block locator, twice)
 347 //   With an existing prefix (expect the block locator, twice)
 348 //   With a valid but non-existing prefix (expect "\n")
 349 //   With an invalid prefix (expect error)
 350 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 351         kc := runProxy(c, nil, false)
 352         defer closeListener()
 353
 354         // Put "index-data" blocks
 355         data := []byte("index-data")
 356         hash := fmt.Sprintf("%x", md5.Sum(data))
 357
 358         hash2, rep, err := kc.PutB(data)
 359         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 360         c.Check(rep, Equals, 2)
 361         c.Check(err, Equals, nil)
 362
 363         reader, blocklen, _, err := kc.Get(hash)
 364         c.Assert(err, Equals, nil)
 365         c.Check(blocklen, Equals, int64(10))
 366         all, err := ioutil.ReadAll(reader)
 367         c.Check(all, DeepEquals, data)
 368
 369         // Put some more blocks
 370         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 371         c.Check(err, Equals, nil)
 372
 373         kc.Arvados.ApiToken = arvadostest.DataManagerToken
 374
 375         // Invoke GetIndex
 376         for _, spec := range []struct {
 377                 prefix         string
 378                 expectTestHash bool
 379                 expectOther    bool
 380         }{
 381                 {"", true, true},         // with no prefix
 382                 {hash[:3], true, false},  // with matching prefix
 383                 {"abcdef", false, false}, // with no such prefix
 384         } {
 385                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 386                 c.Assert(err, Equals, nil)
 387                 indexResp, err := ioutil.ReadAll(indexReader)
 388                 c.Assert(err, Equals, nil)
 389                 locators := strings.Split(string(indexResp), "\n")
 390                 gotTestHash := 0
 391                 gotOther := 0
 392                 for _, locator := range locators {
 393                         if locator == "" {
 394                                 continue
 395                         }
 396                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 397                         if locator[:32] == hash {
 398                                 gotTestHash++
 399                         } else {
 400                                 gotOther++
 401                         }
 402                 }
 403                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 404                 c.Check(gotOther > 0, Equals, spec.expectOther)
 405         }
 406
 407         // GetIndex with invalid prefix
 408         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 409         c.Assert((err != nil), Equals, true)
 410 }
 411
 412 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 413         kc := runProxy(c, nil, false)
 414         defer closeListener()
 415
 416         // Put a test block
 417         hash, rep, err := kc.PutB([]byte("foo"))
 418         c.Check(err, Equals, nil)
 419         c.Check(rep, Equals, 2)
 420
 421         for _, token := range []string{
 422                 "nosuchtoken",
 423                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 424         } {
 425                 // Change token to given bad token
 426                 kc.Arvados.ApiToken = token
 427
 428                 // Ask should result in error
 429                 _, _, err = kc.Ask(hash)
 430                 c.Check(err, NotNil)
 431                 errNotFound, _ := err.(keepclient.ErrNotFound)
 432                 c.Check(errNotFound.Temporary(), Equals, false)
 433                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 434
 435                 // Get should result in error
 436                 _, _, _, err = kc.Get(hash)
 437                 c.Check(err, NotNil)
 438                 errNotFound, _ = err.(keepclient.ErrNotFound)
 439                 c.Check(errNotFound.Temporary(), Equals, false)
 440                 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
 441         }
 442 }
 443
 444 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 445         arv, err := arvadosclient.MakeArvadosClient()
 446         c.Assert(err, Equals, nil)
 447
 448         // keepclient with no such keep server
 449         kc := keepclient.New(&arv)
 450         locals := map[string]string{
 451                 TestProxyUUID: "http://localhost:12345",
 452         }
 453         kc.SetServiceRoots(locals, nil, nil)
 454
 455         // Ask should result in temporary connection refused error
 456         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 457         _, _, err = kc.Ask(hash)
 458         c.Check(err, NotNil)
 459         errNotFound, _ := err.(*keepclient.ErrNotFound)
 460         c.Check(errNotFound.Temporary(), Equals, true)
 461         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 462
 463         // Get should result in temporary connection refused error
 464         _, _, _, err = kc.Get(hash)
 465         c.Check(err, NotNil)
 466         errNotFound, _ = err.(*keepclient.ErrNotFound)
 467         c.Check(errNotFound.Temporary(), Equals, true)
 468         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 469 }
 470
 471 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 472         kc := runProxy(c, nil, false)
 473         defer closeListener()
 474
 475         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 476         for _, f := range []func() error{
 477                 func() error {
 478                         _, _, err := kc.Ask(hash)
 479                         return err
 480                 },
 481                 func() error {
 482                         _, _, _, err := kc.Get(hash)
 483                         return err
 484                 },
 485         } {
 486                 err := f()
 487                 c.Assert(err, NotNil)
 488                 errNotFound, _ := err.(*keepclient.ErrNotFound)
 489                 c.Check(errNotFound.Temporary(), Equals, true)
 490                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
 491         }
 492 }