services/keepproxy/keepproxy_test.go

   1 package main
   2
   3 import (
   4         "crypto/md5"
   5         "fmt"
   6         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   8         "git.curoverse.com/arvados.git/sdk/go/keepclient"
   9         "io/ioutil"
  10         "log"
  11         "net/http"
  12         "os"
  13         "strings"
  14         "testing"
  15         "time"
  16
  17         . "gopkg.in/check.v1"
  18 )
  19
  20 // Gocheck boilerplate
  21 func Test(t *testing.T) {
  22         TestingT(t)
  23 }
  24
  25 // Gocheck boilerplate
  26 var _ = Suite(&ServerRequiredSuite{})
  27
  28 // Tests that require the Keep server running
  29 type ServerRequiredSuite struct{}
  30
  31 // Gocheck boilerplate
  32 var _ = Suite(&NoKeepServerSuite{})
  33
  34 // Test with no keepserver to simulate errors
  35 type NoKeepServerSuite struct{}
  36
  37 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  38
  39 // Wait (up to 1 second) for keepproxy to listen on a port. This
  40 // avoids a race condition where we hit a "connection refused" error
  41 // because we start testing the proxy too soon.
  42 func waitForListener() {
  43         const (
  44                 ms = 5
  45         )
  46         for i := 0; listener == nil && i < 1000; i += ms {
  47                 time.Sleep(ms * time.Millisecond)
  48         }
  49         if listener == nil {
  50                 log.Fatalf("Timed out waiting for listener to start")
  51         }
  52 }
  53
  54 func closeListener() {
  55         if listener != nil {
  56                 listener.Close()
  57         }
  58 }
  59
  60 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  61         arvadostest.StartAPI()
  62         arvadostest.StartKeep(2, false)
  63 }
  64
  65 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  66         arvadostest.ResetEnv()
  67 }
  68
  69 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  70         arvadostest.StopKeep(2)
  71         arvadostest.StopAPI()
  72 }
  73
  74 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
  75         arvadostest.StartAPI()
  76 }
  77
  78 func (s *NoKeepServerSuite) SetUpTest(c *C) {
  79         arvadostest.ResetEnv()
  80 }
  81
  82 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
  83         arvadostest.StopAPI()
  84 }
  85
  86 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
  87         args = append([]string{"keepproxy"}, args...)
  88         os.Args = append(args, "-listen=:0")
  89         listener = nil
  90         go main()
  91         waitForListener()
  92
  93         arv, err := arvadosclient.MakeArvadosClient()
  94         c.Assert(err, Equals, nil)
  95         if bogusClientToken {
  96                 arv.ApiToken = "bogus-token"
  97         }
  98         kc := keepclient.New(&arv)
  99         sr := map[string]string{
 100                 TestProxyUUID: "http://" + listener.Addr().String(),
 101         }
 102         kc.SetServiceRoots(sr, sr, sr)
 103         kc.Arvados.External = true
 104         kc.Using_proxy = true
 105
 106         return &kc
 107 }
 108
 109 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 110         kc := runProxy(c, nil, false)
 111         defer closeListener()
 112
 113         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 114         var hash2 string
 115
 116         {
 117                 _, _, err := kc.Ask(hash)
 118                 c.Check(err, Equals, keepclient.BlockNotFound)
 119                 log.Print("Finished Ask (expected BlockNotFound)")
 120         }
 121
 122         {
 123                 reader, _, _, err := kc.Get(hash)
 124                 c.Check(reader, Equals, nil)
 125                 c.Check(err, Equals, keepclient.BlockNotFound)
 126                 log.Print("Finished Get (expected BlockNotFound)")
 127         }
 128
 129         // Note in bug #5309 among other errors keepproxy would set
 130         // Content-Length incorrectly on the 404 BlockNotFound response, this
 131         // would result in a protocol violation that would prevent reuse of the
 132         // connection, which would manifest by the next attempt to use the
 133         // connection (in this case the PutB below) failing.  So to test for
 134         // that bug it's necessary to trigger an error response (such as
 135         // BlockNotFound) and then do something else with the same httpClient
 136         // connection.
 137
 138         {
 139                 var rep int
 140                 var err error
 141                 hash2, rep, err = kc.PutB([]byte("foo"))
 142                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 143                 c.Check(rep, Equals, 2)
 144                 c.Check(err, Equals, nil)
 145                 log.Print("Finished PutB (expected success)")
 146         }
 147
 148         {
 149                 blocklen, _, err := kc.Ask(hash2)
 150                 c.Assert(err, Equals, nil)
 151                 c.Check(blocklen, Equals, int64(3))
 152                 log.Print("Finished Ask (expected success)")
 153         }
 154
 155         {
 156                 reader, blocklen, _, err := kc.Get(hash2)
 157                 c.Assert(err, Equals, nil)
 158                 all, err := ioutil.ReadAll(reader)
 159                 c.Check(all, DeepEquals, []byte("foo"))
 160                 c.Check(blocklen, Equals, int64(3))
 161                 log.Print("Finished Get (expected success)")
 162         }
 163
 164         {
 165                 var rep int
 166                 var err error
 167                 hash2, rep, err = kc.PutB([]byte(""))
 168                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 169                 c.Check(rep, Equals, 2)
 170                 c.Check(err, Equals, nil)
 171                 log.Print("Finished PutB zero block")
 172         }
 173
 174         {
 175                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 176                 c.Assert(err, Equals, nil)
 177                 all, err := ioutil.ReadAll(reader)
 178                 c.Check(all, DeepEquals, []byte(""))
 179                 c.Check(blocklen, Equals, int64(0))
 180                 log.Print("Finished Get zero block")
 181         }
 182 }
 183
 184 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 185         kc := runProxy(c, nil, true)
 186         defer closeListener()
 187
 188         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 189
 190         {
 191                 _, _, err := kc.Ask(hash)
 192                 errNotFound, _ := err.(keepclient.ErrNotFound)
 193                 c.Check(errNotFound, NotNil)
 194                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 195                 log.Print("Ask 1")
 196         }
 197
 198         {
 199                 hash2, rep, err := kc.PutB([]byte("bar"))
 200                 c.Check(hash2, Equals, "")
 201                 c.Check(rep, Equals, 0)
 202                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 203                 log.Print("PutB")
 204         }
 205
 206         {
 207                 blocklen, _, err := kc.Ask(hash)
 208                 errNotFound, _ := err.(keepclient.ErrNotFound)
 209                 c.Check(errNotFound, NotNil)
 210                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 211                 c.Check(blocklen, Equals, int64(0))
 212                 log.Print("Ask 2")
 213         }
 214
 215         {
 216                 _, blocklen, _, err := kc.Get(hash)
 217                 errNotFound, _ := err.(keepclient.ErrNotFound)
 218                 c.Check(errNotFound, NotNil)
 219                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 220                 c.Check(blocklen, Equals, int64(0))
 221                 log.Print("Get")
 222         }
 223 }
 224
 225 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 226         kc := runProxy(c, []string{"-no-get"}, false)
 227         defer closeListener()
 228
 229         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 230
 231         {
 232                 _, _, err := kc.Ask(hash)
 233                 errNotFound, _ := err.(keepclient.ErrNotFound)
 234                 c.Check(errNotFound, NotNil)
 235                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 236                 log.Print("Ask 1")
 237         }
 238
 239         {
 240                 hash2, rep, err := kc.PutB([]byte("baz"))
 241                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 242                 c.Check(rep, Equals, 2)
 243                 c.Check(err, Equals, nil)
 244                 log.Print("PutB")
 245         }
 246
 247         {
 248                 blocklen, _, err := kc.Ask(hash)
 249                 errNotFound, _ := err.(keepclient.ErrNotFound)
 250                 c.Check(errNotFound, NotNil)
 251                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 252                 c.Check(blocklen, Equals, int64(0))
 253                 log.Print("Ask 2")
 254         }
 255
 256         {
 257                 _, blocklen, _, err := kc.Get(hash)
 258                 errNotFound, _ := err.(keepclient.ErrNotFound)
 259                 c.Check(errNotFound, NotNil)
 260                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
 261                 c.Check(blocklen, Equals, int64(0))
 262                 log.Print("Get")
 263         }
 264 }
 265
 266 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 267         kc := runProxy(c, []string{"-no-put"}, false)
 268         defer closeListener()
 269
 270         hash2, rep, err := kc.PutB([]byte("quux"))
 271         c.Check(hash2, Equals, "")
 272         c.Check(rep, Equals, 0)
 273         c.Check(err, Equals, keepclient.InsufficientReplicasError)
 274 }
 275
 276 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 277         runProxy(c, nil, false)
 278         defer closeListener()
 279
 280         {
 281                 client := http.Client{}
 282                 req, err := http.NewRequest("OPTIONS",
 283                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 284                         nil)
 285                 req.Header.Add("Access-Control-Request-Method", "PUT")
 286                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 287                 resp, err := client.Do(req)
 288                 c.Check(err, Equals, nil)
 289                 c.Check(resp.StatusCode, Equals, 200)
 290                 body, err := ioutil.ReadAll(resp.Body)
 291                 c.Check(string(body), Equals, "")
 292                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 293                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 294         }
 295
 296         {
 297                 resp, err := http.Get(
 298                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 299                 c.Check(err, Equals, nil)
 300                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 301                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 302         }
 303 }
 304
 305 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 306         runProxy(c, nil, false)
 307         defer closeListener()
 308
 309         {
 310                 client := http.Client{}
 311                 req, err := http.NewRequest("POST",
 312                         "http://"+listener.Addr().String()+"/",
 313                         strings.NewReader("qux"))
 314                 req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 315                 req.Header.Add("Content-Type", "application/octet-stream")
 316                 resp, err := client.Do(req)
 317                 c.Check(err, Equals, nil)
 318                 body, err := ioutil.ReadAll(resp.Body)
 319                 c.Check(err, Equals, nil)
 320                 c.Check(string(body), Matches,
 321                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 322         }
 323 }
 324
 325 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 326         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 327                 Equals,
 328                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 329         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 330                 Equals,
 331                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 332         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 333                 Equals,
 334                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 335         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 336                 Equals,
 337                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 338
 339 }
 340
 341 // Test GetIndex
 342 //   Put one block, with 2 replicas
 343 //   With no prefix (expect the block locator, twice)
 344 //   With an existing prefix (expect the block locator, twice)
 345 //   With a valid but non-existing prefix (expect "\n")
 346 //   With an invalid prefix (expect error)
 347 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 348         kc := runProxy(c, nil, false)
 349         defer closeListener()
 350
 351         // Put "index-data" blocks
 352         data := []byte("index-data")
 353         hash := fmt.Sprintf("%x", md5.Sum(data))
 354
 355         hash2, rep, err := kc.PutB(data)
 356         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 357         c.Check(rep, Equals, 2)
 358         c.Check(err, Equals, nil)
 359
 360         reader, blocklen, _, err := kc.Get(hash)
 361         c.Assert(err, Equals, nil)
 362         c.Check(blocklen, Equals, int64(10))
 363         all, err := ioutil.ReadAll(reader)
 364         c.Check(all, DeepEquals, data)
 365
 366         // Put some more blocks
 367         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 368         c.Check(err, Equals, nil)
 369
 370         // Invoke GetIndex
 371         for _, spec := range []struct {
 372                 prefix         string
 373                 expectTestHash bool
 374                 expectOther    bool
 375         }{
 376                 {"", true, true},         // with no prefix
 377                 {hash[:3], true, false},  // with matching prefix
 378                 {"abcdef", false, false}, // with no such prefix
 379         } {
 380                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 381                 c.Assert(err, Equals, nil)
 382                 indexResp, err := ioutil.ReadAll(indexReader)
 383                 c.Assert(err, Equals, nil)
 384                 locators := strings.Split(string(indexResp), "\n")
 385                 gotTestHash := 0
 386                 gotOther := 0
 387                 for _, locator := range locators {
 388                         if locator == "" {
 389                                 continue
 390                         }
 391                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 392                         if locator[:32] == hash {
 393                                 gotTestHash++
 394                         } else {
 395                                 gotOther++
 396                         }
 397                 }
 398                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 399                 c.Check(gotOther > 0, Equals, spec.expectOther)
 400         }
 401
 402         // GetIndex with invalid prefix
 403         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 404         c.Assert((err != nil), Equals, true)
 405 }
 406
 407 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 408         kc := runProxy(c, []string{"keepproxy"}, 28852, false)
 409         waitForListener()
 410         defer closeListener()
 411
 412         // Put a test block
 413         hash, rep, err := kc.PutB([]byte("foo"))
 414         c.Check(err, Equals, nil)
 415         c.Check(rep, Equals, 2)
 416
 417         for _, token := range []string{
 418                 "nosuchtoken",
 419                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
 420         } {
 421                 // Change token to given bad token
 422                 kc.Arvados.ApiToken = token
 423
 424                 // Ask should result in error
 425                 _, _, err = kc.Ask(hash)
 426                 c.Check(err, NotNil)
 427                 errNotFound, _ := err.(keepclient.ErrNotFound)
 428                 c.Check(errNotFound.Temporary(), Equals, false)
 429                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
 430
 431                 // Get should result in error
 432                 _, _, _, err = kc.Get(hash)
 433                 c.Check(err, NotNil)
 434                 errNotFound, _ = err.(keepclient.ErrNotFound)
 435                 c.Check(errNotFound.Temporary(), Equals, false)
 436                 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
 437         }
 438 }
 439
 440 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 441         arv, err := arvadosclient.MakeArvadosClient()
 442         c.Assert(err, Equals, nil)
 443
 444         // keepclient with no such keep server
 445         kc := keepclient.New(&arv)
 446         locals := map[string]string{
 447                 "proxy": "http://localhost:12345",
 448         }
 449         kc.SetServiceRoots(locals, nil, nil)
 450
 451         // Ask should result in temporary connection refused error
 452         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 453         _, _, err = kc.Ask(hash)
 454         c.Check(err, NotNil)
 455         errNotFound, _ := err.(*keepclient.ErrNotFound)
 456         c.Check(errNotFound.Temporary(), Equals, true)
 457         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 458
 459         // Get should result in temporary connection refused error
 460         _, _, _, err = kc.Get(hash)
 461         c.Check(err, NotNil)
 462         errNotFound, _ = err.(*keepclient.ErrNotFound)
 463         c.Check(errNotFound.Temporary(), Equals, true)
 464         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
 465 }
 466
 467 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 468         kc := runProxy(c, []string{"keepproxy"}, 29999, false)
 469         waitForListener()
 470         defer closeListener()
 471
 472         // Ask should result in temporary connection refused error
 473         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 474         _, _, err := kc.Ask(hash)
 475         c.Check(err, NotNil)
 476         errNotFound, _ := err.(*keepclient.ErrNotFound)
 477         c.Check(errNotFound.Temporary(), Equals, true)
 478         c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true)
 479
 480         // Get should result in temporary connection refused error
 481         _, _, _, err = kc.Get(hash)
 482         c.Check(err, NotNil)
 483         errNotFound, _ = err.(*keepclient.ErrNotFound)
 484         c.Check(errNotFound.Temporary(), Equals, true)
 485         c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true)
 486 }