services/keepproxy/keepproxy_test.go

   1 package main
   2
   3 import (
   4         "crypto/md5"
   5         "fmt"
   6         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   8         "git.curoverse.com/arvados.git/sdk/go/keepclient"
   9         "io/ioutil"
  10         "log"
  11         "net/http"
  12         "os"
  13         "strings"
  14         "testing"
  15         "time"
  16
  17         . "gopkg.in/check.v1"
  18 )
  19
  20 // Gocheck boilerplate
  21 func Test(t *testing.T) {
  22         TestingT(t)
  23 }
  24
  25 // Gocheck boilerplate
  26 var _ = Suite(&ServerRequiredSuite{})
  27
  28 // Tests that require the Keep server running
  29 type ServerRequiredSuite struct{}
  30
  31 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
  32
  33 // Wait (up to 1 second) for keepproxy to listen on a port. This
  34 // avoids a race condition where we hit a "connection refused" error
  35 // because we start testing the proxy too soon.
  36 func waitForListener() {
  37         const (
  38                 ms = 5
  39         )
  40         for i := 0; listener == nil && i < 1000; i += ms {
  41                 time.Sleep(ms * time.Millisecond)
  42         }
  43         if listener == nil {
  44                 log.Fatalf("Timed out waiting for listener to start")
  45         }
  46 }
  47
  48 func closeListener() {
  49         if listener != nil {
  50                 listener.Close()
  51         }
  52 }
  53
  54 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  55         arvadostest.StartAPI()
  56         arvadostest.StartKeep()
  57 }
  58
  59 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  60         arvadostest.ResetEnv()
  61 }
  62
  63 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  64         arvadostest.StopKeep()
  65         arvadostest.StopAPI()
  66 }
  67
  68 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
  69         args = append([]string{"keepproxy"}, args...)
  70         os.Args = append(args, "-listen=:0")
  71         listener = nil
  72         go main()
  73         waitForListener()
  74
  75         arv, err := arvadosclient.MakeArvadosClient()
  76         c.Assert(err, Equals, nil)
  77         if bogusClientToken {
  78                 arv.ApiToken = "bogus-token"
  79         }
  80         kc := keepclient.New(&arv)
  81         sr := map[string]string{
  82                 TestProxyUUID: "http://" + listener.Addr().String(),
  83         }
  84         kc.SetServiceRoots(sr, sr, sr)
  85         kc.Arvados.External = true
  86         kc.Using_proxy = true
  87
  88         return kc
  89 }
  90
  91 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
  92         kc := runProxy(c, nil, false)
  93         defer closeListener()
  94
  95         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
  96         var hash2 string
  97
  98         {
  99                 _, _, err := kc.Ask(hash)
 100                 c.Check(err, Equals, keepclient.BlockNotFound)
 101                 log.Print("Finished Ask (expected BlockNotFound)")
 102         }
 103
 104         {
 105                 reader, _, _, err := kc.Get(hash)
 106                 c.Check(reader, Equals, nil)
 107                 c.Check(err, Equals, keepclient.BlockNotFound)
 108                 log.Print("Finished Get (expected BlockNotFound)")
 109         }
 110
 111         // Note in bug #5309 among other errors keepproxy would set
 112         // Content-Length incorrectly on the 404 BlockNotFound response, this
 113         // would result in a protocol violation that would prevent reuse of the
 114         // connection, which would manifest by the next attempt to use the
 115         // connection (in this case the PutB below) failing.  So to test for
 116         // that bug it's necessary to trigger an error response (such as
 117         // BlockNotFound) and then do something else with the same httpClient
 118         // connection.
 119
 120         {
 121                 var rep int
 122                 var err error
 123                 hash2, rep, err = kc.PutB([]byte("foo"))
 124                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 125                 c.Check(rep, Equals, 2)
 126                 c.Check(err, Equals, nil)
 127                 log.Print("Finished PutB (expected success)")
 128         }
 129
 130         {
 131                 blocklen, _, err := kc.Ask(hash2)
 132                 c.Assert(err, Equals, nil)
 133                 c.Check(blocklen, Equals, int64(3))
 134                 log.Print("Finished Ask (expected success)")
 135         }
 136
 137         {
 138                 reader, blocklen, _, err := kc.Get(hash2)
 139                 c.Assert(err, Equals, nil)
 140                 all, err := ioutil.ReadAll(reader)
 141                 c.Check(all, DeepEquals, []byte("foo"))
 142                 c.Check(blocklen, Equals, int64(3))
 143                 log.Print("Finished Get (expected success)")
 144         }
 145
 146         {
 147                 var rep int
 148                 var err error
 149                 hash2, rep, err = kc.PutB([]byte(""))
 150                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 151                 c.Check(rep, Equals, 2)
 152                 c.Check(err, Equals, nil)
 153                 log.Print("Finished PutB zero block")
 154         }
 155
 156         {
 157                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 158                 c.Assert(err, Equals, nil)
 159                 all, err := ioutil.ReadAll(reader)
 160                 c.Check(all, DeepEquals, []byte(""))
 161                 c.Check(blocklen, Equals, int64(0))
 162                 log.Print("Finished Get zero block")
 163         }
 164 }
 165
 166 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 167         kc := runProxy(c, nil, true)
 168         defer closeListener()
 169
 170         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 171
 172         {
 173                 _, _, err := kc.Ask(hash)
 174                 c.Check(err, Equals, keepclient.BlockNotFound)
 175                 log.Print("Ask 1")
 176         }
 177
 178         {
 179                 hash2, rep, err := kc.PutB([]byte("bar"))
 180                 c.Check(hash2, Equals, "")
 181                 c.Check(rep, Equals, 0)
 182                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 183                 log.Print("PutB")
 184         }
 185
 186         {
 187                 blocklen, _, err := kc.Ask(hash)
 188                 c.Assert(err, Equals, keepclient.BlockNotFound)
 189                 c.Check(blocklen, Equals, int64(0))
 190                 log.Print("Ask 2")
 191         }
 192
 193         {
 194                 _, blocklen, _, err := kc.Get(hash)
 195                 c.Assert(err, Equals, keepclient.BlockNotFound)
 196                 c.Check(blocklen, Equals, int64(0))
 197                 log.Print("Get")
 198         }
 199 }
 200
 201 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 202         kc := runProxy(c, []string{"-no-get"}, false)
 203         defer closeListener()
 204
 205         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 206
 207         {
 208                 _, _, err := kc.Ask(hash)
 209                 c.Check(err, Equals, keepclient.BlockNotFound)
 210                 log.Print("Ask 1")
 211         }
 212
 213         {
 214                 hash2, rep, err := kc.PutB([]byte("baz"))
 215                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 216                 c.Check(rep, Equals, 2)
 217                 c.Check(err, Equals, nil)
 218                 log.Print("PutB")
 219         }
 220
 221         {
 222                 blocklen, _, err := kc.Ask(hash)
 223                 c.Assert(err, Equals, keepclient.BlockNotFound)
 224                 c.Check(blocklen, Equals, int64(0))
 225                 log.Print("Ask 2")
 226         }
 227
 228         {
 229                 _, blocklen, _, err := kc.Get(hash)
 230                 c.Assert(err, Equals, keepclient.BlockNotFound)
 231                 c.Check(blocklen, Equals, int64(0))
 232                 log.Print("Get")
 233         }
 234 }
 235
 236 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 237         kc := runProxy(c, []string{"-no-put"}, false)
 238         defer closeListener()
 239
 240         hash2, rep, err := kc.PutB([]byte("quux"))
 241         c.Check(hash2, Equals, "")
 242         c.Check(rep, Equals, 0)
 243         c.Check(err, Equals, keepclient.InsufficientReplicasError)
 244 }
 245
 246 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 247         runProxy(c, nil, false)
 248         defer closeListener()
 249
 250         {
 251                 client := http.Client{}
 252                 req, err := http.NewRequest("OPTIONS",
 253                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 254                         nil)
 255                 req.Header.Add("Access-Control-Request-Method", "PUT")
 256                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 257                 resp, err := client.Do(req)
 258                 c.Check(err, Equals, nil)
 259                 c.Check(resp.StatusCode, Equals, 200)
 260                 body, err := ioutil.ReadAll(resp.Body)
 261                 c.Check(string(body), Equals, "")
 262                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 263                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 264         }
 265
 266         {
 267                 resp, err := http.Get(
 268                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 269                 c.Check(err, Equals, nil)
 270                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 271                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 272         }
 273 }
 274
 275 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 276         runProxy(c, nil, false)
 277         defer closeListener()
 278
 279         {
 280                 client := http.Client{}
 281                 req, err := http.NewRequest("POST",
 282                         "http://"+listener.Addr().String()+"/",
 283                         strings.NewReader("qux"))
 284                 req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 285                 req.Header.Add("Content-Type", "application/octet-stream")
 286                 resp, err := client.Do(req)
 287                 c.Check(err, Equals, nil)
 288                 body, err := ioutil.ReadAll(resp.Body)
 289                 c.Check(err, Equals, nil)
 290                 c.Check(string(body), Matches,
 291                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 292         }
 293 }
 294
 295 func (s *ServerRequiredSuite) TestStripHint(c *C) {
 296         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
 297                 Equals,
 298                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 299         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 300                 Equals,
 301                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 302         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
 303                 Equals,
 304                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 305         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
 306                 Equals,
 307                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 308
 309 }
 310
 311 // Test GetIndex
 312 //   Put one block, with 2 replicas
 313 //   With no prefix (expect the block locator, twice)
 314 //   With an existing prefix (expect the block locator, twice)
 315 //   With a valid but non-existing prefix (expect "\n")
 316 //   With an invalid prefix (expect error)
 317 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 318         kc := runProxy(c, nil, false)
 319         defer closeListener()
 320
 321         // Put "index-data" blocks
 322         data := []byte("index-data")
 323         hash := fmt.Sprintf("%x", md5.Sum(data))
 324
 325         hash2, rep, err := kc.PutB(data)
 326         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
 327         c.Check(rep, Equals, 2)
 328         c.Check(err, Equals, nil)
 329
 330         reader, blocklen, _, err := kc.Get(hash)
 331         c.Assert(err, Equals, nil)
 332         c.Check(blocklen, Equals, int64(10))
 333         all, err := ioutil.ReadAll(reader)
 334         c.Check(all, DeepEquals, data)
 335
 336         // Put some more blocks
 337         _, rep, err = kc.PutB([]byte("some-more-index-data"))
 338         c.Check(err, Equals, nil)
 339
 340         // Invoke GetIndex
 341         for _, spec := range []struct {
 342                 prefix         string
 343                 expectTestHash bool
 344                 expectOther    bool
 345         }{
 346                 {"", true, true},         // with no prefix
 347                 {hash[:3], true, false},  // with matching prefix
 348                 {"abcdef", false, false}, // with no such prefix
 349         } {
 350                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
 351                 c.Assert(err, Equals, nil)
 352                 indexResp, err := ioutil.ReadAll(indexReader)
 353                 c.Assert(err, Equals, nil)
 354                 locators := strings.Split(string(indexResp), "\n")
 355                 gotTestHash := 0
 356                 gotOther := 0
 357                 for _, locator := range locators {
 358                         if locator == "" {
 359                                 continue
 360                         }
 361                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
 362                         if locator[:32] == hash {
 363                                 gotTestHash++
 364                         } else {
 365                                 gotOther++
 366                         }
 367                 }
 368                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
 369                 c.Check(gotOther > 0, Equals, spec.expectOther)
 370         }
 371
 372         // GetIndex with invalid prefix
 373         _, err = kc.GetIndex(TestProxyUUID, "xyz")
 374         c.Assert((err != nil), Equals, true)
 375 }