services/keepproxy/keepproxy_test.go

   1 package main
   2
   3 import (
   4         "crypto/md5"
   5         "crypto/tls"
   6         "fmt"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   8         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   9         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  10         . "gopkg.in/check.v1"
  11         "io"
  12         "io/ioutil"
  13         "log"
  14         "net/http"
  15         "net/url"
  16         "os"
  17         "strings"
  18         "testing"
  19         "time"
  20 )
  21
  22 // Gocheck boilerplate
  23 func Test(t *testing.T) {
  24         TestingT(t)
  25 }
  26
  27 // Gocheck boilerplate
  28 var _ = Suite(&ServerRequiredSuite{})
  29
  30 // Tests that require the Keep server running
  31 type ServerRequiredSuite struct{}
  32
  33 // Wait (up to 1 second) for keepproxy to listen on a port. This
  34 // avoids a race condition where we hit a "connection refused" error
  35 // because we start testing the proxy too soon.
  36 func waitForListener() {
  37         const (
  38                 ms = 5
  39         )
  40         for i := 0; listener == nil && i < 1000; i += ms {
  41                 time.Sleep(ms * time.Millisecond)
  42         }
  43         if listener == nil {
  44                 log.Fatalf("Timed out waiting for listener to start")
  45         }
  46 }
  47
  48 func closeListener() {
  49         if listener != nil {
  50                 listener.Close()
  51         }
  52 }
  53
  54 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  55         arvadostest.StartAPI()
  56         arvadostest.StartKeep()
  57 }
  58
  59 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  60         arvadostest.ResetEnv()
  61 }
  62
  63 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  64         arvadostest.StopKeep()
  65         arvadostest.StopAPI()
  66 }
  67
  68 func setupProxyService() {
  69
  70         client := &http.Client{Transport: &http.Transport{
  71                 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
  72
  73         var req *http.Request
  74         var err error
  75         if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil {
  76                 panic(err.Error())
  77         }
  78         req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN")))
  79
  80         reader, writer := io.Pipe()
  81
  82         req.Body = reader
  83
  84         go func() {
  85                 data := url.Values{}
  86                 data.Set("keep_service", `{
  87   "service_host": "localhost",
  88   "service_port": 29950,
  89   "service_ssl_flag": false,
  90   "service_type": "proxy"
  91 }`)
  92
  93                 writer.Write([]byte(data.Encode()))
  94                 writer.Close()
  95         }()
  96
  97         var resp *http.Response
  98         if resp, err = client.Do(req); err != nil {
  99                 panic(err.Error())
 100         }
 101         if resp.StatusCode != 200 {
 102                 panic(resp.Status)
 103         }
 104 }
 105
 106 func runProxy(c *C, args []string, port int, bogusClientToken bool) keepclient.KeepClient {
 107         if bogusClientToken {
 108                 os.Setenv("ARVADOS_API_TOKEN", "bogus-token")
 109         }
 110         arv, err := arvadosclient.MakeArvadosClient()
 111         c.Assert(err, Equals, nil)
 112         kc := keepclient.KeepClient{
 113                 Arvados:       &arv,
 114                 Want_replicas: 2,
 115                 Using_proxy:   true,
 116                 Client:        &http.Client{},
 117         }
 118         kc.SetServiceRoots(map[string]string{
 119                 "proxy": fmt.Sprintf("http://localhost:%v", port),
 120         }, nil)
 121         c.Check(kc.Using_proxy, Equals, true)
 122         c.Check(len(kc.LocalRoots()), Equals, 1)
 123         for _, root := range kc.LocalRoots() {
 124                 c.Check(root, Equals, fmt.Sprintf("http://localhost:%v", port))
 125         }
 126         log.Print("keepclient created")
 127         if bogusClientToken {
 128                 arvadostest.ResetEnv()
 129         }
 130
 131         {
 132                 os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
 133                 listener = nil
 134                 go main()
 135         }
 136
 137         return kc
 138 }
 139
 140 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 141         log.Print("TestPutAndGet start")
 142
 143         os.Args = []string{"keepproxy", "-listen=:29950"}
 144         listener = nil
 145         go main()
 146         time.Sleep(100 * time.Millisecond)
 147
 148         setupProxyService()
 149
 150         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
 151         arv, err := arvadosclient.MakeArvadosClient()
 152         c.Assert(err, Equals, nil)
 153         kc, err := keepclient.MakeKeepClient(&arv)
 154         c.Assert(err, Equals, nil)
 155         c.Check(kc.Arvados.External, Equals, true)
 156         c.Check(kc.Using_proxy, Equals, true)
 157         c.Check(len(kc.LocalRoots()), Equals, 1)
 158         for _, root := range kc.LocalRoots() {
 159                 c.Check(root, Equals, "http://localhost:29950")
 160         }
 161         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
 162
 163         waitForListener()
 164         defer closeListener()
 165
 166         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 167         var hash2 string
 168
 169         {
 170                 _, _, err := kc.Ask(hash)
 171                 c.Check(err, Equals, keepclient.BlockNotFound)
 172                 log.Print("Finished Ask (expected BlockNotFound)")
 173         }
 174
 175         {
 176                 reader, _, _, err := kc.Get(hash)
 177                 c.Check(reader, Equals, nil)
 178                 c.Check(err, Equals, keepclient.BlockNotFound)
 179                 log.Print("Finished Get (expected BlockNotFound)")
 180         }
 181
 182         // Note in bug #5309 among other errors keepproxy would set
 183         // Content-Length incorrectly on the 404 BlockNotFound response, this
 184         // would result in a protocol violation that would prevent reuse of the
 185         // connection, which would manifest by the next attempt to use the
 186         // connection (in this case the PutB below) failing.  So to test for
 187         // that bug it's necessary to trigger an error response (such as
 188         // BlockNotFound) and then do something else with the same httpClient
 189         // connection.
 190
 191         {
 192                 var rep int
 193                 var err error
 194                 hash2, rep, err = kc.PutB([]byte("foo"))
 195                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 196                 c.Check(rep, Equals, 2)
 197                 c.Check(err, Equals, nil)
 198                 log.Print("Finished PutB (expected success)")
 199         }
 200
 201         {
 202                 blocklen, _, err := kc.Ask(hash2)
 203                 c.Assert(err, Equals, nil)
 204                 c.Check(blocklen, Equals, int64(3))
 205                 log.Print("Finished Ask (expected success)")
 206         }
 207
 208         {
 209                 reader, blocklen, _, err := kc.Get(hash2)
 210                 c.Assert(err, Equals, nil)
 211                 all, err := ioutil.ReadAll(reader)
 212                 c.Check(all, DeepEquals, []byte("foo"))
 213                 c.Check(blocklen, Equals, int64(3))
 214                 log.Print("Finished Get (expected success)")
 215         }
 216
 217         {
 218                 var rep int
 219                 var err error
 220                 hash2, rep, err = kc.PutB([]byte(""))
 221                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 222                 c.Check(rep, Equals, 2)
 223                 c.Check(err, Equals, nil)
 224                 log.Print("Finished PutB zero block")
 225         }
 226
 227         {
 228                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 229                 c.Assert(err, Equals, nil)
 230                 all, err := ioutil.ReadAll(reader)
 231                 c.Check(all, DeepEquals, []byte(""))
 232                 c.Check(blocklen, Equals, int64(0))
 233                 log.Print("Finished Get zero block")
 234         }
 235
 236         log.Print("TestPutAndGet done")
 237 }
 238
 239 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 240         log.Print("TestPutAskGetForbidden start")
 241
 242         kc := runProxy(c, []string{"keepproxy"}, 29951, true)
 243         waitForListener()
 244         defer closeListener()
 245
 246         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 247
 248         {
 249                 _, _, err := kc.Ask(hash)
 250                 c.Check(err, Equals, keepclient.BlockNotFound)
 251                 log.Print("Ask 1")
 252         }
 253
 254         {
 255                 hash2, rep, err := kc.PutB([]byte("bar"))
 256                 c.Check(hash2, Equals, "")
 257                 c.Check(rep, Equals, 0)
 258                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 259                 log.Print("PutB")
 260         }
 261
 262         {
 263                 blocklen, _, err := kc.Ask(hash)
 264                 c.Assert(err, Equals, keepclient.BlockNotFound)
 265                 c.Check(blocklen, Equals, int64(0))
 266                 log.Print("Ask 2")
 267         }
 268
 269         {
 270                 _, blocklen, _, err := kc.Get(hash)
 271                 c.Assert(err, Equals, keepclient.BlockNotFound)
 272                 c.Check(blocklen, Equals, int64(0))
 273                 log.Print("Get")
 274         }
 275
 276         log.Print("TestPutAskGetForbidden done")
 277 }
 278
 279 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 280         log.Print("TestGetDisabled start")
 281
 282         kc := runProxy(c, []string{"keepproxy", "-no-get"}, 29952, false)
 283         waitForListener()
 284         defer closeListener()
 285
 286         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 287
 288         {
 289                 _, _, err := kc.Ask(hash)
 290                 c.Check(err, Equals, keepclient.BlockNotFound)
 291                 log.Print("Ask 1")
 292         }
 293
 294         {
 295                 hash2, rep, err := kc.PutB([]byte("baz"))
 296                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 297                 c.Check(rep, Equals, 2)
 298                 c.Check(err, Equals, nil)
 299                 log.Print("PutB")
 300         }
 301
 302         {
 303                 blocklen, _, err := kc.Ask(hash)
 304                 c.Assert(err, Equals, keepclient.BlockNotFound)
 305                 c.Check(blocklen, Equals, int64(0))
 306                 log.Print("Ask 2")
 307         }
 308
 309         {
 310                 _, blocklen, _, err := kc.Get(hash)
 311                 c.Assert(err, Equals, keepclient.BlockNotFound)
 312                 c.Check(blocklen, Equals, int64(0))
 313                 log.Print("Get")
 314         }
 315
 316         log.Print("TestGetDisabled done")
 317 }
 318
 319 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 320         log.Print("TestPutDisabled start")
 321
 322         kc := runProxy(c, []string{"keepproxy", "-no-put"}, 29953, false)
 323         waitForListener()
 324         defer closeListener()
 325
 326         {
 327                 hash2, rep, err := kc.PutB([]byte("quux"))
 328                 c.Check(hash2, Equals, "")
 329                 c.Check(rep, Equals, 0)
 330                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 331                 log.Print("PutB")
 332         }
 333
 334         log.Print("TestPutDisabled done")
 335 }
 336
 337 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 338         runProxy(c, []string{"keepproxy"}, 29954, false)
 339         waitForListener()
 340         defer closeListener()
 341
 342         {
 343                 client := http.Client{}
 344                 req, err := http.NewRequest("OPTIONS",
 345                         fmt.Sprintf("http://localhost:29954/%x+3",
 346                                 md5.Sum([]byte("foo"))),
 347                         nil)
 348                 req.Header.Add("Access-Control-Request-Method", "PUT")
 349                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 350                 resp, err := client.Do(req)
 351                 c.Check(err, Equals, nil)
 352                 c.Check(resp.StatusCode, Equals, 200)
 353                 body, err := ioutil.ReadAll(resp.Body)
 354                 c.Check(string(body), Equals, "")
 355                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 356                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 357         }
 358
 359         {
 360                 resp, err := http.Get(
 361                         fmt.Sprintf("http://localhost:29954/%x+3",
 362                                 md5.Sum([]byte("foo"))))
 363                 c.Check(err, Equals, nil)
 364                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 365                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
 366         }
 367 }
 368
 369 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
 370         runProxy(c, []string{"keepproxy"}, 29955, false)
 371         waitForListener()
 372         defer closeListener()
 373
 374         {
 375                 client := http.Client{}
 376                 req, err := http.NewRequest("POST",
 377                         "http://localhost:29955/",
 378                         strings.NewReader("qux"))
 379                 req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 380                 req.Header.Add("Content-Type", "application/octet-stream")
 381                 resp, err := client.Do(req)
 382                 c.Check(err, Equals, nil)
 383                 body, err := ioutil.ReadAll(resp.Body)
 384                 c.Check(err, Equals, nil)
 385                 c.Check(string(body), Equals,
 386                         fmt.Sprintf("%x+%d", md5.Sum([]byte("qux")), 3))
 387         }
 388 }