services/keep-web/handler_test.go

   1 package main
   2
   3 import (
   4         "html"
   5         "io/ioutil"
   6         "net/http"
   7         "net/http/httptest"
   8         "net/url"
   9         "regexp"
  10         "strings"
  11
  12         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  13         "git.curoverse.com/arvados.git/sdk/go/auth"
  14         check "gopkg.in/check.v1"
  15 )
  16
  17 var _ = check.Suite(&UnitSuite{})
  18
  19 type UnitSuite struct{}
  20
  21 func mustParseURL(s string) *url.URL {
  22         r, err := url.Parse(s)
  23         if err != nil {
  24                 panic("parse URL: " + s)
  25         }
  26         return r
  27 }
  28
  29 func (s *IntegrationSuite) TestVhost404(c *check.C) {
  30         for _, testURL := range []string{
  31                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
  32                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  33         } {
  34                 resp := httptest.NewRecorder()
  35                 u := mustParseURL(testURL)
  36                 req := &http.Request{
  37                         Method:     "GET",
  38                         URL:        u,
  39                         RequestURI: u.RequestURI(),
  40                 }
  41                 s.testServer.Handler.ServeHTTP(resp, req)
  42                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
  43                 c.Check(resp.Body.String(), check.Equals, "")
  44         }
  45 }
  46
  47 // An authorizer modifies an HTTP request to make use of the given
  48 // token -- by adding it to a header, cookie, query param, or whatever
  49 // -- and returns the HTTP status code we should expect from keep-web if
  50 // the token is invalid.
  51 type authorizer func(*http.Request, string) int
  52
  53 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
  54         s.doVhostRequests(c, authzViaAuthzHeader)
  55 }
  56 func authzViaAuthzHeader(r *http.Request, tok string) int {
  57         r.Header.Add("Authorization", "OAuth2 "+tok)
  58         return http.StatusUnauthorized
  59 }
  60
  61 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
  62         s.doVhostRequests(c, authzViaCookieValue)
  63 }
  64 func authzViaCookieValue(r *http.Request, tok string) int {
  65         r.AddCookie(&http.Cookie{
  66                 Name:  "arvados_api_token",
  67                 Value: auth.EncodeTokenCookie([]byte(tok)),
  68         })
  69         return http.StatusUnauthorized
  70 }
  71
  72 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
  73         s.doVhostRequests(c, authzViaPath)
  74 }
  75 func authzViaPath(r *http.Request, tok string) int {
  76         r.URL.Path = "/t=" + tok + r.URL.Path
  77         return http.StatusNotFound
  78 }
  79
  80 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
  81         s.doVhostRequests(c, authzViaQueryString)
  82 }
  83 func authzViaQueryString(r *http.Request, tok string) int {
  84         r.URL.RawQuery = "api_token=" + tok
  85         return http.StatusUnauthorized
  86 }
  87
  88 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
  89         s.doVhostRequests(c, authzViaPOST)
  90 }
  91 func authzViaPOST(r *http.Request, tok string) int {
  92         r.Method = "POST"
  93         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
  94         r.Body = ioutil.NopCloser(strings.NewReader(
  95                 url.Values{"api_token": {tok}}.Encode()))
  96         return http.StatusUnauthorized
  97 }
  98
  99 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 100         s.doVhostRequests(c, authzViaPOST)
 101 }
 102 func authzViaXHRPOST(r *http.Request, tok string) int {
 103         r.Method = "POST"
 104         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 105         r.Header.Add("Origin", "https://origin.example")
 106         r.Body = ioutil.NopCloser(strings.NewReader(
 107                 url.Values{
 108                         "api_token":   {tok},
 109                         "disposition": {"attachment"},
 110                 }.Encode()))
 111         return http.StatusUnauthorized
 112 }
 113
 114 // Try some combinations of {url, token} using the given authorization
 115 // mechanism, and verify the result is correct.
 116 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 117         for _, hostPath := range []string{
 118                 arvadostest.FooCollection + ".example.com/foo",
 119                 arvadostest.FooCollection + "--collections.example.com/foo",
 120                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 121                 arvadostest.FooPdh + ".example.com/foo",
 122                 strings.Replace(arvadostest.FooPdh, "+", "-", -1) + "--collections.example.com/foo",
 123                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 124         } {
 125                 c.Log("doRequests: ", hostPath)
 126                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 127         }
 128 }
 129
 130 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 131         for _, tok := range []string{
 132                 arvadostest.ActiveToken,
 133                 arvadostest.ActiveToken[:15],
 134                 arvadostest.SpectatorToken,
 135                 "bogus",
 136                 "",
 137         } {
 138                 u := mustParseURL("http://" + hostPath)
 139                 req := &http.Request{
 140                         Method:     "GET",
 141                         Host:       u.Host,
 142                         URL:        u,
 143                         RequestURI: u.RequestURI(),
 144                         Header:     http.Header{},
 145                 }
 146                 failCode := authz(req, tok)
 147                 req, resp := s.doReq(req)
 148                 code, body := resp.Code, resp.Body.String()
 149
 150                 // If the initial request had a (non-empty) token
 151                 // showing in the query string, we should have been
 152                 // redirected in order to hide it in a cookie.
 153                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 154
 155                 if tok == arvadostest.ActiveToken {
 156                         c.Check(code, check.Equals, http.StatusOK)
 157                         c.Check(body, check.Equals, "foo")
 158
 159                 } else {
 160                         c.Check(code >= 400, check.Equals, true)
 161                         c.Check(code < 500, check.Equals, true)
 162                         if tok == arvadostest.SpectatorToken {
 163                                 // Valid token never offers to retry
 164                                 // with different credentials.
 165                                 c.Check(code, check.Equals, http.StatusNotFound)
 166                         } else {
 167                                 // Invalid token can ask to retry
 168                                 // depending on the authz method.
 169                                 c.Check(code, check.Equals, failCode)
 170                         }
 171                         c.Check(body, check.Equals, "")
 172                 }
 173         }
 174 }
 175
 176 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 177         resp := httptest.NewRecorder()
 178         s.testServer.Handler.ServeHTTP(resp, req)
 179         if resp.Code != http.StatusSeeOther {
 180                 return req, resp
 181         }
 182         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 183         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 184         req = &http.Request{
 185                 Method:     "GET",
 186                 Host:       u.Host,
 187                 URL:        u,
 188                 RequestURI: u.RequestURI(),
 189                 Header:     http.Header{},
 190         }
 191         for _, c := range cookies {
 192                 req.AddCookie(c)
 193         }
 194         return s.doReq(req)
 195 }
 196
 197 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 198         s.testVhostRedirectTokenToCookie(c, "GET",
 199                 arvadostest.FooCollection+".example.com/foo",
 200                 "?api_token="+arvadostest.ActiveToken,
 201                 "",
 202                 "",
 203                 http.StatusOK,
 204                 "foo",
 205         )
 206 }
 207
 208 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 209         s.testVhostRedirectTokenToCookie(c, "GET",
 210                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 211                 "",
 212                 "",
 213                 "",
 214                 http.StatusOK,
 215                 "foo",
 216         )
 217 }
 218
 219 // Bad token in URL is 404 Not Found because it doesn't make sense to
 220 // retry the same URL with different authorization.
 221 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 222         s.testVhostRedirectTokenToCookie(c, "GET",
 223                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 224                 "",
 225                 "",
 226                 "",
 227                 http.StatusNotFound,
 228                 "",
 229         )
 230 }
 231
 232 // Bad token in a cookie (even if it got there via our own
 233 // query-string-to-cookie redirect) is, in principle, retryable at the
 234 // same URL so it's 401 Unauthorized.
 235 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 236         s.testVhostRedirectTokenToCookie(c, "GET",
 237                 arvadostest.FooCollection+".example.com/foo",
 238                 "?api_token=thisisabogustoken",
 239                 "",
 240                 "",
 241                 http.StatusUnauthorized,
 242                 "",
 243         )
 244 }
 245
 246 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 247         s.testVhostRedirectTokenToCookie(c, "GET",
 248                 "example.com/c="+arvadostest.FooCollection+"/foo",
 249                 "?api_token="+arvadostest.ActiveToken,
 250                 "",
 251                 "",
 252                 http.StatusBadRequest,
 253                 "",
 254         )
 255 }
 256
 257 // If client requests an attachment by putting ?disposition=attachment
 258 // in the query string, and gets redirected, the redirect target
 259 // should respond with an attachment.
 260 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 261         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 262                 arvadostest.FooCollection+".example.com/foo",
 263                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 264                 "",
 265                 "",
 266                 http.StatusOK,
 267                 "foo",
 268         )
 269         c.Check(strings.Split(resp.Header().Get("Content-Disposition"), ";")[0], check.Equals, "attachment")
 270 }
 271
 272 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 273         s.testServer.Config.TrustAllContent = true
 274         s.testVhostRedirectTokenToCookie(c, "GET",
 275                 "example.com/c="+arvadostest.FooCollection+"/foo",
 276                 "?api_token="+arvadostest.ActiveToken,
 277                 "",
 278                 "",
 279                 http.StatusOK,
 280                 "foo",
 281         )
 282 }
 283
 284 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 285         s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
 286
 287         s.testVhostRedirectTokenToCookie(c, "GET",
 288                 "example.com/c="+arvadostest.FooCollection+"/foo",
 289                 "?api_token="+arvadostest.ActiveToken,
 290                 "",
 291                 "",
 292                 http.StatusBadRequest,
 293                 "",
 294         )
 295
 296         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 297                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 298                 "?api_token="+arvadostest.ActiveToken,
 299                 "",
 300                 "",
 301                 http.StatusOK,
 302                 "foo",
 303         )
 304         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 305 }
 306
 307 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 308         s.testVhostRedirectTokenToCookie(c, "POST",
 309                 arvadostest.FooCollection+".example.com/foo",
 310                 "",
 311                 "application/x-www-form-urlencoded",
 312                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 313                 http.StatusOK,
 314                 "foo",
 315         )
 316 }
 317
 318 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 319         s.testVhostRedirectTokenToCookie(c, "POST",
 320                 arvadostest.FooCollection+".example.com/foo",
 321                 "",
 322                 "application/x-www-form-urlencoded",
 323                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 324                 http.StatusNotFound,
 325                 "",
 326         )
 327 }
 328
 329 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 330         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
 331         s.testVhostRedirectTokenToCookie(c, "GET",
 332                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 333                 "",
 334                 "",
 335                 "",
 336                 http.StatusOK,
 337                 "Hello world\n",
 338         )
 339 }
 340
 341 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 342         s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
 343         s.testVhostRedirectTokenToCookie(c, "GET",
 344                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 345                 "",
 346                 "",
 347                 "",
 348                 http.StatusNotFound,
 349                 "",
 350         )
 351 }
 352
 353 func (s *IntegrationSuite) TestRange(c *check.C) {
 354         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
 355         u, _ := url.Parse("http://example.com/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt")
 356         req := &http.Request{
 357                 Method:     "GET",
 358                 Host:       u.Host,
 359                 URL:        u,
 360                 RequestURI: u.RequestURI(),
 361                 Header:     http.Header{"Range": {"bytes=0-4"}},
 362         }
 363         resp := httptest.NewRecorder()
 364         s.testServer.Handler.ServeHTTP(resp, req)
 365         c.Check(resp.Code, check.Equals, http.StatusPartialContent)
 366         c.Check(resp.Body.String(), check.Equals, "Hello")
 367         c.Check(resp.Header().Get("Content-Length"), check.Equals, "5")
 368         c.Check(resp.Header().Get("Content-Range"), check.Equals, "bytes 0-4/12")
 369
 370         req.Header.Set("Range", "bytes=0-")
 371         resp = httptest.NewRecorder()
 372         s.testServer.Handler.ServeHTTP(resp, req)
 373         // 200 and 206 are both correct:
 374         c.Check(resp.Code, check.Equals, http.StatusOK)
 375         c.Check(resp.Body.String(), check.Equals, "Hello world\n")
 376         c.Check(resp.Header().Get("Content-Length"), check.Equals, "12")
 377
 378         // Unsupported ranges are ignored
 379         for _, hdr := range []string{
 380                 "bytes=5-5",  // non-zero start byte
 381                 "bytes=-5",   // last 5 bytes
 382                 "cubits=0-5", // unsupported unit
 383                 "bytes=0-340282366920938463463374607431768211456", // 2^128
 384         } {
 385                 req.Header.Set("Range", hdr)
 386                 resp = httptest.NewRecorder()
 387                 s.testServer.Handler.ServeHTTP(resp, req)
 388                 c.Check(resp.Code, check.Equals, http.StatusOK)
 389                 c.Check(resp.Body.String(), check.Equals, "Hello world\n")
 390                 c.Check(resp.Header().Get("Content-Length"), check.Equals, "12")
 391                 c.Check(resp.Header().Get("Content-Range"), check.Equals, "")
 392                 c.Check(resp.Header().Get("Accept-Ranges"), check.Equals, "bytes")
 393         }
 394 }
 395
 396 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 397 // and disposition=attachment (telling us it's acceptable to respond
 398 // with content instead of a redirect) and an Origin header that gets
 399 // added automatically by the browser (telling us it's desirable to do
 400 // so).
 401 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 402         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 403         req := &http.Request{
 404                 Method:     "POST",
 405                 Host:       u.Host,
 406                 URL:        u,
 407                 RequestURI: u.RequestURI(),
 408                 Header: http.Header{
 409                         "Origin":       {"https://origin.example"},
 410                         "Content-Type": {"application/x-www-form-urlencoded"},
 411                 },
 412                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 413                         "api_token":   {arvadostest.ActiveToken},
 414                         "disposition": {"attachment"},
 415                 }.Encode())),
 416         }
 417         resp := httptest.NewRecorder()
 418         s.testServer.Handler.ServeHTTP(resp, req)
 419         c.Check(resp.Code, check.Equals, http.StatusOK)
 420         c.Check(resp.Body.String(), check.Equals, "foo")
 421         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 422 }
 423
 424 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 425         u, _ := url.Parse(`http://` + hostPath + queryString)
 426         req := &http.Request{
 427                 Method:     method,
 428                 Host:       u.Host,
 429                 URL:        u,
 430                 RequestURI: u.RequestURI(),
 431                 Header:     http.Header{"Content-Type": {contentType}},
 432                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 433         }
 434
 435         resp := httptest.NewRecorder()
 436         defer func() {
 437                 c.Check(resp.Code, check.Equals, expectStatus)
 438                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 439         }()
 440
 441         s.testServer.Handler.ServeHTTP(resp, req)
 442         if resp.Code != http.StatusSeeOther {
 443                 return resp
 444         }
 445         c.Check(resp.Body.String(), check.Matches, `.*href="//`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 446         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 447
 448         u, _ = u.Parse(resp.Header().Get("Location"))
 449         req = &http.Request{
 450                 Method:     "GET",
 451                 Host:       u.Host,
 452                 URL:        u,
 453                 RequestURI: u.RequestURI(),
 454                 Header:     http.Header{},
 455         }
 456         for _, c := range cookies {
 457                 req.AddCookie(c)
 458         }
 459
 460         resp = httptest.NewRecorder()
 461         s.testServer.Handler.ServeHTTP(resp, req)
 462         c.Check(resp.Header().Get("Location"), check.Equals, "")
 463         return resp
 464 }