services/keep-web/handler_test.go

   1 package main
   2
   3 import (
   4         "html"
   5         "io/ioutil"
   6         "net/http"
   7         "net/http/httptest"
   8         "net/url"
   9         "regexp"
  10         "strings"
  11
  12         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  13         "git.curoverse.com/arvados.git/sdk/go/auth"
  14         check "gopkg.in/check.v1"
  15 )
  16
  17 var _ = check.Suite(&UnitSuite{})
  18
  19 type UnitSuite struct{}
  20
  21 func mustParseURL(s string) *url.URL {
  22         r, err := url.Parse(s)
  23         if err != nil {
  24                 panic("parse URL: " + s)
  25         }
  26         return r
  27 }
  28
  29 func (s *IntegrationSuite) TestVhost404(c *check.C) {
  30         for _, testURL := range []string{
  31                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
  32                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  33         } {
  34                 resp := httptest.NewRecorder()
  35                 u := mustParseURL(testURL)
  36                 req := &http.Request{
  37                         Method:     "GET",
  38                         URL:        u,
  39                         RequestURI: u.RequestURI(),
  40                 }
  41                 (&handler{}).ServeHTTP(resp, req)
  42                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
  43                 c.Check(resp.Body.String(), check.Equals, "")
  44         }
  45 }
  46
  47 // An authorizer modifies an HTTP request to make use of the given
  48 // token -- by adding it to a header, cookie, query param, or whatever
  49 // -- and returns the HTTP status code we should expect from keep-web if
  50 // the token is invalid.
  51 type authorizer func(*http.Request, string) int
  52
  53 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
  54         doVhostRequests(c, authzViaAuthzHeader)
  55 }
  56 func authzViaAuthzHeader(r *http.Request, tok string) int {
  57         r.Header.Add("Authorization", "OAuth2 "+tok)
  58         return http.StatusUnauthorized
  59 }
  60
  61 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
  62         doVhostRequests(c, authzViaCookieValue)
  63 }
  64 func authzViaCookieValue(r *http.Request, tok string) int {
  65         r.AddCookie(&http.Cookie{
  66                 Name:  "arvados_api_token",
  67                 Value: auth.EncodeTokenCookie([]byte(tok)),
  68         })
  69         return http.StatusUnauthorized
  70 }
  71
  72 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
  73         doVhostRequests(c, authzViaPath)
  74 }
  75 func authzViaPath(r *http.Request, tok string) int {
  76         r.URL.Path = "/t=" + tok + r.URL.Path
  77         return http.StatusNotFound
  78 }
  79
  80 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
  81         doVhostRequests(c, authzViaQueryString)
  82 }
  83 func authzViaQueryString(r *http.Request, tok string) int {
  84         r.URL.RawQuery = "api_token=" + tok
  85         return http.StatusUnauthorized
  86 }
  87
  88 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
  89         doVhostRequests(c, authzViaPOST)
  90 }
  91 func authzViaPOST(r *http.Request, tok string) int {
  92         r.Method = "POST"
  93         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
  94         r.Body = ioutil.NopCloser(strings.NewReader(
  95                 url.Values{"api_token": {tok}}.Encode()))
  96         return http.StatusUnauthorized
  97 }
  98
  99 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 100         doVhostRequests(c, authzViaPOST)
 101 }
 102 func authzViaXHRPOST(r *http.Request, tok string) int {
 103         r.Method = "POST"
 104         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 105         r.Header.Add("Origin", "https://origin.example")
 106         r.Body = ioutil.NopCloser(strings.NewReader(
 107                 url.Values{
 108                         "api_token":   {tok},
 109                         "disposition": {"attachment"},
 110                 }.Encode()))
 111         return http.StatusUnauthorized
 112 }
 113
 114 // Try some combinations of {url, token} using the given authorization
 115 // mechanism, and verify the result is correct.
 116 func doVhostRequests(c *check.C, authz authorizer) {
 117         for _, hostPath := range []string{
 118                 arvadostest.FooCollection + ".example.com/foo",
 119                 arvadostest.FooCollection + "--collections.example.com/foo",
 120                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 121                 arvadostest.FooPdh + ".example.com/foo",
 122                 strings.Replace(arvadostest.FooPdh, "+", "-", -1) + "--collections.example.com/foo",
 123                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 124         } {
 125                 c.Log("doRequests: ", hostPath)
 126                 doVhostRequestsWithHostPath(c, authz, hostPath)
 127         }
 128 }
 129
 130 func doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 131         for _, tok := range []string{
 132                 arvadostest.ActiveToken,
 133                 arvadostest.ActiveToken[:15],
 134                 arvadostest.SpectatorToken,
 135                 "bogus",
 136                 "",
 137         } {
 138                 u := mustParseURL("http://" + hostPath)
 139                 req := &http.Request{
 140                         Method:     "GET",
 141                         Host:       u.Host,
 142                         URL:        u,
 143                         RequestURI: u.RequestURI(),
 144                         Header:     http.Header{},
 145                 }
 146                 failCode := authz(req, tok)
 147                 req, resp := doReq(req)
 148                 code, body := resp.Code, resp.Body.String()
 149
 150                 // If the initial request had a (non-empty) token
 151                 // showing in the query string, we should have been
 152                 // redirected in order to hide it in a cookie.
 153                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 154
 155                 if tok == arvadostest.ActiveToken {
 156                         c.Check(code, check.Equals, http.StatusOK)
 157                         c.Check(body, check.Equals, "foo")
 158
 159                 } else {
 160                         c.Check(code >= 400, check.Equals, true)
 161                         c.Check(code < 500, check.Equals, true)
 162                         if tok == arvadostest.SpectatorToken {
 163                                 // Valid token never offers to retry
 164                                 // with different credentials.
 165                                 c.Check(code, check.Equals, http.StatusNotFound)
 166                         } else {
 167                                 // Invalid token can ask to retry
 168                                 // depending on the authz method.
 169                                 c.Check(code, check.Equals, failCode)
 170                         }
 171                         c.Check(body, check.Equals, "")
 172                 }
 173         }
 174 }
 175
 176 func doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 177         resp := httptest.NewRecorder()
 178         (&handler{}).ServeHTTP(resp, req)
 179         if resp.Code != http.StatusSeeOther {
 180                 return req, resp
 181         }
 182         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 183         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 184         req = &http.Request{
 185                 Method:     "GET",
 186                 Host:       u.Host,
 187                 URL:        u,
 188                 RequestURI: u.RequestURI(),
 189                 Header:     http.Header{},
 190         }
 191         for _, c := range cookies {
 192                 req.AddCookie(c)
 193         }
 194         return doReq(req)
 195 }
 196
 197 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 198         s.testVhostRedirectTokenToCookie(c, "GET",
 199                 arvadostest.FooCollection+".example.com/foo",
 200                 "?api_token="+arvadostest.ActiveToken,
 201                 "",
 202                 "",
 203                 http.StatusOK,
 204                 "foo",
 205         )
 206 }
 207
 208 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 209         s.testVhostRedirectTokenToCookie(c, "GET",
 210                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 211                 "",
 212                 "",
 213                 "",
 214                 http.StatusOK,
 215                 "foo",
 216         )
 217 }
 218
 219 // Bad token in URL is 404 Not Found because it doesn't make sense to
 220 // retry the same URL with different authorization.
 221 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 222         s.testVhostRedirectTokenToCookie(c, "GET",
 223                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 224                 "",
 225                 "",
 226                 "",
 227                 http.StatusNotFound,
 228                 "",
 229         )
 230 }
 231
 232 // Bad token in a cookie (even if it got there via our own
 233 // query-string-to-cookie redirect) is, in principle, retryable at the
 234 // same URL so it's 401 Unauthorized.
 235 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 236         s.testVhostRedirectTokenToCookie(c, "GET",
 237                 arvadostest.FooCollection+".example.com/foo",
 238                 "?api_token=thisisabogustoken",
 239                 "",
 240                 "",
 241                 http.StatusUnauthorized,
 242                 "",
 243         )
 244 }
 245
 246 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 247         s.testVhostRedirectTokenToCookie(c, "GET",
 248                 "example.com/c="+arvadostest.FooCollection+"/foo",
 249                 "?api_token="+arvadostest.ActiveToken,
 250                 "",
 251                 "",
 252                 http.StatusBadRequest,
 253                 "",
 254         )
 255 }
 256
 257 // If client requests an attachment by putting ?disposition=attachment
 258 // in the query string, and gets redirected, the redirect target
 259 // should respond with an attachment.
 260 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 261         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 262                 arvadostest.FooCollection+".example.com/foo",
 263                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 264                 "",
 265                 "",
 266                 http.StatusOK,
 267                 "foo",
 268         )
 269         c.Check(strings.Split(resp.Header().Get("Content-Disposition"), ";")[0], check.Equals, "attachment")
 270 }
 271
 272 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 273         defer func(orig bool) {
 274                 trustAllContent = orig
 275         }(trustAllContent)
 276         trustAllContent = true
 277         s.testVhostRedirectTokenToCookie(c, "GET",
 278                 "example.com/c="+arvadostest.FooCollection+"/foo",
 279                 "?api_token="+arvadostest.ActiveToken,
 280                 "",
 281                 "",
 282                 http.StatusOK,
 283                 "foo",
 284         )
 285 }
 286
 287 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 288         defer func(orig string) {
 289                 attachmentOnlyHost = orig
 290         }(attachmentOnlyHost)
 291         attachmentOnlyHost = "example.com:1234"
 292
 293         s.testVhostRedirectTokenToCookie(c, "GET",
 294                 "example.com/c="+arvadostest.FooCollection+"/foo",
 295                 "?api_token="+arvadostest.ActiveToken,
 296                 "",
 297                 "",
 298                 http.StatusBadRequest,
 299                 "",
 300         )
 301
 302         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 303                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 304                 "?api_token="+arvadostest.ActiveToken,
 305                 "",
 306                 "",
 307                 http.StatusOK,
 308                 "foo",
 309         )
 310         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 311 }
 312
 313 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 314         s.testVhostRedirectTokenToCookie(c, "POST",
 315                 arvadostest.FooCollection+".example.com/foo",
 316                 "",
 317                 "application/x-www-form-urlencoded",
 318                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 319                 http.StatusOK,
 320                 "foo",
 321         )
 322 }
 323
 324 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 325         s.testVhostRedirectTokenToCookie(c, "POST",
 326                 arvadostest.FooCollection+".example.com/foo",
 327                 "",
 328                 "application/x-www-form-urlencoded",
 329                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 330                 http.StatusNotFound,
 331                 "",
 332         )
 333 }
 334
 335 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 336         anonymousTokens = []string{arvadostest.AnonymousToken}
 337         s.testVhostRedirectTokenToCookie(c, "GET",
 338                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 339                 "",
 340                 "",
 341                 "",
 342                 http.StatusOK,
 343                 "Hello world\n",
 344         )
 345 }
 346
 347 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 348         anonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
 349         s.testVhostRedirectTokenToCookie(c, "GET",
 350                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 351                 "",
 352                 "",
 353                 "",
 354                 http.StatusNotFound,
 355                 "",
 356         )
 357 }
 358
 359 func (s *IntegrationSuite) TestRange(c *check.C) {
 360         u, _ := url.Parse("http://example.com/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt")
 361         req := &http.Request{
 362                 Method:     "GET",
 363                 Host:       u.Host,
 364                 URL:        u,
 365                 RequestURI: u.RequestURI(),
 366                 Header:     http.Header{"Range": {"bytes=0-4"}},
 367         }
 368         resp := httptest.NewRecorder()
 369         (&handler{}).ServeHTTP(resp, req)
 370         c.Check(resp.Code, check.Equals, http.StatusPartialContent)
 371         c.Check(resp.Body.String(), check.Equals, "Hello")
 372         c.Check(resp.Header().Get("Content-Length"), check.Equals, "5")
 373         c.Check(resp.Header().Get("Content-Range"), check.Equals, "bytes 0-4/12")
 374
 375         req.Header.Set("Range", "bytes=0-")
 376         resp = httptest.NewRecorder()
 377         (&handler{}).ServeHTTP(resp, req)
 378         // 200 and 206 are both correct:
 379         c.Check(resp.Code, check.Equals, http.StatusOK)
 380         c.Check(resp.Body.String(), check.Equals, "Hello world\n")
 381         c.Check(resp.Header().Get("Content-Length"), check.Equals, "12")
 382
 383         // Unsupported ranges are ignored
 384         for _, hdr := range []string{
 385                 "bytes=5-5",  // non-zero start byte
 386                 "bytes=-5",   // last 5 bytes
 387                 "cubits=0-5", // unsupported unit
 388                 "bytes=0-340282366920938463463374607431768211456", // 2^128
 389         } {
 390                 req.Header.Set("Range", hdr)
 391                 resp = httptest.NewRecorder()
 392                 (&handler{}).ServeHTTP(resp, req)
 393                 c.Check(resp.Code, check.Equals, http.StatusOK)
 394                 c.Check(resp.Body.String(), check.Equals, "Hello world\n")
 395                 c.Check(resp.Header().Get("Content-Length"), check.Equals, "12")
 396                 c.Check(resp.Header().Get("Content-Range"), check.Equals, "")
 397                 c.Check(resp.Header().Get("Accept-Ranges"), check.Equals, "bytes")
 398         }
 399 }
 400
 401 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 402 // and disposition=attachment (telling us it's acceptable to respond
 403 // with content instead of a redirect) and an Origin header that gets
 404 // added automatically by the browser (telling us it's desirable to do
 405 // so).
 406 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 407         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 408         req := &http.Request{
 409                 Method:     "POST",
 410                 Host:       u.Host,
 411                 URL:        u,
 412                 RequestURI: u.RequestURI(),
 413                 Header: http.Header{
 414                         "Origin":       {"https://origin.example"},
 415                         "Content-Type": {"application/x-www-form-urlencoded"},
 416                 },
 417                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 418                         "api_token":   {arvadostest.ActiveToken},
 419                         "disposition": {"attachment"},
 420                 }.Encode())),
 421         }
 422         resp := httptest.NewRecorder()
 423         (&handler{}).ServeHTTP(resp, req)
 424         c.Check(resp.Code, check.Equals, http.StatusOK)
 425         c.Check(resp.Body.String(), check.Equals, "foo")
 426         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 427 }
 428
 429 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 430         u, _ := url.Parse(`http://` + hostPath + queryString)
 431         req := &http.Request{
 432                 Method:     method,
 433                 Host:       u.Host,
 434                 URL:        u,
 435                 RequestURI: u.RequestURI(),
 436                 Header:     http.Header{"Content-Type": {contentType}},
 437                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 438         }
 439
 440         resp := httptest.NewRecorder()
 441         defer func() {
 442                 c.Check(resp.Code, check.Equals, expectStatus)
 443                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 444         }()
 445
 446         (&handler{}).ServeHTTP(resp, req)
 447         if resp.Code != http.StatusSeeOther {
 448                 return resp
 449         }
 450         c.Check(resp.Body.String(), check.Matches, `.*href="//`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 451         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 452
 453         u, _ = u.Parse(resp.Header().Get("Location"))
 454         req = &http.Request{
 455                 Method:     "GET",
 456                 Host:       u.Host,
 457                 URL:        u,
 458                 RequestURI: u.RequestURI(),
 459                 Header:     http.Header{},
 460         }
 461         for _, c := range cookies {
 462                 req.AddCookie(c)
 463         }
 464
 465         resp = httptest.NewRecorder()
 466         (&handler{}).ServeHTTP(resp, req)
 467         c.Check(resp.Header().Get("Location"), check.Equals, "")
 468         return resp
 469 }