8784: Fix test for latest firefox.
[arvados.git] / services / keep-web / handler_test.go
1 package main
2
3 import (
4         "fmt"
5         "html"
6         "io/ioutil"
7         "net/http"
8         "net/http/httptest"
9         "net/url"
10         "regexp"
11         "strings"
12
13         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
14         "git.curoverse.com/arvados.git/sdk/go/auth"
15         check "gopkg.in/check.v1"
16 )
17
18 var _ = check.Suite(&UnitSuite{})
19
20 type UnitSuite struct{}
21
22 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
23         h := handler{Config: DefaultConfig()}
24         u, _ := url.Parse("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
25         req := &http.Request{
26                 Method:     "OPTIONS",
27                 Host:       u.Host,
28                 URL:        u,
29                 RequestURI: u.RequestURI(),
30                 Header: http.Header{
31                         "Origin":                        {"https://workbench.example"},
32                         "Access-Control-Request-Method": {"POST"},
33                 },
34         }
35
36         // Check preflight for an allowed request
37         resp := httptest.NewRecorder()
38         h.ServeHTTP(resp, req)
39         c.Check(resp.Code, check.Equals, http.StatusOK)
40         c.Check(resp.Body.String(), check.Equals, "")
41         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
42         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "GET, POST")
43         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Range")
44
45         // Check preflight for a disallowed request
46         resp = httptest.NewRecorder()
47         req.Header.Set("Access-Control-Request-Method", "DELETE")
48         h.ServeHTTP(resp, req)
49         c.Check(resp.Body.String(), check.Equals, "")
50         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
51 }
52
53 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
54         bogusID := strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "-"
55         token := arvadostest.ActiveToken
56         for _, trial := range []string{
57                 "http://keep-web/c=" + bogusID + "/foo",
58                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
59                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
60                 "http://keep-web/collections/" + bogusID + "/foo",
61                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
62                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
63         } {
64                 c.Log(trial)
65                 u, err := url.Parse(trial)
66                 c.Assert(err, check.IsNil)
67                 req := &http.Request{
68                         Method:     "GET",
69                         Host:       u.Host,
70                         URL:        u,
71                         RequestURI: u.RequestURI(),
72                 }
73                 resp := httptest.NewRecorder()
74                 cfg := DefaultConfig()
75                 cfg.AnonymousTokens = []string{arvadostest.AnonymousToken}
76                 h := handler{Config: cfg}
77                 h.ServeHTTP(resp, req)
78                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
79         }
80 }
81
82 func mustParseURL(s string) *url.URL {
83         r, err := url.Parse(s)
84         if err != nil {
85                 panic("parse URL: " + s)
86         }
87         return r
88 }
89
90 func (s *IntegrationSuite) TestVhost404(c *check.C) {
91         for _, testURL := range []string{
92                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
93                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
94         } {
95                 resp := httptest.NewRecorder()
96                 u := mustParseURL(testURL)
97                 req := &http.Request{
98                         Method:     "GET",
99                         URL:        u,
100                         RequestURI: u.RequestURI(),
101                 }
102                 s.testServer.Handler.ServeHTTP(resp, req)
103                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
104                 c.Check(resp.Body.String(), check.Equals, "")
105         }
106 }
107
108 // An authorizer modifies an HTTP request to make use of the given
109 // token -- by adding it to a header, cookie, query param, or whatever
110 // -- and returns the HTTP status code we should expect from keep-web if
111 // the token is invalid.
112 type authorizer func(*http.Request, string) int
113
114 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
115         s.doVhostRequests(c, authzViaAuthzHeader)
116 }
117 func authzViaAuthzHeader(r *http.Request, tok string) int {
118         r.Header.Add("Authorization", "OAuth2 "+tok)
119         return http.StatusUnauthorized
120 }
121
122 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
123         s.doVhostRequests(c, authzViaCookieValue)
124 }
125 func authzViaCookieValue(r *http.Request, tok string) int {
126         r.AddCookie(&http.Cookie{
127                 Name:  "arvados_api_token",
128                 Value: auth.EncodeTokenCookie([]byte(tok)),
129         })
130         return http.StatusUnauthorized
131 }
132
133 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
134         s.doVhostRequests(c, authzViaPath)
135 }
136 func authzViaPath(r *http.Request, tok string) int {
137         r.URL.Path = "/t=" + tok + r.URL.Path
138         return http.StatusNotFound
139 }
140
141 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
142         s.doVhostRequests(c, authzViaQueryString)
143 }
144 func authzViaQueryString(r *http.Request, tok string) int {
145         r.URL.RawQuery = "api_token=" + tok
146         return http.StatusUnauthorized
147 }
148
149 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
150         s.doVhostRequests(c, authzViaPOST)
151 }
152 func authzViaPOST(r *http.Request, tok string) int {
153         r.Method = "POST"
154         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
155         r.Body = ioutil.NopCloser(strings.NewReader(
156                 url.Values{"api_token": {tok}}.Encode()))
157         return http.StatusUnauthorized
158 }
159
160 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
161         s.doVhostRequests(c, authzViaPOST)
162 }
163 func authzViaXHRPOST(r *http.Request, tok string) int {
164         r.Method = "POST"
165         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
166         r.Header.Add("Origin", "https://origin.example")
167         r.Body = ioutil.NopCloser(strings.NewReader(
168                 url.Values{
169                         "api_token":   {tok},
170                         "disposition": {"attachment"},
171                 }.Encode()))
172         return http.StatusUnauthorized
173 }
174
175 // Try some combinations of {url, token} using the given authorization
176 // mechanism, and verify the result is correct.
177 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
178         for _, hostPath := range []string{
179                 arvadostest.FooCollection + ".example.com/foo",
180                 arvadostest.FooCollection + "--collections.example.com/foo",
181                 arvadostest.FooCollection + "--collections.example.com/_/foo",
182                 arvadostest.FooPdh + ".example.com/foo",
183                 strings.Replace(arvadostest.FooPdh, "+", "-", -1) + "--collections.example.com/foo",
184                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
185         } {
186                 c.Log("doRequests: ", hostPath)
187                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
188         }
189 }
190
191 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
192         for _, tok := range []string{
193                 arvadostest.ActiveToken,
194                 arvadostest.ActiveToken[:15],
195                 arvadostest.SpectatorToken,
196                 "bogus",
197                 "",
198         } {
199                 u := mustParseURL("http://" + hostPath)
200                 req := &http.Request{
201                         Method:     "GET",
202                         Host:       u.Host,
203                         URL:        u,
204                         RequestURI: u.RequestURI(),
205                         Header:     http.Header{},
206                 }
207                 failCode := authz(req, tok)
208                 req, resp := s.doReq(req)
209                 code, body := resp.Code, resp.Body.String()
210
211                 // If the initial request had a (non-empty) token
212                 // showing in the query string, we should have been
213                 // redirected in order to hide it in a cookie.
214                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
215
216                 if tok == arvadostest.ActiveToken {
217                         c.Check(code, check.Equals, http.StatusOK)
218                         c.Check(body, check.Equals, "foo")
219
220                 } else {
221                         c.Check(code >= 400, check.Equals, true)
222                         c.Check(code < 500, check.Equals, true)
223                         if tok == arvadostest.SpectatorToken {
224                                 // Valid token never offers to retry
225                                 // with different credentials.
226                                 c.Check(code, check.Equals, http.StatusNotFound)
227                         } else {
228                                 // Invalid token can ask to retry
229                                 // depending on the authz method.
230                                 c.Check(code, check.Equals, failCode)
231                         }
232                         c.Check(body, check.Equals, "")
233                 }
234         }
235 }
236
237 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
238         resp := httptest.NewRecorder()
239         s.testServer.Handler.ServeHTTP(resp, req)
240         if resp.Code != http.StatusSeeOther {
241                 return req, resp
242         }
243         cookies := (&http.Response{Header: resp.Header()}).Cookies()
244         u, _ := req.URL.Parse(resp.Header().Get("Location"))
245         req = &http.Request{
246                 Method:     "GET",
247                 Host:       u.Host,
248                 URL:        u,
249                 RequestURI: u.RequestURI(),
250                 Header:     http.Header{},
251         }
252         for _, c := range cookies {
253                 req.AddCookie(c)
254         }
255         return s.doReq(req)
256 }
257
258 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
259         s.testVhostRedirectTokenToCookie(c, "GET",
260                 arvadostest.FooCollection+".example.com/foo",
261                 "?api_token="+arvadostest.ActiveToken,
262                 "",
263                 "",
264                 http.StatusOK,
265                 "foo",
266         )
267 }
268
269 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
270         s.testVhostRedirectTokenToCookie(c, "GET",
271                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
272                 "",
273                 "",
274                 "",
275                 http.StatusOK,
276                 "foo",
277         )
278 }
279
280 // Bad token in URL is 404 Not Found because it doesn't make sense to
281 // retry the same URL with different authorization.
282 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
283         s.testVhostRedirectTokenToCookie(c, "GET",
284                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
285                 "",
286                 "",
287                 "",
288                 http.StatusNotFound,
289                 "",
290         )
291 }
292
293 // Bad token in a cookie (even if it got there via our own
294 // query-string-to-cookie redirect) is, in principle, retryable at the
295 // same URL so it's 401 Unauthorized.
296 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
297         s.testVhostRedirectTokenToCookie(c, "GET",
298                 arvadostest.FooCollection+".example.com/foo",
299                 "?api_token=thisisabogustoken",
300                 "",
301                 "",
302                 http.StatusUnauthorized,
303                 "",
304         )
305 }
306
307 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
308         s.testVhostRedirectTokenToCookie(c, "GET",
309                 "example.com/c="+arvadostest.FooCollection+"/foo",
310                 "?api_token="+arvadostest.ActiveToken,
311                 "",
312                 "",
313                 http.StatusBadRequest,
314                 "",
315         )
316 }
317
318 // If client requests an attachment by putting ?disposition=attachment
319 // in the query string, and gets redirected, the redirect target
320 // should respond with an attachment.
321 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
322         resp := s.testVhostRedirectTokenToCookie(c, "GET",
323                 arvadostest.FooCollection+".example.com/foo",
324                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
325                 "",
326                 "",
327                 http.StatusOK,
328                 "foo",
329         )
330         c.Check(strings.Split(resp.Header().Get("Content-Disposition"), ";")[0], check.Equals, "attachment")
331 }
332
333 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
334         s.testServer.Config.TrustAllContent = true
335         s.testVhostRedirectTokenToCookie(c, "GET",
336                 "example.com/c="+arvadostest.FooCollection+"/foo",
337                 "?api_token="+arvadostest.ActiveToken,
338                 "",
339                 "",
340                 http.StatusOK,
341                 "foo",
342         )
343 }
344
345 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
346         s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
347
348         s.testVhostRedirectTokenToCookie(c, "GET",
349                 "example.com/c="+arvadostest.FooCollection+"/foo",
350                 "?api_token="+arvadostest.ActiveToken,
351                 "",
352                 "",
353                 http.StatusBadRequest,
354                 "",
355         )
356
357         resp := s.testVhostRedirectTokenToCookie(c, "GET",
358                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
359                 "?api_token="+arvadostest.ActiveToken,
360                 "",
361                 "",
362                 http.StatusOK,
363                 "foo",
364         )
365         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
366 }
367
368 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
369         s.testVhostRedirectTokenToCookie(c, "POST",
370                 arvadostest.FooCollection+".example.com/foo",
371                 "",
372                 "application/x-www-form-urlencoded",
373                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
374                 http.StatusOK,
375                 "foo",
376         )
377 }
378
379 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
380         s.testVhostRedirectTokenToCookie(c, "POST",
381                 arvadostest.FooCollection+".example.com/foo",
382                 "",
383                 "application/x-www-form-urlencoded",
384                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
385                 http.StatusNotFound,
386                 "",
387         )
388 }
389
390 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
391         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
392         s.testVhostRedirectTokenToCookie(c, "GET",
393                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
394                 "",
395                 "",
396                 "",
397                 http.StatusOK,
398                 "Hello world\n",
399         )
400 }
401
402 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
403         s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
404         s.testVhostRedirectTokenToCookie(c, "GET",
405                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
406                 "",
407                 "",
408                 "",
409                 http.StatusNotFound,
410                 "",
411         )
412 }
413
414 // XHRs can't follow redirect-with-cookie so they rely on method=POST
415 // and disposition=attachment (telling us it's acceptable to respond
416 // with content instead of a redirect) and an Origin header that gets
417 // added automatically by the browser (telling us it's desirable to do
418 // so).
419 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
420         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
421         req := &http.Request{
422                 Method:     "POST",
423                 Host:       u.Host,
424                 URL:        u,
425                 RequestURI: u.RequestURI(),
426                 Header: http.Header{
427                         "Origin":       {"https://origin.example"},
428                         "Content-Type": {"application/x-www-form-urlencoded"},
429                 },
430                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
431                         "api_token":   {arvadostest.ActiveToken},
432                         "disposition": {"attachment"},
433                 }.Encode())),
434         }
435         resp := httptest.NewRecorder()
436         s.testServer.Handler.ServeHTTP(resp, req)
437         c.Check(resp.Code, check.Equals, http.StatusOK)
438         c.Check(resp.Body.String(), check.Equals, "foo")
439         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
440 }
441
442 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
443         u, _ := url.Parse(`http://` + hostPath + queryString)
444         req := &http.Request{
445                 Method:     method,
446                 Host:       u.Host,
447                 URL:        u,
448                 RequestURI: u.RequestURI(),
449                 Header:     http.Header{"Content-Type": {contentType}},
450                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
451         }
452
453         resp := httptest.NewRecorder()
454         defer func() {
455                 c.Check(resp.Code, check.Equals, expectStatus)
456                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
457         }()
458
459         s.testServer.Handler.ServeHTTP(resp, req)
460         if resp.Code != http.StatusSeeOther {
461                 return resp
462         }
463         c.Check(resp.Body.String(), check.Matches, `.*href="//`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
464         cookies := (&http.Response{Header: resp.Header()}).Cookies()
465
466         u, _ = u.Parse(resp.Header().Get("Location"))
467         req = &http.Request{
468                 Method:     "GET",
469                 Host:       u.Host,
470                 URL:        u,
471                 RequestURI: u.RequestURI(),
472                 Header:     http.Header{},
473         }
474         for _, c := range cookies {
475                 req.AddCookie(c)
476         }
477
478         resp = httptest.NewRecorder()
479         s.testServer.Handler.ServeHTTP(resp, req)
480         c.Check(resp.Header().Get("Location"), check.Equals, "")
481         return resp
482 }
483
484 func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
485         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
486         authHeader := http.Header{
487                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
488         }
489         for _, trial := range []struct {
490                 uri     string
491                 header  http.Header
492                 expect  []string
493                 cutDirs int
494         }{
495                 {
496                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
497                         header:  authHeader,
498                         expect:  []string{"dir1/foo", "dir1/bar"},
499                         cutDirs: 0,
500                 },
501                 {
502                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
503                         header:  authHeader,
504                         expect:  []string{"foo", "bar"},
505                         cutDirs: 0,
506                 },
507                 {
508                         uri:     "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
509                         header:  authHeader,
510                         expect:  []string{"dir1/foo", "dir1/bar"},
511                         cutDirs: 2,
512                 },
513                 {
514                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
515                         header:  nil,
516                         expect:  []string{"dir1/foo", "dir1/bar"},
517                         cutDirs: 4,
518                 },
519                 {
520                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
521                         header:  nil,
522                         expect:  []string{"dir1/foo", "dir1/bar"},
523                         cutDirs: 2,
524                 },
525                 {
526                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
527                         header:  authHeader,
528                         expect:  []string{"foo", "bar"},
529                         cutDirs: 1,
530                 },
531                 {
532                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
533                         header:  authHeader,
534                         expect:  []string{"foo", "bar"},
535                         cutDirs: 2,
536                 },
537                 {
538                         uri:     arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
539                         header:  authHeader,
540                         expect:  []string{"foo", "bar"},
541                         cutDirs: 0,
542                 },
543                 {
544                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
545                         header: authHeader,
546                         expect: nil,
547                 },
548         } {
549                 c.Logf("%q => %q", trial.uri, trial.expect)
550                 resp := httptest.NewRecorder()
551                 u := mustParseURL("//" + trial.uri)
552                 req := &http.Request{
553                         Method:     "GET",
554                         Host:       u.Host,
555                         URL:        u,
556                         RequestURI: u.RequestURI(),
557                         Header:     trial.header,
558                 }
559                 s.testServer.Handler.ServeHTTP(resp, req)
560                 var cookies []*http.Cookie
561                 for resp.Code == http.StatusSeeOther {
562                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
563                         req = &http.Request{
564                                 Method:     "GET",
565                                 Host:       u.Host,
566                                 URL:        u,
567                                 RequestURI: u.RequestURI(),
568                                 Header:     http.Header{},
569                         }
570                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
571                         for _, c := range cookies {
572                                 req.AddCookie(c)
573                         }
574                         resp = httptest.NewRecorder()
575                         s.testServer.Handler.ServeHTTP(resp, req)
576                 }
577                 if trial.expect == nil {
578                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
579                 } else {
580                         c.Check(resp.Code, check.Equals, http.StatusOK)
581                         for _, e := range trial.expect {
582                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="`+e+`".*`)
583                         }
584                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
585                 }
586         }
587 }