Merge branch '8784-dir-listings'
[arvados.git] / services / keep-web / handler_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package main
6
7 import (
8         "fmt"
9         "html"
10         "io/ioutil"
11         "net/http"
12         "net/http/httptest"
13         "net/url"
14         "regexp"
15         "strings"
16
17         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
18         "git.curoverse.com/arvados.git/sdk/go/auth"
19         check "gopkg.in/check.v1"
20 )
21
22 var _ = check.Suite(&UnitSuite{})
23
24 type UnitSuite struct{}
25
26 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
27         h := handler{Config: DefaultConfig()}
28         u, _ := url.Parse("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
29         req := &http.Request{
30                 Method:     "OPTIONS",
31                 Host:       u.Host,
32                 URL:        u,
33                 RequestURI: u.RequestURI(),
34                 Header: http.Header{
35                         "Origin":                        {"https://workbench.example"},
36                         "Access-Control-Request-Method": {"POST"},
37                 },
38         }
39
40         // Check preflight for an allowed request
41         resp := httptest.NewRecorder()
42         h.ServeHTTP(resp, req)
43         c.Check(resp.Code, check.Equals, http.StatusOK)
44         c.Check(resp.Body.String(), check.Equals, "")
45         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
46         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "GET, POST")
47         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Range")
48
49         // Check preflight for a disallowed request
50         resp = httptest.NewRecorder()
51         req.Header.Set("Access-Control-Request-Method", "DELETE")
52         h.ServeHTTP(resp, req)
53         c.Check(resp.Body.String(), check.Equals, "")
54         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
55 }
56
57 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
58         bogusID := strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "-"
59         token := arvadostest.ActiveToken
60         for _, trial := range []string{
61                 "http://keep-web/c=" + bogusID + "/foo",
62                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
63                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
64                 "http://keep-web/collections/" + bogusID + "/foo",
65                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
66                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
67         } {
68                 c.Log(trial)
69                 u, err := url.Parse(trial)
70                 c.Assert(err, check.IsNil)
71                 req := &http.Request{
72                         Method:     "GET",
73                         Host:       u.Host,
74                         URL:        u,
75                         RequestURI: u.RequestURI(),
76                 }
77                 resp := httptest.NewRecorder()
78                 cfg := DefaultConfig()
79                 cfg.AnonymousTokens = []string{arvadostest.AnonymousToken}
80                 h := handler{Config: cfg}
81                 h.ServeHTTP(resp, req)
82                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
83         }
84 }
85
86 func mustParseURL(s string) *url.URL {
87         r, err := url.Parse(s)
88         if err != nil {
89                 panic("parse URL: " + s)
90         }
91         return r
92 }
93
94 func (s *IntegrationSuite) TestVhost404(c *check.C) {
95         for _, testURL := range []string{
96                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
97                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
98         } {
99                 resp := httptest.NewRecorder()
100                 u := mustParseURL(testURL)
101                 req := &http.Request{
102                         Method:     "GET",
103                         URL:        u,
104                         RequestURI: u.RequestURI(),
105                 }
106                 s.testServer.Handler.ServeHTTP(resp, req)
107                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
108                 c.Check(resp.Body.String(), check.Equals, "")
109         }
110 }
111
112 // An authorizer modifies an HTTP request to make use of the given
113 // token -- by adding it to a header, cookie, query param, or whatever
114 // -- and returns the HTTP status code we should expect from keep-web if
115 // the token is invalid.
116 type authorizer func(*http.Request, string) int
117
118 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
119         s.doVhostRequests(c, authzViaAuthzHeader)
120 }
121 func authzViaAuthzHeader(r *http.Request, tok string) int {
122         r.Header.Add("Authorization", "OAuth2 "+tok)
123         return http.StatusUnauthorized
124 }
125
126 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
127         s.doVhostRequests(c, authzViaCookieValue)
128 }
129 func authzViaCookieValue(r *http.Request, tok string) int {
130         r.AddCookie(&http.Cookie{
131                 Name:  "arvados_api_token",
132                 Value: auth.EncodeTokenCookie([]byte(tok)),
133         })
134         return http.StatusUnauthorized
135 }
136
137 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
138         s.doVhostRequests(c, authzViaPath)
139 }
140 func authzViaPath(r *http.Request, tok string) int {
141         r.URL.Path = "/t=" + tok + r.URL.Path
142         return http.StatusNotFound
143 }
144
145 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
146         s.doVhostRequests(c, authzViaQueryString)
147 }
148 func authzViaQueryString(r *http.Request, tok string) int {
149         r.URL.RawQuery = "api_token=" + tok
150         return http.StatusUnauthorized
151 }
152
153 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
154         s.doVhostRequests(c, authzViaPOST)
155 }
156 func authzViaPOST(r *http.Request, tok string) int {
157         r.Method = "POST"
158         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
159         r.Body = ioutil.NopCloser(strings.NewReader(
160                 url.Values{"api_token": {tok}}.Encode()))
161         return http.StatusUnauthorized
162 }
163
164 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
165         s.doVhostRequests(c, authzViaPOST)
166 }
167 func authzViaXHRPOST(r *http.Request, tok string) int {
168         r.Method = "POST"
169         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
170         r.Header.Add("Origin", "https://origin.example")
171         r.Body = ioutil.NopCloser(strings.NewReader(
172                 url.Values{
173                         "api_token":   {tok},
174                         "disposition": {"attachment"},
175                 }.Encode()))
176         return http.StatusUnauthorized
177 }
178
179 // Try some combinations of {url, token} using the given authorization
180 // mechanism, and verify the result is correct.
181 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
182         for _, hostPath := range []string{
183                 arvadostest.FooCollection + ".example.com/foo",
184                 arvadostest.FooCollection + "--collections.example.com/foo",
185                 arvadostest.FooCollection + "--collections.example.com/_/foo",
186                 arvadostest.FooPdh + ".example.com/foo",
187                 strings.Replace(arvadostest.FooPdh, "+", "-", -1) + "--collections.example.com/foo",
188                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
189         } {
190                 c.Log("doRequests: ", hostPath)
191                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
192         }
193 }
194
195 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
196         for _, tok := range []string{
197                 arvadostest.ActiveToken,
198                 arvadostest.ActiveToken[:15],
199                 arvadostest.SpectatorToken,
200                 "bogus",
201                 "",
202         } {
203                 u := mustParseURL("http://" + hostPath)
204                 req := &http.Request{
205                         Method:     "GET",
206                         Host:       u.Host,
207                         URL:        u,
208                         RequestURI: u.RequestURI(),
209                         Header:     http.Header{},
210                 }
211                 failCode := authz(req, tok)
212                 req, resp := s.doReq(req)
213                 code, body := resp.Code, resp.Body.String()
214
215                 // If the initial request had a (non-empty) token
216                 // showing in the query string, we should have been
217                 // redirected in order to hide it in a cookie.
218                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
219
220                 if tok == arvadostest.ActiveToken {
221                         c.Check(code, check.Equals, http.StatusOK)
222                         c.Check(body, check.Equals, "foo")
223
224                 } else {
225                         c.Check(code >= 400, check.Equals, true)
226                         c.Check(code < 500, check.Equals, true)
227                         if tok == arvadostest.SpectatorToken {
228                                 // Valid token never offers to retry
229                                 // with different credentials.
230                                 c.Check(code, check.Equals, http.StatusNotFound)
231                         } else {
232                                 // Invalid token can ask to retry
233                                 // depending on the authz method.
234                                 c.Check(code, check.Equals, failCode)
235                         }
236                         c.Check(body, check.Equals, "")
237                 }
238         }
239 }
240
241 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
242         resp := httptest.NewRecorder()
243         s.testServer.Handler.ServeHTTP(resp, req)
244         if resp.Code != http.StatusSeeOther {
245                 return req, resp
246         }
247         cookies := (&http.Response{Header: resp.Header()}).Cookies()
248         u, _ := req.URL.Parse(resp.Header().Get("Location"))
249         req = &http.Request{
250                 Method:     "GET",
251                 Host:       u.Host,
252                 URL:        u,
253                 RequestURI: u.RequestURI(),
254                 Header:     http.Header{},
255         }
256         for _, c := range cookies {
257                 req.AddCookie(c)
258         }
259         return s.doReq(req)
260 }
261
262 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
263         s.testVhostRedirectTokenToCookie(c, "GET",
264                 arvadostest.FooCollection+".example.com/foo",
265                 "?api_token="+arvadostest.ActiveToken,
266                 "",
267                 "",
268                 http.StatusOK,
269                 "foo",
270         )
271 }
272
273 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
274         s.testVhostRedirectTokenToCookie(c, "GET",
275                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
276                 "",
277                 "",
278                 "",
279                 http.StatusOK,
280                 "foo",
281         )
282 }
283
284 // Bad token in URL is 404 Not Found because it doesn't make sense to
285 // retry the same URL with different authorization.
286 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
287         s.testVhostRedirectTokenToCookie(c, "GET",
288                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
289                 "",
290                 "",
291                 "",
292                 http.StatusNotFound,
293                 "",
294         )
295 }
296
297 // Bad token in a cookie (even if it got there via our own
298 // query-string-to-cookie redirect) is, in principle, retryable at the
299 // same URL so it's 401 Unauthorized.
300 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
301         s.testVhostRedirectTokenToCookie(c, "GET",
302                 arvadostest.FooCollection+".example.com/foo",
303                 "?api_token=thisisabogustoken",
304                 "",
305                 "",
306                 http.StatusUnauthorized,
307                 "",
308         )
309 }
310
311 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
312         s.testVhostRedirectTokenToCookie(c, "GET",
313                 "example.com/c="+arvadostest.FooCollection+"/foo",
314                 "?api_token="+arvadostest.ActiveToken,
315                 "",
316                 "",
317                 http.StatusBadRequest,
318                 "",
319         )
320 }
321
322 // If client requests an attachment by putting ?disposition=attachment
323 // in the query string, and gets redirected, the redirect target
324 // should respond with an attachment.
325 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
326         resp := s.testVhostRedirectTokenToCookie(c, "GET",
327                 arvadostest.FooCollection+".example.com/foo",
328                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
329                 "",
330                 "",
331                 http.StatusOK,
332                 "foo",
333         )
334         c.Check(strings.Split(resp.Header().Get("Content-Disposition"), ";")[0], check.Equals, "attachment")
335 }
336
337 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
338         s.testServer.Config.TrustAllContent = true
339         s.testVhostRedirectTokenToCookie(c, "GET",
340                 "example.com/c="+arvadostest.FooCollection+"/foo",
341                 "?api_token="+arvadostest.ActiveToken,
342                 "",
343                 "",
344                 http.StatusOK,
345                 "foo",
346         )
347 }
348
349 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
350         s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
351
352         s.testVhostRedirectTokenToCookie(c, "GET",
353                 "example.com/c="+arvadostest.FooCollection+"/foo",
354                 "?api_token="+arvadostest.ActiveToken,
355                 "",
356                 "",
357                 http.StatusBadRequest,
358                 "",
359         )
360
361         resp := s.testVhostRedirectTokenToCookie(c, "GET",
362                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
363                 "?api_token="+arvadostest.ActiveToken,
364                 "",
365                 "",
366                 http.StatusOK,
367                 "foo",
368         )
369         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
370 }
371
372 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
373         s.testVhostRedirectTokenToCookie(c, "POST",
374                 arvadostest.FooCollection+".example.com/foo",
375                 "",
376                 "application/x-www-form-urlencoded",
377                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
378                 http.StatusOK,
379                 "foo",
380         )
381 }
382
383 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
384         s.testVhostRedirectTokenToCookie(c, "POST",
385                 arvadostest.FooCollection+".example.com/foo",
386                 "",
387                 "application/x-www-form-urlencoded",
388                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
389                 http.StatusNotFound,
390                 "",
391         )
392 }
393
394 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
395         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
396         s.testVhostRedirectTokenToCookie(c, "GET",
397                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
398                 "",
399                 "",
400                 "",
401                 http.StatusOK,
402                 "Hello world\n",
403         )
404 }
405
406 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
407         s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
408         s.testVhostRedirectTokenToCookie(c, "GET",
409                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
410                 "",
411                 "",
412                 "",
413                 http.StatusNotFound,
414                 "",
415         )
416 }
417
418 // XHRs can't follow redirect-with-cookie so they rely on method=POST
419 // and disposition=attachment (telling us it's acceptable to respond
420 // with content instead of a redirect) and an Origin header that gets
421 // added automatically by the browser (telling us it's desirable to do
422 // so).
423 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
424         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
425         req := &http.Request{
426                 Method:     "POST",
427                 Host:       u.Host,
428                 URL:        u,
429                 RequestURI: u.RequestURI(),
430                 Header: http.Header{
431                         "Origin":       {"https://origin.example"},
432                         "Content-Type": {"application/x-www-form-urlencoded"},
433                 },
434                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
435                         "api_token":   {arvadostest.ActiveToken},
436                         "disposition": {"attachment"},
437                 }.Encode())),
438         }
439         resp := httptest.NewRecorder()
440         s.testServer.Handler.ServeHTTP(resp, req)
441         c.Check(resp.Code, check.Equals, http.StatusOK)
442         c.Check(resp.Body.String(), check.Equals, "foo")
443         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
444 }
445
446 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
447         u, _ := url.Parse(`http://` + hostPath + queryString)
448         req := &http.Request{
449                 Method:     method,
450                 Host:       u.Host,
451                 URL:        u,
452                 RequestURI: u.RequestURI(),
453                 Header:     http.Header{"Content-Type": {contentType}},
454                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
455         }
456
457         resp := httptest.NewRecorder()
458         defer func() {
459                 c.Check(resp.Code, check.Equals, expectStatus)
460                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
461         }()
462
463         s.testServer.Handler.ServeHTTP(resp, req)
464         if resp.Code != http.StatusSeeOther {
465                 return resp
466         }
467         c.Check(resp.Body.String(), check.Matches, `.*href="//`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
468         cookies := (&http.Response{Header: resp.Header()}).Cookies()
469
470         u, _ = u.Parse(resp.Header().Get("Location"))
471         req = &http.Request{
472                 Method:     "GET",
473                 Host:       u.Host,
474                 URL:        u,
475                 RequestURI: u.RequestURI(),
476                 Header:     http.Header{},
477         }
478         for _, c := range cookies {
479                 req.AddCookie(c)
480         }
481
482         resp = httptest.NewRecorder()
483         s.testServer.Handler.ServeHTTP(resp, req)
484         c.Check(resp.Header().Get("Location"), check.Equals, "")
485         return resp
486 }
487
488 func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
489         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
490         authHeader := http.Header{
491                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
492         }
493         for _, trial := range []struct {
494                 uri     string
495                 header  http.Header
496                 expect  []string
497                 cutDirs int
498         }{
499                 {
500                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
501                         header:  authHeader,
502                         expect:  []string{"dir1/foo", "dir1/bar"},
503                         cutDirs: 0,
504                 },
505                 {
506                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
507                         header:  authHeader,
508                         expect:  []string{"foo", "bar"},
509                         cutDirs: 0,
510                 },
511                 {
512                         uri:     "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
513                         header:  authHeader,
514                         expect:  []string{"dir1/foo", "dir1/bar"},
515                         cutDirs: 2,
516                 },
517                 {
518                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
519                         header:  nil,
520                         expect:  []string{"dir1/foo", "dir1/bar"},
521                         cutDirs: 4,
522                 },
523                 {
524                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
525                         header:  nil,
526                         expect:  []string{"dir1/foo", "dir1/bar"},
527                         cutDirs: 2,
528                 },
529                 {
530                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
531                         header:  authHeader,
532                         expect:  []string{"foo", "bar"},
533                         cutDirs: 1,
534                 },
535                 {
536                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
537                         header:  authHeader,
538                         expect:  []string{"foo", "bar"},
539                         cutDirs: 2,
540                 },
541                 {
542                         uri:     arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
543                         header:  authHeader,
544                         expect:  []string{"foo", "bar"},
545                         cutDirs: 0,
546                 },
547                 {
548                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
549                         header: authHeader,
550                         expect: nil,
551                 },
552         } {
553                 c.Logf("%q => %q", trial.uri, trial.expect)
554                 resp := httptest.NewRecorder()
555                 u := mustParseURL("//" + trial.uri)
556                 req := &http.Request{
557                         Method:     "GET",
558                         Host:       u.Host,
559                         URL:        u,
560                         RequestURI: u.RequestURI(),
561                         Header:     trial.header,
562                 }
563                 s.testServer.Handler.ServeHTTP(resp, req)
564                 var cookies []*http.Cookie
565                 for resp.Code == http.StatusSeeOther {
566                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
567                         req = &http.Request{
568                                 Method:     "GET",
569                                 Host:       u.Host,
570                                 URL:        u,
571                                 RequestURI: u.RequestURI(),
572                                 Header:     http.Header{},
573                         }
574                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
575                         for _, c := range cookies {
576                                 req.AddCookie(c)
577                         }
578                         resp = httptest.NewRecorder()
579                         s.testServer.Handler.ServeHTTP(resp, req)
580                 }
581                 if trial.expect == nil {
582                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
583                 } else {
584                         c.Check(resp.Code, check.Equals, http.StatusOK)
585                         for _, e := range trial.expect {
586                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="`+e+`".*`)
587                         }
588                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
589                 }
590         }
591 }