1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
22 "git.arvados.org/arvados.git/lib/config"
23 "git.arvados.org/arvados.git/sdk/go/arvados"
24 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
25 "git.arvados.org/arvados.git/sdk/go/arvadostest"
26 "git.arvados.org/arvados.git/sdk/go/auth"
27 "git.arvados.org/arvados.git/sdk/go/ctxlog"
28 "git.arvados.org/arvados.git/sdk/go/keepclient"
29 "github.com/sirupsen/logrus"
30 check "gopkg.in/check.v1"
33 var _ = check.Suite(&UnitSuite{})
35 type UnitSuite struct {
36 Config *arvados.Config
39 func (s *UnitSuite) SetUpTest(c *check.C) {
40 ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
42 cfg, err := ldr.Load()
43 c.Assert(err, check.IsNil)
47 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
48 h := handler{Config: newConfig(s.Config)}
49 u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
54 RequestURI: u.RequestURI(),
56 "Origin": {"https://workbench.example"},
57 "Access-Control-Request-Method": {"POST"},
61 // Check preflight for an allowed request
62 resp := httptest.NewRecorder()
63 h.ServeHTTP(resp, req)
64 c.Check(resp.Code, check.Equals, http.StatusOK)
65 c.Check(resp.Body.String(), check.Equals, "")
66 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
67 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
68 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
70 // Check preflight for a disallowed request
71 resp = httptest.NewRecorder()
72 req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
73 h.ServeHTTP(resp, req)
74 c.Check(resp.Body.String(), check.Equals, "")
75 c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
78 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
79 for _, trial := range []struct {
85 // If we return no content due to a Keep read error,
86 // we should emit a log message.
87 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
89 // If we return no content because the client sent an
90 // If-Modified-Since header, our response should be
91 // 304, and we should not emit a log message.
92 {true, true, http.StatusNotModified, ``},
94 c.Logf("trial: %+v", trial)
95 arvadostest.StartKeep(2, true)
97 arv, err := arvadosclient.MakeArvadosClient()
98 c.Assert(err, check.IsNil)
99 arv.ApiToken = arvadostest.ActiveToken
100 kc, err := keepclient.MakeKeepClient(arv)
101 c.Assert(err, check.IsNil)
102 _, _, err = kc.PutB([]byte("foo"))
103 c.Assert(err, check.IsNil)
106 h := handler{Config: newConfig(s.Config)}
107 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
108 req := &http.Request{
112 RequestURI: u.RequestURI(),
114 "Authorization": {"Bearer " + arvadostest.ActiveToken},
117 if trial.sendIMSHeader {
118 req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
121 var logbuf bytes.Buffer
122 logger := logrus.New()
124 req = req.WithContext(ctxlog.Context(context.Background(), logger))
126 resp := httptest.NewRecorder()
127 h.ServeHTTP(resp, req)
128 c.Check(resp.Code, check.Equals, trial.expectStatus)
129 c.Check(resp.Body.String(), check.Equals, "")
131 c.Log(logbuf.String())
132 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
136 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
137 bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
138 token := arvadostest.ActiveToken
139 for _, trial := range []string{
140 "http://keep-web/c=" + bogusID + "/foo",
141 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
142 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
143 "http://keep-web/collections/" + bogusID + "/foo",
144 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
145 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
148 u := mustParseURL(trial)
149 req := &http.Request{
153 RequestURI: u.RequestURI(),
155 resp := httptest.NewRecorder()
156 cfg := newConfig(s.Config)
157 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
158 h := handler{Config: cfg}
159 h.ServeHTTP(resp, req)
160 c.Check(resp.Code, check.Equals, http.StatusNotFound)
164 func mustParseURL(s string) *url.URL {
165 r, err := url.Parse(s)
167 panic("parse URL: " + s)
172 func (s *IntegrationSuite) TestVhost404(c *check.C) {
173 for _, testURL := range []string{
174 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
175 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
177 resp := httptest.NewRecorder()
178 u := mustParseURL(testURL)
179 req := &http.Request{
182 RequestURI: u.RequestURI(),
184 s.testServer.Handler.ServeHTTP(resp, req)
185 c.Check(resp.Code, check.Equals, http.StatusNotFound)
186 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
190 // An authorizer modifies an HTTP request to make use of the given
191 // token -- by adding it to a header, cookie, query param, or whatever
192 // -- and returns the HTTP status code we should expect from keep-web if
193 // the token is invalid.
194 type authorizer func(*http.Request, string) int
196 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
197 s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
199 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
200 r.Header.Add("Authorization", "Bearer "+tok)
201 return http.StatusUnauthorized
203 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
204 s.doVhostRequests(c, authzViaAuthzHeaderBearer)
206 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
207 r.Header.Add("Authorization", "Bearer "+tok)
208 return http.StatusUnauthorized
211 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
212 s.doVhostRequests(c, authzViaCookieValue)
214 func authzViaCookieValue(r *http.Request, tok string) int {
215 r.AddCookie(&http.Cookie{
216 Name: "arvados_api_token",
217 Value: auth.EncodeTokenCookie([]byte(tok)),
219 return http.StatusUnauthorized
222 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
223 s.doVhostRequests(c, authzViaPath)
225 func authzViaPath(r *http.Request, tok string) int {
226 r.URL.Path = "/t=" + tok + r.URL.Path
227 return http.StatusNotFound
230 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
231 s.doVhostRequests(c, authzViaQueryString)
233 func authzViaQueryString(r *http.Request, tok string) int {
234 r.URL.RawQuery = "api_token=" + tok
235 return http.StatusUnauthorized
238 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
239 s.doVhostRequests(c, authzViaPOST)
241 func authzViaPOST(r *http.Request, tok string) int {
243 r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
244 r.Body = ioutil.NopCloser(strings.NewReader(
245 url.Values{"api_token": {tok}}.Encode()))
246 return http.StatusUnauthorized
249 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
250 s.doVhostRequests(c, authzViaPOST)
252 func authzViaXHRPOST(r *http.Request, tok string) int {
254 r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
255 r.Header.Add("Origin", "https://origin.example")
256 r.Body = ioutil.NopCloser(strings.NewReader(
259 "disposition": {"attachment"},
261 return http.StatusUnauthorized
264 // Try some combinations of {url, token} using the given authorization
265 // mechanism, and verify the result is correct.
266 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
267 for _, hostPath := range []string{
268 arvadostest.FooCollection + ".example.com/foo",
269 arvadostest.FooCollection + "--collections.example.com/foo",
270 arvadostest.FooCollection + "--collections.example.com/_/foo",
271 arvadostest.FooCollectionPDH + ".example.com/foo",
272 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
273 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
275 c.Log("doRequests: ", hostPath)
276 s.doVhostRequestsWithHostPath(c, authz, hostPath)
280 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
281 for _, tok := range []string{
282 arvadostest.ActiveToken,
283 arvadostest.ActiveToken[:15],
284 arvadostest.SpectatorToken,
288 u := mustParseURL("http://" + hostPath)
289 req := &http.Request{
293 RequestURI: u.RequestURI(),
294 Header: http.Header{},
296 failCode := authz(req, tok)
297 req, resp := s.doReq(req)
298 code, body := resp.Code, resp.Body.String()
300 // If the initial request had a (non-empty) token
301 // showing in the query string, we should have been
302 // redirected in order to hide it in a cookie.
303 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
305 if tok == arvadostest.ActiveToken {
306 c.Check(code, check.Equals, http.StatusOK)
307 c.Check(body, check.Equals, "foo")
309 c.Check(code >= 400, check.Equals, true)
310 c.Check(code < 500, check.Equals, true)
311 if tok == arvadostest.SpectatorToken {
312 // Valid token never offers to retry
313 // with different credentials.
314 c.Check(code, check.Equals, http.StatusNotFound)
316 // Invalid token can ask to retry
317 // depending on the authz method.
318 c.Check(code, check.Equals, failCode)
321 c.Check(body, check.Equals, notFoundMessage+"\n")
323 c.Check(body, check.Equals, unauthorizedMessage+"\n")
329 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
330 for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
331 for _, port := range []string{"80", "443", "8000"} {
332 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
333 u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
334 req := &http.Request{
338 RequestURI: u.RequestURI(),
339 Header: http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
341 req, resp := s.doReq(req)
342 code, _ := resp.Code, resp.Body.String()
345 c.Check(code, check.Equals, 401)
347 c.Check(code, check.Equals, 200)
353 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
354 resp := httptest.NewRecorder()
355 s.testServer.Handler.ServeHTTP(resp, req)
356 if resp.Code != http.StatusSeeOther {
359 cookies := (&http.Response{Header: resp.Header()}).Cookies()
360 u, _ := req.URL.Parse(resp.Header().Get("Location"))
365 RequestURI: u.RequestURI(),
366 Header: http.Header{},
368 for _, c := range cookies {
374 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
375 s.testVhostRedirectTokenToCookie(c, "GET",
376 arvadostest.FooCollection+".example.com/foo",
377 "?api_token="+arvadostest.ActiveToken,
385 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
386 s.testVhostRedirectTokenToCookie(c, "GET",
387 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
396 // Bad token in URL is 404 Not Found because it doesn't make sense to
397 // retry the same URL with different authorization.
398 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
399 s.testVhostRedirectTokenToCookie(c, "GET",
400 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
405 notFoundMessage+"\n",
409 // Bad token in a cookie (even if it got there via our own
410 // query-string-to-cookie redirect) is, in principle, retryable at the
411 // same URL so it's 401 Unauthorized.
412 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
413 s.testVhostRedirectTokenToCookie(c, "GET",
414 arvadostest.FooCollection+".example.com/foo",
415 "?api_token=thisisabogustoken",
418 http.StatusUnauthorized,
419 unauthorizedMessage+"\n",
423 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
424 s.testVhostRedirectTokenToCookie(c, "GET",
425 "example.com/c="+arvadostest.FooCollection+"/foo",
426 "?api_token="+arvadostest.ActiveToken,
429 http.StatusBadRequest,
430 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
434 // If client requests an attachment by putting ?disposition=attachment
435 // in the query string, and gets redirected, the redirect target
436 // should respond with an attachment.
437 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
438 resp := s.testVhostRedirectTokenToCookie(c, "GET",
439 arvadostest.FooCollection+".example.com/foo",
440 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
446 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
449 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
450 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
451 resp := s.testVhostRedirectTokenToCookie(c, "GET",
452 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
453 "?api_token="+arvadostest.ActiveToken,
459 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
462 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
463 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
464 resp := s.testVhostRedirectTokenToCookie(c, "GET",
465 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
466 "?api_token="+arvadostest.ActiveToken,
472 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
473 resp = s.testVhostRedirectTokenToCookie(c, "GET",
474 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
475 "?api_token="+arvadostest.ActiveToken,
481 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
484 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
485 s.testServer.Config.cluster.Collections.TrustAllContent = true
486 s.testVhostRedirectTokenToCookie(c, "GET",
487 "example.com/c="+arvadostest.FooCollection+"/foo",
488 "?api_token="+arvadostest.ActiveToken,
496 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
497 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
499 s.testVhostRedirectTokenToCookie(c, "GET",
500 "example.com/c="+arvadostest.FooCollection+"/foo",
501 "?api_token="+arvadostest.ActiveToken,
504 http.StatusBadRequest,
505 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
508 resp := s.testVhostRedirectTokenToCookie(c, "GET",
509 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
510 "?api_token="+arvadostest.ActiveToken,
516 c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
519 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
520 s.testVhostRedirectTokenToCookie(c, "POST",
521 arvadostest.FooCollection+".example.com/foo",
523 "application/x-www-form-urlencoded",
524 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
530 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
531 s.testVhostRedirectTokenToCookie(c, "POST",
532 arvadostest.FooCollection+".example.com/foo",
534 "application/x-www-form-urlencoded",
535 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
537 notFoundMessage+"\n",
541 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
542 s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
543 s.testVhostRedirectTokenToCookie(c, "GET",
544 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
553 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
554 s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
555 s.testVhostRedirectTokenToCookie(c, "GET",
556 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
561 notFoundMessage+"\n",
565 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
566 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
568 client := s.testServer.Config.Client
569 client.AuthToken = arvadostest.ActiveToken
570 fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
571 c.Assert(err, check.IsNil)
572 f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
573 c.Assert(err, check.IsNil)
575 mtxt, err := fs.MarshalManifest(".")
576 c.Assert(err, check.IsNil)
577 var coll arvados.Collection
578 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
579 "collection": map[string]string{
580 "manifest_text": mtxt,
583 c.Assert(err, check.IsNil)
585 u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
586 req := &http.Request{
590 RequestURI: u.RequestURI(),
592 "Authorization": {"Bearer " + client.AuthToken},
595 resp := httptest.NewRecorder()
596 s.testServer.Handler.ServeHTTP(resp, req)
597 c.Check(resp.Code, check.Equals, http.StatusOK)
598 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`)
601 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
602 arv := arvados.NewClientFromEnv()
603 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
604 s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
605 name := "foo/bar/baz"
606 nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
607 nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
609 client := s.testServer.Config.Client
610 client.AuthToken = arvadostest.ActiveToken
611 fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
612 c.Assert(err, check.IsNil)
613 f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
614 c.Assert(err, check.IsNil)
616 mtxt, err := fs.MarshalManifest(".")
617 c.Assert(err, check.IsNil)
618 var coll arvados.Collection
619 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
620 "collection": map[string]string{
621 "manifest_text": mtxt,
623 "owner_uuid": arvadostest.AProjectUUID,
626 c.Assert(err, check.IsNil)
627 defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
629 base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
630 for tryURL, expectRegexp := range map[string]string{
631 base: `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
632 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
634 u, _ := url.Parse(tryURL)
635 req := &http.Request{
639 RequestURI: u.RequestURI(),
641 "Authorization": {"Bearer " + client.AuthToken},
644 resp := httptest.NewRecorder()
645 s.testServer.Handler.ServeHTTP(resp, req)
646 c.Check(resp.Code, check.Equals, http.StatusOK)
647 c.Check(resp.Body.String(), check.Matches, expectRegexp)
651 // XHRs can't follow redirect-with-cookie so they rely on method=POST
652 // and disposition=attachment (telling us it's acceptable to respond
653 // with content instead of a redirect) and an Origin header that gets
654 // added automatically by the browser (telling us it's desirable to do
656 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
657 u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
658 req := &http.Request{
662 RequestURI: u.RequestURI(),
664 "Origin": {"https://origin.example"},
665 "Content-Type": {"application/x-www-form-urlencoded"},
667 Body: ioutil.NopCloser(strings.NewReader(url.Values{
668 "api_token": {arvadostest.ActiveToken},
669 "disposition": {"attachment"},
672 resp := httptest.NewRecorder()
673 s.testServer.Handler.ServeHTTP(resp, req)
674 c.Check(resp.Code, check.Equals, http.StatusOK)
675 c.Check(resp.Body.String(), check.Equals, "foo")
676 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
678 // GET + Origin header is representative of both AJAX GET
679 // requests and inline images via <IMG crossorigin="anonymous"
681 u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
686 RequestURI: u.RequestURI(),
688 "Origin": {"https://origin.example"},
691 resp = httptest.NewRecorder()
692 s.testServer.Handler.ServeHTTP(resp, req)
693 c.Check(resp.Code, check.Equals, http.StatusOK)
694 c.Check(resp.Body.String(), check.Equals, "foo")
695 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
698 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
699 u, _ := url.Parse(`http://` + hostPath + queryString)
700 req := &http.Request{
704 RequestURI: u.RequestURI(),
705 Header: http.Header{"Content-Type": {contentType}},
706 Body: ioutil.NopCloser(strings.NewReader(reqBody)),
709 resp := httptest.NewRecorder()
711 c.Check(resp.Code, check.Equals, expectStatus)
712 c.Check(resp.Body.String(), check.Equals, expectRespBody)
715 s.testServer.Handler.ServeHTTP(resp, req)
716 if resp.Code != http.StatusSeeOther {
719 c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
720 cookies := (&http.Response{Header: resp.Header()}).Cookies()
722 u, _ = u.Parse(resp.Header().Get("Location"))
727 RequestURI: u.RequestURI(),
728 Header: http.Header{},
730 for _, c := range cookies {
734 resp = httptest.NewRecorder()
735 s.testServer.Handler.ServeHTTP(resp, req)
736 c.Check(resp.Header().Get("Location"), check.Equals, "")
740 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
741 s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
742 s.testDirectoryListing(c)
745 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
746 s.testServer.Config.cluster.Users.AnonymousUserToken = ""
747 s.testDirectoryListing(c)
750 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
751 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
752 authHeader := http.Header{
753 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
755 for _, trial := range []struct {
763 uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
765 expect: []string{"dir1/foo", "dir1/bar"},
769 uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
771 expect: []string{"foo", "bar"},
775 // URLs of this form ignore authHeader, and
776 // FooAndBarFilesInDirUUID isn't public, so
778 uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
783 uri: "download.example.com/users/active/foo_file_in_dir/",
785 expect: []string{"dir1/"},
789 uri: "download.example.com/users/active/foo_file_in_dir/dir1/",
791 expect: []string{"bar"},
795 uri: "download.example.com/",
797 expect: []string{"users/"},
801 uri: "download.example.com/users",
804 expect: []string{"active/"},
808 uri: "download.example.com/users/",
810 expect: []string{"active/"},
814 uri: "download.example.com/users/active",
816 redirect: "/users/active/",
817 expect: []string{"foo_file_in_dir/"},
821 uri: "download.example.com/users/active/",
823 expect: []string{"foo_file_in_dir/"},
827 uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
829 expect: []string{"dir1/foo", "dir1/bar"},
833 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
835 expect: []string{"dir1/foo", "dir1/bar"},
839 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
841 expect: []string{"dir1/foo", "dir1/bar"},
845 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
847 expect: []string{"dir1/foo", "dir1/bar"},
851 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
853 redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
854 expect: []string{"foo", "bar"},
858 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
860 expect: []string{"foo", "bar"},
864 uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
867 expect: []string{"foo", "bar"},
871 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
876 uri: "download.example.com/c=" + arvadostest.WazVersion1Collection,
878 expect: []string{"waz"},
882 uri: "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
884 expect: []string{"waz"},
888 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
889 resp := httptest.NewRecorder()
890 u := mustParseURL("//" + trial.uri)
891 req := &http.Request{
895 RequestURI: u.RequestURI(),
896 Header: copyHeader(trial.header),
898 s.testServer.Handler.ServeHTTP(resp, req)
899 var cookies []*http.Cookie
900 for resp.Code == http.StatusSeeOther {
901 u, _ := req.URL.Parse(resp.Header().Get("Location"))
906 RequestURI: u.RequestURI(),
907 Header: copyHeader(trial.header),
909 cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
910 for _, c := range cookies {
913 resp = httptest.NewRecorder()
914 s.testServer.Handler.ServeHTTP(resp, req)
916 if trial.redirect != "" {
917 c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
919 if trial.expect == nil {
920 if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
921 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
923 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
926 c.Check(resp.Code, check.Equals, http.StatusOK, comment)
927 for _, e := range trial.expect {
928 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
930 c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
933 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
938 RequestURI: u.RequestURI(),
939 Header: copyHeader(trial.header),
940 Body: ioutil.NopCloser(&bytes.Buffer{}),
942 resp = httptest.NewRecorder()
943 s.testServer.Handler.ServeHTTP(resp, req)
944 if trial.expect == nil {
945 if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
946 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
948 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
951 c.Check(resp.Code, check.Equals, http.StatusOK, comment)
958 RequestURI: u.RequestURI(),
959 Header: copyHeader(trial.header),
960 Body: ioutil.NopCloser(&bytes.Buffer{}),
962 resp = httptest.NewRecorder()
963 s.testServer.Handler.ServeHTTP(resp, req)
964 if trial.expect == nil {
965 if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
966 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
968 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
971 c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
972 for _, e := range trial.expect {
973 if strings.HasSuffix(e, "/") {
974 e = filepath.Join(u.Path, e) + "/"
976 e = filepath.Join(u.Path, e)
978 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
984 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
985 arv := arvados.NewClientFromEnv()
986 var newCollection arvados.Collection
987 err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
988 "collection": map[string]string{
989 "owner_uuid": arvadostest.ActiveUserUUID,
990 "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
991 "name": "keep-web test collection",
993 "ensure_unique_name": true,
995 c.Assert(err, check.IsNil)
996 defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
998 var updated arvados.Collection
999 for _, fnm := range []string{"foo.txt", "bar.txt"} {
1000 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1001 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1002 req := &http.Request{
1006 RequestURI: u.RequestURI(),
1007 Header: http.Header{
1008 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1011 resp := httptest.NewRecorder()
1012 s.testServer.Handler.ServeHTTP(resp, req)
1013 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1015 updated = arvados.Collection{}
1016 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1017 c.Check(err, check.IsNil)
1018 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1019 c.Logf("updated manifest_text %q", updated.ManifestText)
1021 c.Check(updated.ManifestText, check.Equals, "")
1024 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
1025 s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
1026 authHeader := http.Header{
1027 "Authorization": {"Bearer " + arvadostest.ManagementToken},
1030 resp := httptest.NewRecorder()
1031 u := mustParseURL("http://download.example.com/_health/ping")
1032 req := &http.Request{
1036 RequestURI: u.RequestURI(),
1039 s.testServer.Handler.ServeHTTP(resp, req)
1041 c.Check(resp.Code, check.Equals, http.StatusOK)
1042 c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
1045 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1046 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1048 client := s.testServer.Config.Client
1049 client.AuthToken = arvadostest.ActiveToken
1050 arv, err := arvadosclient.New(&client)
1051 c.Assert(err, check.Equals, nil)
1052 kc, err := keepclient.MakeKeepClient(arv)
1053 c.Assert(err, check.Equals, nil)
1055 fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
1056 c.Assert(err, check.IsNil)
1058 trials := []struct {
1063 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1064 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
1065 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1066 {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp
1067 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1069 for _, trial := range trials {
1070 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1071 c.Assert(err, check.IsNil)
1072 _, err = f.Write([]byte(trial.content))
1073 c.Assert(err, check.IsNil)
1074 c.Assert(f.Close(), check.IsNil)
1076 mtxt, err := fs.MarshalManifest(".")
1077 c.Assert(err, check.IsNil)
1078 var coll arvados.Collection
1079 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1080 "collection": map[string]string{
1081 "manifest_text": mtxt,
1084 c.Assert(err, check.IsNil)
1086 for _, trial := range trials {
1087 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1088 req := &http.Request{
1092 RequestURI: u.RequestURI(),
1093 Header: http.Header{
1094 "Authorization": {"Bearer " + client.AuthToken},
1097 resp := httptest.NewRecorder()
1098 s.testServer.Handler.ServeHTTP(resp, req)
1099 c.Check(resp.Code, check.Equals, http.StatusOK)
1100 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
1101 c.Check(resp.Body.String(), check.Equals, trial.content)
1105 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1106 s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1107 c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1108 u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1109 req := &http.Request{
1113 RequestURI: u.RequestURI(),
1115 resp := httptest.NewRecorder()
1116 s.testServer.Handler.ServeHTTP(resp, req)
1117 c.Check(resp.Code, check.Equals, http.StatusOK)
1118 c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1121 func copyHeader(h http.Header) http.Header {
1123 for k, v := range h {
1124 hc[k] = append([]string(nil), v...)