1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
24 "git.arvados.org/arvados.git/lib/config"
25 "git.arvados.org/arvados.git/sdk/go/arvados"
26 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
27 "git.arvados.org/arvados.git/sdk/go/arvadostest"
28 "git.arvados.org/arvados.git/sdk/go/auth"
29 "git.arvados.org/arvados.git/sdk/go/ctxlog"
30 "git.arvados.org/arvados.git/sdk/go/keepclient"
31 "github.com/prometheus/client_golang/prometheus"
32 "github.com/sirupsen/logrus"
33 check "gopkg.in/check.v1"
36 var _ = check.Suite(&UnitSuite{})
39 arvados.DebugLocksPanicMode = true
42 type UnitSuite struct {
43 cluster *arvados.Cluster
47 func (s *UnitSuite) SetUpTest(c *check.C) {
48 logger := ctxlog.TestLogger(c)
49 ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
51 cfg, err := ldr.Load()
52 c.Assert(err, check.IsNil)
53 cc, err := cfg.GetCluster("")
54 c.Assert(err, check.IsNil)
61 registry: prometheus.NewRegistry(),
66 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
68 u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
73 RequestURI: u.RequestURI(),
75 "Origin": {"https://workbench.example"},
76 "Access-Control-Request-Method": {"POST"},
80 // Check preflight for an allowed request
81 resp := httptest.NewRecorder()
82 h.ServeHTTP(resp, req)
83 c.Check(resp.Code, check.Equals, http.StatusOK)
84 c.Check(resp.Body.String(), check.Equals, "")
85 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
86 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
87 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
89 // Check preflight for a disallowed request
90 resp = httptest.NewRecorder()
91 req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
92 h.ServeHTTP(resp, req)
93 c.Check(resp.Body.String(), check.Equals, "")
94 c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
97 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
98 for _, trial := range []struct {
126 path: "/prefix/dir1/foo",
132 path: "/prefix/dir1/foo",
138 path: "/prefix/dir1/foo",
181 c.Logf("trial %+v", trial)
182 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
183 req := &http.Request{
184 Method: trial.method,
187 RequestURI: u.RequestURI(),
189 "Authorization": {"Bearer " + arvadostest.ActiveTokenV2},
190 "X-Webdav-Prefix": {trial.prefix},
191 "X-Webdav-Source": {trial.source},
193 Body: ioutil.NopCloser(bytes.NewReader(nil)),
196 resp := httptest.NewRecorder()
197 s.handler.ServeHTTP(resp, req)
199 c.Check(resp.Code, check.Equals, http.StatusNotFound)
200 } else if trial.method == "PROPFIND" {
201 c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
202 c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
203 } else if trial.seeOther {
204 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
206 c.Check(resp.Code, check.Equals, http.StatusOK)
211 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
212 // Ensure we start with an empty cache
213 defer os.Setenv("HOME", os.Getenv("HOME"))
214 os.Setenv("HOME", c.MkDir())
216 for _, trial := range []struct {
222 // If we return no content due to a Keep read error,
223 // we should emit a log message.
224 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
226 // If we return no content because the client sent an
227 // If-Modified-Since header, our response should be
228 // 304. We still expect a "File download" log since it
229 // counts as a file access for auditing.
230 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
232 c.Logf("trial: %+v", trial)
233 arvadostest.StartKeep(2, true)
234 if trial.dataExists {
235 arv, err := arvadosclient.MakeArvadosClient()
236 c.Assert(err, check.IsNil)
237 arv.ApiToken = arvadostest.ActiveToken
238 kc, err := keepclient.MakeKeepClient(arv)
239 c.Assert(err, check.IsNil)
240 _, _, err = kc.PutB([]byte("foo"))
241 c.Assert(err, check.IsNil)
244 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
245 req := &http.Request{
249 RequestURI: u.RequestURI(),
251 "Authorization": {"Bearer " + arvadostest.ActiveToken},
254 if trial.sendIMSHeader {
255 req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
258 var logbuf bytes.Buffer
259 logger := logrus.New()
261 req = req.WithContext(ctxlog.Context(context.Background(), logger))
263 resp := httptest.NewRecorder()
264 s.handler.ServeHTTP(resp, req)
265 c.Check(resp.Code, check.Equals, trial.expectStatus)
266 c.Check(resp.Body.String(), check.Equals, "")
268 c.Log(logbuf.String())
269 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
273 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
274 bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
275 token := arvadostest.ActiveToken
276 for _, trial := range []string{
277 "http://keep-web/c=" + bogusID + "/foo",
278 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
279 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
280 "http://keep-web/collections/" + bogusID + "/foo",
281 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
282 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
285 u := mustParseURL(trial)
286 req := &http.Request{
290 RequestURI: u.RequestURI(),
292 resp := httptest.NewRecorder()
293 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
294 s.handler.ServeHTTP(resp, req)
295 c.Check(resp.Code, check.Equals, http.StatusNotFound)
299 func mustParseURL(s string) *url.URL {
300 r, err := url.Parse(s)
302 panic("parse URL: " + s)
307 func (s *IntegrationSuite) TestVhost404(c *check.C) {
308 for _, testURL := range []string{
309 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
310 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
312 resp := httptest.NewRecorder()
313 u := mustParseURL(testURL)
314 req := &http.Request{
317 RequestURI: u.RequestURI(),
319 s.handler.ServeHTTP(resp, req)
320 c.Check(resp.Code, check.Equals, http.StatusNotFound)
321 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
325 // An authorizer modifies an HTTP request to make use of the given
326 // token -- by adding it to a header, cookie, query param, or whatever
327 // -- and returns the HTTP status code we should expect from keep-web if
328 // the token is invalid.
329 type authorizer func(*http.Request, string) int
331 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
332 s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
334 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
335 r.Header.Add("Authorization", "OAuth2 "+tok)
336 return http.StatusUnauthorized
339 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
340 s.doVhostRequests(c, authzViaAuthzHeaderBearer)
342 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
343 r.Header.Add("Authorization", "Bearer "+tok)
344 return http.StatusUnauthorized
347 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
348 s.doVhostRequests(c, authzViaCookieValue)
350 func authzViaCookieValue(r *http.Request, tok string) int {
351 r.AddCookie(&http.Cookie{
352 Name: "arvados_api_token",
353 Value: auth.EncodeTokenCookie([]byte(tok)),
355 return http.StatusUnauthorized
358 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
359 s.doVhostRequests(c, authzViaHTTPBasicAuth)
361 func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
362 r.AddCookie(&http.Cookie{
363 Name: "arvados_api_token",
364 Value: auth.EncodeTokenCookie([]byte(tok)),
366 return http.StatusUnauthorized
369 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
370 s.doVhostRequests(c, func(r *http.Request, tok string) int {
371 r.AddCookie(&http.Cookie{
372 Name: "arvados_api_token",
373 Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
375 return http.StatusUnauthorized
379 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
380 s.doVhostRequests(c, authzViaPath)
382 func authzViaPath(r *http.Request, tok string) int {
383 r.URL.Path = "/t=" + tok + r.URL.Path
384 return http.StatusNotFound
387 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
388 s.doVhostRequests(c, authzViaQueryString)
390 func authzViaQueryString(r *http.Request, tok string) int {
391 r.URL.RawQuery = "api_token=" + tok
392 return http.StatusUnauthorized
395 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
396 s.doVhostRequests(c, authzViaPOST)
398 func authzViaPOST(r *http.Request, tok string) int {
400 r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
401 r.Body = ioutil.NopCloser(strings.NewReader(
402 url.Values{"api_token": {tok}}.Encode()))
403 return http.StatusUnauthorized
406 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
407 s.doVhostRequests(c, authzViaPOST)
409 func authzViaXHRPOST(r *http.Request, tok string) int {
411 r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
412 r.Header.Add("Origin", "https://origin.example")
413 r.Body = ioutil.NopCloser(strings.NewReader(
416 "disposition": {"attachment"},
418 return http.StatusUnauthorized
421 // Try some combinations of {url, token} using the given authorization
422 // mechanism, and verify the result is correct.
423 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
424 for _, hostPath := range []string{
425 arvadostest.FooCollection + ".example.com/foo",
426 arvadostest.FooCollection + "--collections.example.com/foo",
427 arvadostest.FooCollection + "--collections.example.com/_/foo",
428 arvadostest.FooCollectionPDH + ".example.com/foo",
429 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
430 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
432 c.Log("doRequests: ", hostPath)
433 s.doVhostRequestsWithHostPath(c, authz, hostPath)
437 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
438 for _, tok := range []string{
439 arvadostest.ActiveToken,
440 arvadostest.ActiveToken[:15],
441 arvadostest.SpectatorToken,
445 u := mustParseURL("http://" + hostPath)
446 req := &http.Request{
450 RequestURI: u.RequestURI(),
451 Header: http.Header{},
453 failCode := authz(req, tok)
454 req, resp := s.doReq(req)
455 code, body := resp.Code, resp.Body.String()
457 // If the initial request had a (non-empty) token
458 // showing in the query string, we should have been
459 // redirected in order to hide it in a cookie.
460 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
462 if tok == arvadostest.ActiveToken {
463 c.Check(code, check.Equals, http.StatusOK)
464 c.Check(body, check.Equals, "foo")
466 c.Check(code >= 400, check.Equals, true)
467 c.Check(code < 500, check.Equals, true)
468 if tok == arvadostest.SpectatorToken {
469 // Valid token never offers to retry
470 // with different credentials.
471 c.Check(code, check.Equals, http.StatusNotFound)
473 // Invalid token can ask to retry
474 // depending on the authz method.
475 c.Check(code, check.Equals, failCode)
478 c.Check(body, check.Equals, notFoundMessage+"\n")
480 c.Check(body, check.Equals, unauthorizedMessage+"\n")
486 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
487 for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
488 for _, port := range []string{"80", "443", "8000"} {
489 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
490 u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
491 req := &http.Request{
495 RequestURI: u.RequestURI(),
496 Header: http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
498 req, resp := s.doReq(req)
499 code, _ := resp.Code, resp.Body.String()
502 c.Check(code, check.Equals, 401)
504 c.Check(code, check.Equals, 200)
510 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
511 u := mustParseURL(urlstring)
512 if hdr == nil && token != "" {
513 hdr = http.Header{"Authorization": {"Bearer " + token}}
514 } else if hdr == nil {
516 } else if token != "" {
517 panic("must not pass both token and hdr")
519 return s.doReq(&http.Request{
523 RequestURI: u.RequestURI(),
528 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
529 resp := httptest.NewRecorder()
530 s.handler.ServeHTTP(resp, req)
531 if resp.Code != http.StatusSeeOther {
534 cookies := (&http.Response{Header: resp.Header()}).Cookies()
535 u, _ := req.URL.Parse(resp.Header().Get("Location"))
540 RequestURI: u.RequestURI(),
541 Header: http.Header{},
543 for _, c := range cookies {
549 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
550 s.testVhostRedirectTokenToCookie(c, "GET",
551 arvadostest.FooCollection+".example.com/foo",
552 "?api_token="+arvadostest.ActiveToken,
560 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
561 s.testVhostRedirectTokenToCookie(c, "GET",
562 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
571 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
572 s.testVhostRedirectTokenToCookie(c, "GET",
573 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
580 // Same valid sharing token, but requesting a different collection
581 s.testVhostRedirectTokenToCookie(c, "GET",
582 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
587 regexp.QuoteMeta(notFoundMessage+"\n"),
591 // Bad token in URL is 404 Not Found because it doesn't make sense to
592 // retry the same URL with different authorization.
593 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
594 s.testVhostRedirectTokenToCookie(c, "GET",
595 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
600 regexp.QuoteMeta(notFoundMessage+"\n"),
604 // Bad token in a cookie (even if it got there via our own
605 // query-string-to-cookie redirect) is, in principle, retryable via
606 // wb2-login-and-redirect flow.
607 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
609 resp := s.testVhostRedirectTokenToCookie(c, "GET",
610 arvadostest.FooCollection+".example.com/foo",
611 "?api_token=thisisabogustoken",
612 http.Header{"Sec-Fetch-Mode": {"navigate"}},
617 u, err := url.Parse(resp.Header().Get("Location"))
618 c.Assert(err, check.IsNil)
619 c.Logf("redirected to %s", u)
620 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
621 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
622 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
624 // Download/attachment indicated by ?disposition=attachment
625 resp = s.testVhostRedirectTokenToCookie(c, "GET",
626 arvadostest.FooCollection+".example.com/foo",
627 "?api_token=thisisabogustoken&disposition=attachment",
628 http.Header{"Sec-Fetch-Mode": {"navigate"}},
633 u, err = url.Parse(resp.Header().Get("Location"))
634 c.Assert(err, check.IsNil)
635 c.Logf("redirected to %s", u)
636 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
637 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
638 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
640 // Download/attachment indicated by vhost
641 resp = s.testVhostRedirectTokenToCookie(c, "GET",
642 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
643 "?api_token=thisisabogustoken",
644 http.Header{"Sec-Fetch-Mode": {"navigate"}},
649 u, err = url.Parse(resp.Header().Get("Location"))
650 c.Assert(err, check.IsNil)
651 c.Logf("redirected to %s", u)
652 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
653 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
654 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
656 // Without "Sec-Fetch-Mode: navigate" header, just 401.
657 s.testVhostRedirectTokenToCookie(c, "GET",
658 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
659 "?api_token=thisisabogustoken",
660 http.Header{"Sec-Fetch-Mode": {"cors"}},
662 http.StatusUnauthorized,
663 regexp.QuoteMeta(unauthorizedMessage+"\n"),
665 s.testVhostRedirectTokenToCookie(c, "GET",
666 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
667 "?api_token=thisisabogustoken",
670 http.StatusUnauthorized,
671 regexp.QuoteMeta(unauthorizedMessage+"\n"),
675 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
676 resp := s.testVhostRedirectTokenToCookie(c, "GET",
677 arvadostest.FooCollection+".example.com/foo",
678 "?api_token=thisisabogustoken",
680 "Sec-Fetch-Mode": {"navigate"},
681 "Cache-Control": {"no-cache"},
687 u, err := url.Parse(resp.Header().Get("Location"))
688 c.Assert(err, check.IsNil)
689 c.Logf("redirected to %s", u)
690 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
691 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
692 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
695 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
696 for _, trial := range []struct {
701 {cacheControl: "no-cache"},
703 {anonToken: true, cacheControl: "no-cache"},
705 c.Logf("trial: %+v", trial)
708 s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
710 s.handler.Cluster.Users.AnonymousUserToken = ""
712 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
713 c.Assert(err, check.IsNil)
714 req.Header.Set("Sec-Fetch-Mode", "navigate")
715 if trial.cacheControl != "" {
716 req.Header.Set("Cache-Control", trial.cacheControl)
718 resp := httptest.NewRecorder()
719 s.handler.ServeHTTP(resp, req)
720 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
721 u, err := url.Parse(resp.Header().Get("Location"))
722 c.Assert(err, check.IsNil)
723 c.Logf("redirected to %q", u)
724 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
725 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
726 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
730 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
731 s.testVhostRedirectTokenToCookie(c, "GET",
732 "example.com/c="+arvadostest.FooCollection+"/foo",
733 "?api_token="+arvadostest.ActiveToken,
736 http.StatusBadRequest,
737 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
741 // If client requests an attachment by putting ?disposition=attachment
742 // in the query string, and gets redirected, the redirect target
743 // should respond with an attachment.
744 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
745 resp := s.testVhostRedirectTokenToCookie(c, "GET",
746 arvadostest.FooCollection+".example.com/foo",
747 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
753 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
756 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
757 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
758 resp := s.testVhostRedirectTokenToCookie(c, "GET",
759 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
760 "?api_token="+arvadostest.ActiveToken,
766 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
769 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
770 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
771 resp := s.testVhostRedirectTokenToCookie(c, "GET",
772 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
773 "?api_token="+arvadostest.ActiveToken,
779 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
780 resp = s.testVhostRedirectTokenToCookie(c, "GET",
781 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
782 "?api_token="+arvadostest.ActiveToken,
788 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
791 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
792 s.handler.Cluster.Collections.TrustAllContent = true
793 s.testVhostRedirectTokenToCookie(c, "GET",
794 "example.com/c="+arvadostest.FooCollection+"/foo",
795 "?api_token="+arvadostest.ActiveToken,
803 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
804 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
806 s.testVhostRedirectTokenToCookie(c, "GET",
807 "example.com/c="+arvadostest.FooCollection+"/foo",
808 "?api_token="+arvadostest.ActiveToken,
811 http.StatusBadRequest,
812 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
815 resp := s.testVhostRedirectTokenToCookie(c, "GET",
816 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
817 "?api_token="+arvadostest.ActiveToken,
823 c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
826 func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
827 baseUrl := arvadostest.FooCollection + ".example.com/foo"
828 query := url.Values{}
830 // The intent of these tests is to check that requests are redirected
831 // correctly in the presence of multiple API tokens. The exact response
832 // codes and content are not closely considered: they're just how
833 // keep-web responded when we made the smallest possible fix. Changing
834 // those responses may be okay, but you should still test all these
835 // different cases and the associated redirect logic.
836 query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
837 s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
838 query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
839 s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
840 query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
841 s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
842 query["api_token"] = []string{"", arvadostest.ActiveToken}
843 s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
845 expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
846 query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
847 s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
848 query["api_token"] = []string{arvadostest.AnonymousToken, ""}
849 s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
850 query["api_token"] = []string{"", arvadostest.AnonymousToken}
851 s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
854 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
855 s.testVhostRedirectTokenToCookie(c, "POST",
856 arvadostest.FooCollection+".example.com/foo",
858 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
859 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
865 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
866 s.testVhostRedirectTokenToCookie(c, "POST",
867 arvadostest.FooCollection+".example.com/foo",
869 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
870 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
872 regexp.QuoteMeta(notFoundMessage+"\n"),
876 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
877 s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
878 s.testVhostRedirectTokenToCookie(c, "GET",
879 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
888 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
889 s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
890 s.testVhostRedirectTokenToCookie(c, "GET",
891 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
895 http.StatusUnauthorized,
896 "Authorization tokens are not accepted here: .*\n",
900 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
901 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
903 client := arvados.NewClientFromEnv()
904 client.AuthToken = arvadostest.ActiveToken
905 fs, err := (&arvados.Collection{}).FileSystem(client, nil)
906 c.Assert(err, check.IsNil)
907 f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
908 c.Assert(err, check.IsNil)
910 mtxt, err := fs.MarshalManifest(".")
911 c.Assert(err, check.IsNil)
912 var coll arvados.Collection
913 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
914 "collection": map[string]string{
915 "manifest_text": mtxt,
918 c.Assert(err, check.IsNil)
920 u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
921 req := &http.Request{
925 RequestURI: u.RequestURI(),
927 "Authorization": {"Bearer " + client.AuthToken},
930 resp := httptest.NewRecorder()
931 s.handler.ServeHTTP(resp, req)
932 c.Check(resp.Code, check.Equals, http.StatusOK)
933 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`)
936 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
937 arv := arvados.NewClientFromEnv()
938 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
939 s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
940 name := "foo/bar/baz"
941 nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
942 nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
944 client := arvados.NewClientFromEnv()
945 client.AuthToken = arvadostest.ActiveToken
946 fs, err := (&arvados.Collection{}).FileSystem(client, nil)
947 c.Assert(err, check.IsNil)
948 f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
949 c.Assert(err, check.IsNil)
951 mtxt, err := fs.MarshalManifest(".")
952 c.Assert(err, check.IsNil)
953 var coll arvados.Collection
954 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
955 "collection": map[string]string{
956 "manifest_text": mtxt,
958 "owner_uuid": arvadostest.AProjectUUID,
961 c.Assert(err, check.IsNil)
962 defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
964 base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
965 for tryURL, expectRegexp := range map[string]string{
966 base: `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
967 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
969 u, _ := url.Parse(tryURL)
970 req := &http.Request{
974 RequestURI: u.RequestURI(),
976 "Authorization": {"Bearer " + client.AuthToken},
979 resp := httptest.NewRecorder()
980 s.handler.ServeHTTP(resp, req)
981 c.Check(resp.Code, check.Equals, http.StatusOK)
982 c.Check(resp.Body.String(), check.Matches, expectRegexp)
986 // XHRs can't follow redirect-with-cookie so they rely on method=POST
987 // and disposition=attachment (telling us it's acceptable to respond
988 // with content instead of a redirect) and an Origin header that gets
989 // added automatically by the browser (telling us it's desirable to do
991 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
992 u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
993 req := &http.Request{
997 RequestURI: u.RequestURI(),
999 "Origin": {"https://origin.example"},
1000 "Content-Type": {"application/x-www-form-urlencoded"},
1002 Body: ioutil.NopCloser(strings.NewReader(url.Values{
1003 "api_token": {arvadostest.ActiveToken},
1004 "disposition": {"attachment"},
1007 resp := httptest.NewRecorder()
1008 s.handler.ServeHTTP(resp, req)
1009 c.Check(resp.Code, check.Equals, http.StatusOK)
1010 c.Check(resp.Body.String(), check.Equals, "foo")
1011 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1013 // GET + Origin header is representative of both AJAX GET
1014 // requests and inline images via <IMG crossorigin="anonymous"
1016 u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
1017 req = &http.Request{
1021 RequestURI: u.RequestURI(),
1022 Header: http.Header{
1023 "Origin": {"https://origin.example"},
1026 resp = httptest.NewRecorder()
1027 s.handler.ServeHTTP(resp, req)
1028 c.Check(resp.Code, check.Equals, http.StatusOK)
1029 c.Check(resp.Body.String(), check.Equals, "foo")
1030 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1033 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
1034 if reqHeader == nil {
1035 reqHeader = http.Header{}
1037 u, _ := url.Parse(`http://` + hostPath + queryString)
1038 c.Logf("requesting %s", u)
1039 req := &http.Request{
1043 RequestURI: u.RequestURI(),
1045 Body: ioutil.NopCloser(strings.NewReader(reqBody)),
1048 resp := httptest.NewRecorder()
1050 c.Check(resp.Code, check.Equals, expectStatus)
1051 c.Check(resp.Body.String(), check.Matches, matchRespBody)
1054 s.handler.ServeHTTP(resp, req)
1055 if resp.Code != http.StatusSeeOther {
1056 attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
1057 // Since we're not redirecting, check that any api_token in the URL is
1059 // If there is no token in the URL, then we're good.
1060 // Otherwise, if the response code is an error, the body is expected to
1061 // be static content, and nothing that might maliciously introspect the
1062 // URL. It's considered safe and allowed.
1063 // Otherwise, if the response content has attachment disposition,
1064 // that's considered safe for all the reasons explained in the
1065 // safeAttachment comment in handler.go.
1066 c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
1070 loc, err := url.Parse(resp.Header().Get("Location"))
1071 c.Assert(err, check.IsNil)
1072 c.Check(loc.Scheme, check.Equals, u.Scheme)
1073 c.Check(loc.Host, check.Equals, u.Host)
1074 c.Check(loc.RawPath, check.Equals, u.RawPath)
1075 // If the response was a redirect, it should never include an API token.
1076 c.Check(loc.Query().Has("api_token"), check.Equals, false)
1077 c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1078 cookies := (&http.Response{Header: resp.Header()}).Cookies()
1080 c.Logf("following redirect to %s", u)
1081 req = &http.Request{
1085 RequestURI: loc.RequestURI(),
1088 for _, c := range cookies {
1092 resp = httptest.NewRecorder()
1093 s.handler.ServeHTTP(resp, req)
1095 if resp.Code != http.StatusSeeOther {
1096 c.Check(resp.Header().Get("Location"), check.Equals, "")
1101 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1102 s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1103 s.testDirectoryListing(c)
1106 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1107 s.handler.Cluster.Users.AnonymousUserToken = ""
1108 s.testDirectoryListing(c)
1111 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1112 // The "ownership cycle" test fixtures are reachable from the
1113 // "filter group without filters" group, causing webdav's
1114 // walkfs to recurse indefinitely. Avoid that by deleting one
1115 // of the bogus fixtures.
1116 arv := arvados.NewClientFromEnv()
1117 err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil)
1119 c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{})
1120 c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
1123 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1124 authHeader := http.Header{
1125 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
1127 for _, trial := range []struct {
1135 uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1137 expect: []string{"dir1/foo", "dir1/bar"},
1141 uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1143 expect: []string{"foo", "bar"},
1147 // URLs of this form ignore authHeader, and
1148 // FooAndBarFilesInDirUUID isn't public, so
1149 // this returns 401.
1150 uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1155 uri: "download.example.com/users/active/foo_file_in_dir/",
1157 expect: []string{"dir1/"},
1161 uri: "download.example.com/users/active/foo_file_in_dir/dir1/",
1163 expect: []string{"bar"},
1167 uri: "download.example.com/",
1169 expect: []string{"users/"},
1173 uri: "download.example.com/users",
1175 redirect: "/users/",
1176 expect: []string{"active/"},
1180 uri: "download.example.com/users/",
1182 expect: []string{"active/"},
1186 uri: "download.example.com/users/active",
1188 redirect: "/users/active/",
1189 expect: []string{"foo_file_in_dir/"},
1193 uri: "download.example.com/users/active/",
1195 expect: []string{"foo_file_in_dir/"},
1199 uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1201 expect: []string{"dir1/foo", "dir1/bar"},
1205 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1207 expect: []string{"dir1/foo", "dir1/bar"},
1211 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1213 expect: []string{"dir1/foo", "dir1/bar"},
1217 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1219 expect: []string{"dir1/foo", "dir1/bar"},
1223 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1225 redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1226 expect: []string{"foo", "bar"},
1230 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1232 expect: []string{"foo", "bar"},
1236 uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1239 expect: []string{"foo", "bar"},
1243 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1248 uri: "download.example.com/c=" + arvadostest.WazVersion1Collection,
1250 expect: []string{"waz"},
1254 uri: "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1256 expect: []string{"waz"},
1260 uri: "download.example.com/users/active/This filter group/",
1262 expect: []string{"A Subproject/"},
1266 uri: "download.example.com/users/active/This filter group/A Subproject",
1268 expect: []string{"baz_file/"},
1272 uri: "download.example.com/by_id/" + arvadostest.AFilterGroupUUID,
1274 expect: []string{"A Subproject/"},
1278 uri: "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject",
1280 expect: []string{"baz_file/"},
1284 comment := check.Commentf("HTML: %q redir %q => %q", trial.uri, trial.redirect, trial.expect)
1285 resp := httptest.NewRecorder()
1286 u := mustParseURL("//" + trial.uri)
1287 req := &http.Request{
1291 RequestURI: u.RequestURI(),
1292 Header: copyHeader(trial.header),
1294 s.handler.ServeHTTP(resp, req)
1295 var cookies []*http.Cookie
1296 for resp.Code == http.StatusSeeOther {
1297 u, _ := req.URL.Parse(resp.Header().Get("Location"))
1298 req = &http.Request{
1302 RequestURI: u.RequestURI(),
1303 Header: copyHeader(trial.header),
1305 cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1306 for _, c := range cookies {
1309 resp = httptest.NewRecorder()
1310 s.handler.ServeHTTP(resp, req)
1312 if trial.redirect != "" {
1313 c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1315 if trial.expect == nil {
1316 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1318 c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1319 for _, e := range trial.expect {
1320 e = strings.Replace(e, " ", "%20", -1)
1321 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
1323 c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
1326 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1327 req = &http.Request{
1331 RequestURI: u.RequestURI(),
1332 Header: copyHeader(trial.header),
1333 Body: ioutil.NopCloser(&bytes.Buffer{}),
1335 resp = httptest.NewRecorder()
1336 s.handler.ServeHTTP(resp, req)
1337 if trial.expect == nil {
1338 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1340 c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1343 req = &http.Request{
1347 RequestURI: u.RequestURI(),
1348 Header: copyHeader(trial.header),
1349 Body: ioutil.NopCloser(&bytes.Buffer{}),
1351 resp = httptest.NewRecorder()
1352 s.handler.ServeHTTP(resp, req)
1353 // This check avoids logging a big XML document in the
1354 // event webdav throws a 500 error after sending
1355 // headers for a 207.
1356 if !c.Check(strings.HasSuffix(resp.Body.String(), "Internal Server Error"), check.Equals, false) {
1359 if trial.expect == nil {
1360 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1362 c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1363 for _, e := range trial.expect {
1364 if strings.HasSuffix(e, "/") {
1365 e = filepath.Join(u.Path, e) + "/"
1367 e = filepath.Join(u.Path, e)
1369 e = strings.Replace(e, " ", "%20", -1)
1370 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1376 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1377 arv := arvados.NewClientFromEnv()
1378 var newCollection arvados.Collection
1379 err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1380 "collection": map[string]string{
1381 "owner_uuid": arvadostest.ActiveUserUUID,
1382 "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1383 "name": "keep-web test collection",
1385 "ensure_unique_name": true,
1387 c.Assert(err, check.IsNil)
1388 defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1390 var updated arvados.Collection
1391 for _, fnm := range []string{"foo.txt", "bar.txt"} {
1392 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1393 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1394 req := &http.Request{
1398 RequestURI: u.RequestURI(),
1399 Header: http.Header{
1400 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1403 resp := httptest.NewRecorder()
1404 s.handler.ServeHTTP(resp, req)
1405 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1407 updated = arvados.Collection{}
1408 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1409 c.Check(err, check.IsNil)
1410 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1411 c.Logf("updated manifest_text %q", updated.ManifestText)
1413 c.Check(updated.ManifestText, check.Equals, "")
1416 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1417 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1419 client := arvados.NewClientFromEnv()
1420 client.AuthToken = arvadostest.ActiveToken
1421 arv, err := arvadosclient.New(client)
1422 c.Assert(err, check.Equals, nil)
1423 kc, err := keepclient.MakeKeepClient(arv)
1424 c.Assert(err, check.Equals, nil)
1426 fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1427 c.Assert(err, check.IsNil)
1429 trials := []struct {
1434 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1435 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1436 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1437 {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp
1438 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1440 for _, trial := range trials {
1441 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1442 c.Assert(err, check.IsNil)
1443 _, err = f.Write([]byte(trial.content))
1444 c.Assert(err, check.IsNil)
1445 c.Assert(f.Close(), check.IsNil)
1447 mtxt, err := fs.MarshalManifest(".")
1448 c.Assert(err, check.IsNil)
1449 var coll arvados.Collection
1450 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1451 "collection": map[string]string{
1452 "manifest_text": mtxt,
1455 c.Assert(err, check.IsNil)
1457 for _, trial := range trials {
1458 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1459 req := &http.Request{
1463 RequestURI: u.RequestURI(),
1464 Header: http.Header{
1465 "Authorization": {"Bearer " + client.AuthToken},
1468 resp := httptest.NewRecorder()
1469 s.handler.ServeHTTP(resp, req)
1470 c.Check(resp.Code, check.Equals, http.StatusOK)
1471 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1472 c.Check(resp.Body.String(), check.Equals, trial.content)
1476 func (s *IntegrationSuite) TestCacheSize(c *check.C) {
1477 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1478 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
1479 c.Assert(err, check.IsNil)
1480 resp := httptest.NewRecorder()
1481 s.handler.ServeHTTP(resp, req)
1482 c.Assert(resp.Code, check.Equals, http.StatusOK)
1483 c.Check(s.handler.Cache.sessions[arvadostest.ActiveTokenV2].client.DiskCacheSize.Percent(), check.Equals, int64(10))
1486 // Writing to a collection shouldn't affect its entry in the
1487 // PDH-to-manifest cache.
1488 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1489 arv, err := arvadosclient.MakeArvadosClient()
1490 c.Assert(err, check.Equals, nil)
1491 arv.ApiToken = arvadostest.ActiveToken
1493 u := mustParseURL("http://x.example/testfile")
1494 req := &http.Request{
1498 RequestURI: u.RequestURI(),
1499 Header: http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1502 checkWithID := func(id string, status int) {
1503 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1504 req.Host = req.URL.Host
1505 resp := httptest.NewRecorder()
1506 s.handler.ServeHTTP(resp, req)
1507 c.Check(resp.Code, check.Equals, status)
1510 var colls [2]arvados.Collection
1511 for i := range colls {
1512 err := arv.Create("collections",
1513 map[string]interface{}{
1514 "ensure_unique_name": true,
1515 "collection": map[string]interface{}{
1516 "name": "test collection",
1519 c.Assert(err, check.Equals, nil)
1522 // Populate cache with empty collection
1523 checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1525 // write a file to colls[0]
1527 reqPut.Method = "PUT"
1528 reqPut.URL.Host = colls[0].UUID + ".example"
1529 reqPut.Host = req.URL.Host
1530 reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1531 resp := httptest.NewRecorder()
1532 s.handler.ServeHTTP(resp, &reqPut)
1533 c.Check(resp.Code, check.Equals, http.StatusCreated)
1535 // new file should not appear in colls[1]
1536 checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1537 checkWithID(colls[1].UUID, http.StatusNotFound)
1539 checkWithID(colls[0].UUID, http.StatusOK)
1542 func copyHeader(h http.Header) http.Header {
1544 for k, v := range h {
1545 hc[k] = append([]string(nil), v...)
1550 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1551 successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
1553 client := arvados.NewClientFromEnv()
1554 client.AuthToken = arvadostest.AdminToken
1555 var logentries arvados.LogList
1557 err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1558 arvados.ResourceListParams{
1560 Order: "created_at desc"})
1561 c.Check(err, check.IsNil)
1562 c.Check(logentries.Items, check.HasLen, 1)
1563 lastLogId := logentries.Items[0].ID
1564 c.Logf("lastLogId: %d", lastLogId)
1566 var logbuf bytes.Buffer
1567 logger := logrus.New()
1568 logger.Out = &logbuf
1569 resp := httptest.NewRecorder()
1570 req = req.WithContext(ctxlog.Context(context.Background(), logger))
1571 s.handler.ServeHTTP(resp, req)
1574 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1575 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1576 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1578 deadline := time.Now().Add(time.Second)
1580 c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1581 logentries = arvados.LogList{}
1582 err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1583 arvados.ResourceListParams{
1584 Filters: []arvados.Filter{
1585 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1586 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1589 Order: "created_at desc",
1591 c.Assert(err, check.IsNil)
1592 if len(logentries.Items) > 0 &&
1593 logentries.Items[0].ID > lastLogId &&
1594 logentries.Items[0].ObjectUUID == userUuid &&
1595 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1596 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
1597 logentries.Items[0].Properties["collection_file_path"] == filepath {
1600 c.Logf("logentries.Items: %+v", logentries.Items)
1601 time.Sleep(50 * time.Millisecond)
1604 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1605 c.Check(logbuf.String(), check.Equals, "")
1609 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1610 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1612 s.handler.Cluster.Collections.TrustAllContent = true
1614 for _, adminperm := range []bool{true, false} {
1615 for _, userperm := range []bool{true, false} {
1616 s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1617 s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1619 // Test admin permission
1620 req := &http.Request{
1624 RequestURI: u.RequestURI(),
1625 Header: http.Header{
1626 "Authorization": {"Bearer " + arvadostest.AdminToken},
1629 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1630 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1632 // Test user permission
1633 req = &http.Request{
1637 RequestURI: u.RequestURI(),
1638 Header: http.Header{
1639 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1642 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1643 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1647 s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1649 for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1650 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1652 u = mustParseURL(tryurl)
1653 req := &http.Request{
1657 RequestURI: u.RequestURI(),
1658 Header: http.Header{
1659 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1662 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1663 arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
1666 u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1667 req := &http.Request{
1671 RequestURI: u.RequestURI(),
1672 Header: http.Header{
1673 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1676 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1677 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
1680 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1681 for _, adminperm := range []bool{true, false} {
1682 for _, userperm := range []bool{true, false} {
1684 arv := arvados.NewClientFromEnv()
1685 arv.AuthToken = arvadostest.ActiveToken
1687 var coll arvados.Collection
1688 err := arv.RequestAndDecode(&coll,
1690 "/arvados/v1/collections",
1692 map[string]interface{}{
1693 "ensure_unique_name": true,
1694 "collection": map[string]interface{}{
1695 "name": "test collection",
1698 c.Assert(err, check.Equals, nil)
1700 u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1702 s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1703 s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1705 // Test admin permission
1706 req := &http.Request{
1710 RequestURI: u.RequestURI(),
1711 Header: http.Header{
1712 "Authorization": {"Bearer " + arvadostest.AdminToken},
1714 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1716 s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1717 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
1719 // Test user permission
1720 req = &http.Request{
1724 RequestURI: u.RequestURI(),
1725 Header: http.Header{
1726 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1728 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1730 s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1731 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
1736 func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
1737 s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
1738 lockTidyInterval = time.Second
1739 client := arvados.NewClientFromEnv()
1740 client.AuthToken = arvadostest.ActiveTokenV2
1741 // Start small, and increase concurrency (2^2, 4^2, ...)
1742 // only until hitting failure. Avoids unnecessarily long
1744 for n := 2; n < 16 && !c.Failed(); n = n * 2 {
1745 c.Logf("%s: n=%d", c.TestName(), n)
1747 var coll arvados.Collection
1748 err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
1749 c.Assert(err, check.IsNil)
1750 defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
1752 var wg sync.WaitGroup
1753 for i := 0; i < n && !c.Failed(); i++ {
1758 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1759 resp := httptest.NewRecorder()
1760 req, err := http.NewRequest("MKCOL", u.String(), nil)
1761 c.Assert(err, check.IsNil)
1762 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1763 s.handler.ServeHTTP(resp, req)
1764 c.Assert(resp.Code, check.Equals, http.StatusCreated)
1765 for j := 0; j < n && !c.Failed(); j++ {
1770 content := fmt.Sprintf("i=%d/j=%d", i, j)
1771 u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
1773 resp := httptest.NewRecorder()
1774 req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
1775 c.Assert(err, check.IsNil)
1776 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1777 s.handler.ServeHTTP(resp, req)
1778 c.Check(resp.Code, check.Equals, http.StatusCreated)
1780 time.Sleep(time.Second)
1781 resp = httptest.NewRecorder()
1782 req, err = http.NewRequest("GET", u.String(), nil)
1783 c.Assert(err, check.IsNil)
1784 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1785 s.handler.ServeHTTP(resp, req)
1786 c.Check(resp.Code, check.Equals, http.StatusOK)
1787 c.Check(resp.Body.String(), check.Equals, content)
1793 for i := 0; i < n; i++ {
1794 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1795 resp := httptest.NewRecorder()
1796 req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
1797 c.Assert(err, check.IsNil)
1798 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1799 s.handler.ServeHTTP(resp, req)
1800 c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)