X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a34999d53489f83f87687ee876750085673d023d..9ad1be8f5e545fef32b0317f04a17f34ba762497:/services/keep-web/handler_test.go diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go index 8715ab24f3..07c7016d3a 100644 --- a/services/keep-web/handler_test.go +++ b/services/keep-web/handler_test.go @@ -2,13 +2,14 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepweb import ( "bytes" "context" "fmt" "html" + "io" "io/ioutil" "net/http" "net/http/httptest" @@ -17,6 +18,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "time" "git.arvados.org/arvados.git/lib/config" @@ -26,6 +28,7 @@ import ( "git.arvados.org/arvados.git/sdk/go/auth" "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/keepclient" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" check "gopkg.in/check.v1" ) @@ -37,19 +40,32 @@ func init() { } type UnitSuite struct { - Config *arvados.Config + cluster *arvados.Cluster + handler *handler } func (s *UnitSuite) SetUpTest(c *check.C) { - ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c)) + logger := ctxlog.TestLogger(c) + ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger) ldr.Path = "-" cfg, err := ldr.Load() c.Assert(err, check.IsNil) - s.Config = cfg + cc, err := cfg.GetCluster("") + c.Assert(err, check.IsNil) + s.cluster = cc + s.handler = &handler{ + Cluster: cc, + Cache: cache{ + cluster: cc, + logger: logger, + registry: prometheus.NewRegistry(), + }, + metrics: newMetrics(prometheus.NewRegistry()), + } } func (s *UnitSuite) TestCORSPreflight(c *check.C) { - h := handler{Config: newConfig(s.Config)} + h := s.handler u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo") req := &http.Request{ Method: "OPTIONS", @@ -69,7 +85,7 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) { c.Check(resp.Body.String(), check.Equals, "") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK") - c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout") + c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control") // Check preflight for a disallowed request resp = httptest.NewRecorder() @@ -79,7 +95,125 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) { c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed) } +func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) { + for _, trial := range []struct { + method string + path string + prefix string + source string + notFound bool + seeOther bool + }{ + { + method: "PROPFIND", + path: "/", + }, + { + method: "PROPFIND", + path: "/dir1", + }, + { + method: "PROPFIND", + path: "/dir1/", + }, + { + method: "PROPFIND", + path: "/dir1/foo", + prefix: "/dir1", + source: "/dir1", + }, + { + method: "PROPFIND", + path: "/prefix/dir1/foo", + prefix: "/prefix/", + source: "", + }, + { + method: "PROPFIND", + path: "/prefix/dir1/foo", + prefix: "/prefix", + source: "", + }, + { + method: "PROPFIND", + path: "/prefix/dir1/foo", + prefix: "/prefix/", + source: "/", + }, + { + method: "PROPFIND", + path: "/prefix/foo", + prefix: "/prefix/", + source: "/dir1/", + }, + { + method: "GET", + path: "/prefix/foo", + prefix: "/prefix/", + source: "/dir1/", + }, + { + method: "PROPFIND", + path: "/prefix/", + prefix: "/prefix", + source: "/dir1", + }, + { + method: "PROPFIND", + path: "/prefix", + prefix: "/prefix", + source: "/dir1/", + }, + { + method: "GET", + path: "/prefix", + prefix: "/prefix", + source: "/dir1", + seeOther: true, + }, + { + method: "PROPFIND", + path: "/dir1/foo", + prefix: "", + source: "/dir1", + notFound: true, + }, + } { + c.Logf("trial %+v", trial) + u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path) + req := &http.Request{ + Method: trial.method, + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveTokenV2}, + "X-Webdav-Prefix": {trial.prefix}, + "X-Webdav-Source": {trial.source}, + }, + Body: ioutil.NopCloser(bytes.NewReader(nil)), + } + + resp := httptest.NewRecorder() + s.handler.ServeHTTP(resp, req) + if trial.notFound { + c.Check(resp.Code, check.Equals, http.StatusNotFound) + } else if trial.method == "PROPFIND" { + c.Check(resp.Code, check.Equals, http.StatusMultiStatus) + c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`) + } else if trial.seeOther { + c.Check(resp.Code, check.Equals, http.StatusSeeOther) + } else { + c.Check(resp.Code, check.Equals, http.StatusOK) + } + } +} + func (s *UnitSuite) TestEmptyResponse(c *check.C) { + // Ensure we start with an empty cache + defer os.Setenv("HOME", os.Getenv("HOME")) + os.Setenv("HOME", c.MkDir()) + for _, trial := range []struct { dataExists bool sendIMSHeader bool @@ -92,8 +226,9 @@ func (s *UnitSuite) TestEmptyResponse(c *check.C) { // If we return no content because the client sent an // If-Modified-Since header, our response should be - // 304, and we should not emit a log message. - {true, true, http.StatusNotModified, ``}, + // 304. We still expect a "File download" log since it + // counts as a file access for auditing. + {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`}, } { c.Logf("trial: %+v", trial) arvadostest.StartKeep(2, true) @@ -107,7 +242,6 @@ func (s *UnitSuite) TestEmptyResponse(c *check.C) { c.Assert(err, check.IsNil) } - h := handler{Config: newConfig(s.Config)} u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo") req := &http.Request{ Method: "GET", @@ -128,7 +262,7 @@ func (s *UnitSuite) TestEmptyResponse(c *check.C) { req = req.WithContext(ctxlog.Context(context.Background(), logger)) resp := httptest.NewRecorder() - h.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, trial.expectStatus) c.Check(resp.Body.String(), check.Equals, "") @@ -157,10 +291,8 @@ func (s *UnitSuite) TestInvalidUUID(c *check.C) { RequestURI: u.RequestURI(), } resp := httptest.NewRecorder() - cfg := newConfig(s.Config) - cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken - h := handler{Config: cfg} - h.ServeHTTP(resp, req) + s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusNotFound) } } @@ -185,7 +317,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) { URL: u, RequestURI: u.RequestURI(), } - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusNotFound) c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n") } @@ -201,9 +333,10 @@ func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) { s.doVhostRequests(c, authzViaAuthzHeaderOAuth2) } func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int { - r.Header.Add("Authorization", "Bearer "+tok) + r.Header.Add("Authorization", "OAuth2 "+tok) return http.StatusUnauthorized } + func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) { s.doVhostRequests(c, authzViaAuthzHeaderBearer) } @@ -223,6 +356,27 @@ func authzViaCookieValue(r *http.Request, tok string) int { return http.StatusUnauthorized } +func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) { + s.doVhostRequests(c, authzViaHTTPBasicAuth) +} +func authzViaHTTPBasicAuth(r *http.Request, tok string) int { + r.AddCookie(&http.Cookie{ + Name: "arvados_api_token", + Value: auth.EncodeTokenCookie([]byte(tok)), + }) + return http.StatusUnauthorized +} + +func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) { + s.doVhostRequests(c, func(r *http.Request, tok string) int { + r.AddCookie(&http.Cookie{ + Name: "arvados_api_token", + Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")), + }) + return http.StatusUnauthorized + }) +} + func (s *IntegrationSuite) TestVhostViaPath(c *check.C) { s.doVhostRequests(c, authzViaPath) } @@ -333,7 +487,7 @@ func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authori func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) { for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} { for _, port := range []string{"80", "443", "8000"} { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port) + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port) u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection)) req := &http.Request{ Method: "GET", @@ -354,9 +508,27 @@ func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) { } } +func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) { + u := mustParseURL(urlstring) + if hdr == nil && token != "" { + hdr = http.Header{"Authorization": {"Bearer " + token}} + } else if hdr == nil { + hdr = http.Header{} + } else if token != "" { + panic("must not pass both token and hdr") + } + return s.doReq(&http.Request{ + Method: method, + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: hdr, + }) +} + func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) { resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) if resp.Code != http.StatusSeeOther { return req, resp } @@ -379,7 +551,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", arvadostest.FooCollection+".example.com/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "foo", @@ -390,11 +562,31 @@ func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo", "", + nil, + "", + http.StatusOK, + "foo", + ) +} + +func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) { + s.testVhostRedirectTokenToCookie(c, "GET", + "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo", "", + nil, "", http.StatusOK, "foo", ) + // Same valid sharing token, but requesting a different collection + s.testVhostRedirectTokenToCookie(c, "GET", + "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo", + "", + nil, + "", + http.StatusNotFound, + regexp.QuoteMeta(notFoundMessage+"\n"), + ) } // Bad token in URL is 404 Not Found because it doesn't make sense to @@ -403,35 +595,147 @@ func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo", "", - "", + nil, "", http.StatusNotFound, - notFoundMessage+"\n", + regexp.QuoteMeta(notFoundMessage+"\n"), ) } // Bad token in a cookie (even if it got there via our own -// query-string-to-cookie redirect) is, in principle, retryable at the -// same URL so it's 401 Unauthorized. +// query-string-to-cookie redirect) is, in principle, retryable via +// wb2-login-and-redirect flow. func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) { - s.testVhostRedirectTokenToCookie(c, "GET", + // Inline + resp := s.testVhostRedirectTokenToCookie(c, "GET", + arvadostest.FooCollection+".example.com/foo", + "?api_token=thisisabogustoken", + http.Header{"Sec-Fetch-Mode": {"navigate"}}, + "", + http.StatusSeeOther, + "", + ) + u, err := url.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Logf("redirected to %s", u) + c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host) + c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo") + c.Check(u.Query().Get("redirectToDownload"), check.Equals, "") + + // Download/attachment indicated by ?disposition=attachment + resp = s.testVhostRedirectTokenToCookie(c, "GET", arvadostest.FooCollection+".example.com/foo", + "?api_token=thisisabogustoken&disposition=attachment", + http.Header{"Sec-Fetch-Mode": {"navigate"}}, + "", + http.StatusSeeOther, + "", + ) + u, err = url.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Logf("redirected to %s", u) + c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host) + c.Check(u.Query().Get("redirectToPreview"), check.Equals, "") + c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo") + + // Download/attachment indicated by vhost + resp = s.testVhostRedirectTokenToCookie(c, "GET", + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo", + "?api_token=thisisabogustoken", + http.Header{"Sec-Fetch-Mode": {"navigate"}}, + "", + http.StatusSeeOther, + "", + ) + u, err = url.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Logf("redirected to %s", u) + c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host) + c.Check(u.Query().Get("redirectToPreview"), check.Equals, "") + c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo") + + // Without "Sec-Fetch-Mode: navigate" header, just 401. + s.testVhostRedirectTokenToCookie(c, "GET", + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo", "?api_token=thisisabogustoken", + http.Header{"Sec-Fetch-Mode": {"cors"}}, "", + http.StatusUnauthorized, + regexp.QuoteMeta(unauthorizedMessage+"\n"), + ) + s.testVhostRedirectTokenToCookie(c, "GET", + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo", + "?api_token=thisisabogustoken", + nil, "", http.StatusUnauthorized, - unauthorizedMessage+"\n", + regexp.QuoteMeta(unauthorizedMessage+"\n"), + ) +} + +func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) { + resp := s.testVhostRedirectTokenToCookie(c, "GET", + arvadostest.FooCollection+".example.com/foo", + "?api_token=thisisabogustoken", + http.Header{ + "Sec-Fetch-Mode": {"navigate"}, + "Cache-Control": {"no-cache"}, + }, + "", + http.StatusSeeOther, + "", ) + u, err := url.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Logf("redirected to %s", u) + c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host) + c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo") + c.Check(u.Query().Get("redirectToDownload"), check.Equals, "") +} + +func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) { + for _, trial := range []struct { + anonToken bool + cacheControl string + }{ + {}, + {cacheControl: "no-cache"}, + {anonToken: true}, + {anonToken: true, cacheControl: "no-cache"}, + } { + c.Logf("trial: %+v", trial) + + if trial.anonToken { + s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken + } else { + s.handler.Cluster.Users.AnonymousUserToken = "" + } + req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil) + c.Assert(err, check.IsNil) + req.Header.Set("Sec-Fetch-Mode", "navigate") + if trial.cacheControl != "" { + req.Header.Set("Cache-Control", trial.cacheControl) + } + resp := httptest.NewRecorder() + s.handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusSeeOther) + u, err := url.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Logf("redirected to %q", u) + c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host) + c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo") + c.Check(u.Query().Get("redirectToDownload"), check.Equals, "") + } } func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusBadRequest, - "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n", + regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"), ) } @@ -442,7 +746,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check resp := s.testVhostRedirectTokenToCookie(c, "GET", arvadostest.FooCollection+".example.com/foo", "?disposition=attachment&api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "foo", @@ -451,11 +755,11 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check } func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" resp := s.testVhostRedirectTokenToCookie(c, "GET", "download.example.com/by_id/"+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "foo", @@ -464,11 +768,11 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) { } func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" resp := s.testVhostRedirectTokenToCookie(c, "GET", "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "waz", @@ -477,7 +781,7 @@ func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) { resp = s.testVhostRedirectTokenToCookie(c, "GET", "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "waz", @@ -486,11 +790,11 @@ func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) { } func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) { - s.testServer.Config.cluster.Collections.TrustAllContent = true + s.handler.Cluster.Collections.TrustAllContent = true s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "foo", @@ -498,21 +802,21 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C } func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234" s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusBadRequest, - "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n", + regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"), ) resp := s.testVhostRedirectTokenToCookie(c, "GET", "example.com:1234/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "foo", @@ -520,11 +824,39 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *chec c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment") } +func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) { + baseUrl := arvadostest.FooCollection + ".example.com/foo" + query := url.Values{} + + // The intent of these tests is to check that requests are redirected + // correctly in the presence of multiple API tokens. The exact response + // codes and content are not closely considered: they're just how + // keep-web responded when we made the smallest possible fix. Changing + // those responses may be okay, but you should still test all these + // different cases and the associated redirect logic. + query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken} + s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo") + query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""} + s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo") + query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken} + s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo") + query["api_token"] = []string{"", arvadostest.ActiveToken} + s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo") + + expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n") + query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"} + s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent) + query["api_token"] = []string{arvadostest.AnonymousToken, ""} + s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent) + query["api_token"] = []string{"", arvadostest.AnonymousToken} + s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent) +} + func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) { s.testVhostRedirectTokenToCookie(c, "POST", arvadostest.FooCollection+".example.com/foo", "", - "application/x-www-form-urlencoded", + http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(), http.StatusOK, "foo", @@ -535,19 +867,19 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) s.testVhostRedirectTokenToCookie(c, "POST", arvadostest.FooCollection+".example.com/foo", "", - "application/x-www-form-urlencoded", + http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(), http.StatusNotFound, - notFoundMessage+"\n", + regexp.QuoteMeta(notFoundMessage+"\n"), ) } func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) { - s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken + s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt", "", - "", + nil, "", http.StatusOK, "Hello world\n", @@ -555,23 +887,23 @@ func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) { } func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) { - s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid" + s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid" s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt", "", + nil, "", - "", - http.StatusNotFound, - notFoundMessage+"\n", + http.StatusUnauthorized, + "Authorization tokens are not accepted here: .*\n", ) } func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" - client := s.testServer.Config.Client + client := arvados.NewClientFromEnv() client.AuthToken = arvadostest.ActiveToken - fs, err := (&arvados.Collection{}).FileSystem(&client, nil) + fs, err := (&arvados.Collection{}).FileSystem(client, nil) c.Assert(err, check.IsNil) f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777) c.Assert(err, check.IsNil) @@ -597,22 +929,22 @@ func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) { }, } resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`) } func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) { arv := arvados.NewClientFromEnv() - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" - s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}" name := "foo/bar/baz" nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1) nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1) - client := s.testServer.Config.Client + client := arvados.NewClientFromEnv() client.AuthToken = arvadostest.ActiveToken - fs, err := (&arvados.Collection{}).FileSystem(&client, nil) + fs, err := (&arvados.Collection{}).FileSystem(client, nil) c.Assert(err, check.IsNil) f, err := fs.OpenFile("filename", os.O_CREATE, 0777) c.Assert(err, check.IsNil) @@ -646,7 +978,7 @@ func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) { }, } resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Matches, expectRegexp) } @@ -674,7 +1006,7 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) { }.Encode())), } resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Equals, "foo") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") @@ -693,66 +1025,103 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) { }, } resp = httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Equals, "foo") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") } -func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder { +func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder { + if reqHeader == nil { + reqHeader = http.Header{} + } u, _ := url.Parse(`http://` + hostPath + queryString) + c.Logf("requesting %s", u) req := &http.Request{ Method: method, Host: u.Host, URL: u, RequestURI: u.RequestURI(), - Header: http.Header{"Content-Type": {contentType}}, + Header: reqHeader, Body: ioutil.NopCloser(strings.NewReader(reqBody)), } resp := httptest.NewRecorder() defer func() { c.Check(resp.Code, check.Equals, expectStatus) - c.Check(resp.Body.String(), check.Equals, expectRespBody) + c.Check(resp.Body.String(), check.Matches, matchRespBody) }() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) if resp.Code != http.StatusSeeOther { + attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition")) + // Since we're not redirecting, check that any api_token in the URL is + // handled safely. + // If there is no token in the URL, then we're good. + // Otherwise, if the response code is an error, the body is expected to + // be static content, and nothing that might maliciously introspect the + // URL. It's considered safe and allowed. + // Otherwise, if the response content has attachment disposition, + // that's considered safe for all the reasons explained in the + // safeAttachment comment in handler.go. + c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true) return resp } + + loc, err := url.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Check(loc.Scheme, check.Equals, u.Scheme) + c.Check(loc.Host, check.Equals, u.Host) + c.Check(loc.RawPath, check.Equals, u.RawPath) + // If the response was a redirect, it should never include an API token. + c.Check(loc.Query().Has("api_token"), check.Equals, false) c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`) cookies := (&http.Response{Header: resp.Header()}).Cookies() - u, _ = u.Parse(resp.Header().Get("Location")) + c.Logf("following redirect to %s", u) req = &http.Request{ Method: "GET", - Host: u.Host, - URL: u, - RequestURI: u.RequestURI(), - Header: http.Header{}, + Host: loc.Host, + URL: loc, + RequestURI: loc.RequestURI(), + Header: reqHeader, } for _, c := range cookies { req.AddCookie(c) } resp = httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) - c.Check(resp.Header().Get("Location"), check.Equals, "") + s.handler.ServeHTTP(resp, req) + + if resp.Code != http.StatusSeeOther { + c.Check(resp.Header().Get("Location"), check.Equals, "") + } return resp } func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) { - s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken + s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken s.testDirectoryListing(c) } func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) { - s.testServer.Config.cluster.Users.AnonymousUserToken = "" + s.handler.Cluster.Users.AnonymousUserToken = "" s.testDirectoryListing(c) } func (s *IntegrationSuite) testDirectoryListing(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + // The "ownership cycle" test fixtures are reachable from the + // "filter group without filters" group, causing webdav's + // walkfs to recurse indefinitely. Avoid that by deleting one + // of the bogus fixtures. + arv := arvados.NewClientFromEnv() + err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil) + if err != nil { + c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{}) + c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404) + } + + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" authHeader := http.Header{ "Authorization": {"OAuth2 " + arvadostest.ActiveToken}, } @@ -888,8 +1257,32 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { expect: []string{"waz"}, cutDirs: 2, }, + { + uri: "download.example.com/users/active/This filter group/", + header: authHeader, + expect: []string{"A Subproject/"}, + cutDirs: 3, + }, + { + uri: "download.example.com/users/active/This filter group/A Subproject", + header: authHeader, + expect: []string{"baz_file/"}, + cutDirs: 4, + }, + { + uri: "download.example.com/by_id/" + arvadostest.AFilterGroupUUID, + header: authHeader, + expect: []string{"A Subproject/"}, + cutDirs: 2, + }, + { + uri: "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject", + header: authHeader, + expect: []string{"baz_file/"}, + cutDirs: 3, + }, } { - comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect) + comment := check.Commentf("HTML: %q redir %q => %q", trial.uri, trial.redirect, trial.expect) resp := httptest.NewRecorder() u := mustParseURL("//" + trial.uri) req := &http.Request{ @@ -899,7 +1292,7 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { RequestURI: u.RequestURI(), Header: copyHeader(trial.header), } - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) var cookies []*http.Cookie for resp.Code == http.StatusSeeOther { u, _ := req.URL.Parse(resp.Header().Get("Location")) @@ -915,20 +1308,17 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { req.AddCookie(c) } resp = httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) } if trial.redirect != "" { c.Check(req.URL.Path, check.Equals, trial.redirect, comment) } if trial.expect == nil { - if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { - c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) - } else { - c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) - } + c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusOK, comment) for _, e := range trial.expect { + e = strings.Replace(e, " ", "%20", -1) c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment) } c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment) @@ -944,13 +1334,9 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { Body: ioutil.NopCloser(&bytes.Buffer{}), } resp = httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) if trial.expect == nil { - if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { - c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) - } else { - c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) - } + c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusOK, comment) } @@ -964,13 +1350,15 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { Body: ioutil.NopCloser(&bytes.Buffer{}), } resp = httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) + // This check avoids logging a big XML document in the + // event webdav throws a 500 error after sending + // headers for a 207. + if !c.Check(strings.HasSuffix(resp.Body.String(), "Internal Server Error"), check.Equals, false) { + continue + } if trial.expect == nil { - if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { - c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) - } else { - c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) - } + c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment) for _, e := range trial.expect { @@ -979,6 +1367,7 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { } else { e = filepath.Join(u.Path, e) } + e = strings.Replace(e, " ", "%20", -1) c.Check(resp.Body.String(), check.Matches, `(?ms).*`+e+`.*`, comment) } } @@ -1001,7 +1390,7 @@ func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) { var updated arvados.Collection for _, fnm := range []string{"foo.txt", "bar.txt"} { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com" u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm) req := &http.Request{ Method: "DELETE", @@ -1013,7 +1402,7 @@ func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) { }, } resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusNoContent) updated = arvados.Collection{} @@ -1025,38 +1414,17 @@ func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) { c.Check(updated.ManifestText, check.Equals, "") } -func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) { - s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken - authHeader := http.Header{ - "Authorization": {"Bearer " + arvadostest.ManagementToken}, - } - - resp := httptest.NewRecorder() - u := mustParseURL("http://download.example.com/_health/ping") - req := &http.Request{ - Method: "GET", - Host: u.Host, - URL: u, - RequestURI: u.RequestURI(), - Header: authHeader, - } - s.testServer.Handler.ServeHTTP(resp, req) - - c.Check(resp.Code, check.Equals, http.StatusOK) - c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`) -} - func (s *IntegrationSuite) TestFileContentType(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" - client := s.testServer.Config.Client + client := arvados.NewClientFromEnv() client.AuthToken = arvadostest.ActiveToken - arv, err := arvadosclient.New(&client) + arv, err := arvadosclient.New(client) c.Assert(err, check.Equals, nil) kc, err := keepclient.MakeKeepClient(arv) c.Assert(err, check.Equals, nil) - fs, err := (&arvados.Collection{}).FileSystem(&client, kc) + fs, err := (&arvados.Collection{}).FileSystem(client, kc) c.Assert(err, check.IsNil) trials := []struct { @@ -1065,7 +1433,7 @@ func (s *IntegrationSuite) TestFileContentType(c *check.C) { contentType string }{ {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"}, - {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"}, + {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"}, {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"}, {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff @@ -1099,27 +1467,21 @@ func (s *IntegrationSuite) TestFileContentType(c *check.C) { }, } resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) - c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType) + c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType) c.Check(resp.Body.String(), check.Equals, trial.content) } } -func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) { - s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42 - c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42) - u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo") - req := &http.Request{ - Method: "GET", - Host: u.Host, - URL: u, - RequestURI: u.RequestURI(), - } +func (s *IntegrationSuite) TestCacheSize(c *check.C) { + req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil) + req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2) + c.Assert(err, check.IsNil) resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) - c.Check(resp.Code, check.Equals, http.StatusOK) - c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42) + s.handler.ServeHTTP(resp, req) + c.Assert(resp.Code, check.Equals, http.StatusOK) + c.Check(s.handler.Cache.sessions[arvadostest.ActiveTokenV2].client.DiskCacheSize.Percent(), check.Equals, int64(10)) } // Writing to a collection shouldn't affect its entry in the @@ -1142,7 +1504,7 @@ func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) { req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example" req.Host = req.URL.Host resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, status) } @@ -1168,7 +1530,7 @@ func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) { reqPut.Host = req.URL.Host reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata")) resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, &reqPut) + s.handler.ServeHTTP(resp, &reqPut) c.Check(resp.Code, check.Equals, http.StatusCreated) // new file should not appear in colls[1] @@ -1185,3 +1547,258 @@ func copyHeader(h http.Header) http.Header { } return hc } + +func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request, + successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) { + + client := arvados.NewClientFromEnv() + client.AuthToken = arvadostest.AdminToken + var logentries arvados.LogList + limit1 := 1 + err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil, + arvados.ResourceListParams{ + Limit: &limit1, + Order: "created_at desc"}) + c.Check(err, check.IsNil) + c.Check(logentries.Items, check.HasLen, 1) + lastLogId := logentries.Items[0].ID + c.Logf("lastLogId: %d", lastLogId) + + var logbuf bytes.Buffer + logger := logrus.New() + logger.Out = &logbuf + resp := httptest.NewRecorder() + req = req.WithContext(ctxlog.Context(context.Background(), logger)) + s.handler.ServeHTTP(resp, req) + + if perm { + c.Check(resp.Result().StatusCode, check.Equals, successCode) + c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`) + c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`) + + deadline := time.Now().Add(time.Second) + for { + c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry")) + logentries = arvados.LogList{} + err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil, + arvados.ResourceListParams{ + Filters: []arvados.Filter{ + {Attr: "event_type", Operator: "=", Operand: "file_" + direction}, + {Attr: "object_uuid", Operator: "=", Operand: userUuid}, + }, + Limit: &limit1, + Order: "created_at desc", + }) + c.Assert(err, check.IsNil) + if len(logentries.Items) > 0 && + logentries.Items[0].ID > lastLogId && + logentries.Items[0].ObjectUUID == userUuid && + logentries.Items[0].Properties["collection_uuid"] == collectionUuid && + (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) && + logentries.Items[0].Properties["collection_file_path"] == filepath { + break + } + c.Logf("logentries.Items: %+v", logentries.Items) + time.Sleep(50 * time.Millisecond) + } + } else { + c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden) + c.Check(logbuf.String(), check.Equals, "") + } +} + +func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) { + u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo") + + s.handler.Cluster.Collections.TrustAllContent = true + + for _, adminperm := range []bool{true, false} { + for _, userperm := range []bool{true, false} { + s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm + s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm + + // Test admin permission + req := &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.AdminToken}, + }, + } + s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm, + arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo") + + // Test user permission + req = &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + }, + } + s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm, + arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo") + } + } + + s.handler.Cluster.Collections.WebDAVPermission.User.Download = true + + for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1", + "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} { + + u = mustParseURL(tryurl) + req := &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + }, + } + s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true, + arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1") + } + + u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo") + req := &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + }, + } + s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true, + arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo") +} + +func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) { + for _, adminperm := range []bool{true, false} { + for _, userperm := range []bool{true, false} { + + arv := arvados.NewClientFromEnv() + arv.AuthToken = arvadostest.ActiveToken + + var coll arvados.Collection + err := arv.RequestAndDecode(&coll, + "POST", + "/arvados/v1/collections", + nil, + map[string]interface{}{ + "ensure_unique_name": true, + "collection": map[string]interface{}{ + "name": "test collection", + }, + }) + c.Assert(err, check.Equals, nil) + + u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar") + + s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm + s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm + + // Test admin permission + req := &http.Request{ + Method: "PUT", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.AdminToken}, + }, + Body: io.NopCloser(bytes.NewReader([]byte("bar"))), + } + s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm, + arvadostest.AdminUserUUID, coll.UUID, "", "bar") + + // Test user permission + req = &http.Request{ + Method: "PUT", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + }, + Body: io.NopCloser(bytes.NewReader([]byte("bar"))), + } + s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm, + arvadostest.ActiveUserUUID, coll.UUID, "", "bar") + } + } +} + +func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) { + s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2) + lockTidyInterval = time.Second + client := arvados.NewClientFromEnv() + client.AuthToken = arvadostest.ActiveTokenV2 + // Start small, and increase concurrency (2^2, 4^2, ...) + // only until hitting failure. Avoids unnecessarily long + // failure reports. + for n := 2; n < 16 && !c.Failed(); n = n * 2 { + c.Logf("%s: n=%d", c.TestName(), n) + + var coll arvados.Collection + err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil) + c.Assert(err, check.IsNil) + defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil) + + var wg sync.WaitGroup + for i := 0; i < n && !c.Failed(); i++ { + i := i + wg.Add(1) + go func() { + defer wg.Done() + u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i)) + resp := httptest.NewRecorder() + req, err := http.NewRequest("MKCOL", u.String(), nil) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + s.handler.ServeHTTP(resp, req) + c.Assert(resp.Code, check.Equals, http.StatusCreated) + for j := 0; j < n && !c.Failed(); j++ { + j := j + wg.Add(1) + go func() { + defer wg.Done() + content := fmt.Sprintf("i=%d/j=%d", i, j) + u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content) + + resp := httptest.NewRecorder() + req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content)) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + s.handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusCreated) + + time.Sleep(time.Second) + resp = httptest.NewRecorder() + req, err = http.NewRequest("GET", u.String(), nil) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + s.handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusOK) + c.Check(resp.Body.String(), check.Equals, content) + }() + } + }() + } + wg.Wait() + for i := 0; i < n; i++ { + u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i)) + resp := httptest.NewRecorder() + req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{}) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + s.handler.ServeHTTP(resp, req) + c.Assert(resp.Code, check.Equals, http.StatusMultiStatus) + } + } +}