X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c8bf4dc1eef2a73b4b01501eabdc922e2a27a276..08b07a1a27a19eecd70a09cf4b47727224a9d36d:/services/keep-web/handler_test.go diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go index 55c122b0ff..768013185a 100644 --- a/services/keep-web/handler_test.go +++ b/services/keep-web/handler_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepweb import ( "bytes" @@ -27,6 +27,7 @@ import ( "git.arvados.org/arvados.git/sdk/go/auth" "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/keepclient" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" check "gopkg.in/check.v1" ) @@ -38,19 +39,31 @@ func init() { } type UnitSuite struct { - Config *arvados.Config + cluster *arvados.Cluster + handler *handler } func (s *UnitSuite) SetUpTest(c *check.C) { - ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c)) + logger := ctxlog.TestLogger(c) + ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger) ldr.Path = "-" cfg, err := ldr.Load() c.Assert(err, check.IsNil) - s.Config = cfg + cc, err := cfg.GetCluster("") + c.Assert(err, check.IsNil) + s.cluster = cc + s.handler = &handler{ + Cluster: cc, + Cache: cache{ + cluster: cc, + logger: logger, + registry: prometheus.NewRegistry(), + }, + } } func (s *UnitSuite) TestCORSPreflight(c *check.C) { - h := handler{Config: newConfig(ctxlog.TestLogger(c), s.Config)} + h := s.handler u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo") req := &http.Request{ Method: "OPTIONS", @@ -109,7 +122,6 @@ func (s *UnitSuite) TestEmptyResponse(c *check.C) { c.Assert(err, check.IsNil) } - h := handler{Config: newConfig(ctxlog.TestLogger(c), s.Config)} u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo") req := &http.Request{ Method: "GET", @@ -130,7 +142,7 @@ func (s *UnitSuite) TestEmptyResponse(c *check.C) { req = req.WithContext(ctxlog.Context(context.Background(), logger)) resp := httptest.NewRecorder() - h.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, trial.expectStatus) c.Check(resp.Body.String(), check.Equals, "") @@ -159,10 +171,8 @@ func (s *UnitSuite) TestInvalidUUID(c *check.C) { RequestURI: u.RequestURI(), } resp := httptest.NewRecorder() - cfg := newConfig(ctxlog.TestLogger(c), s.Config) - cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken - h := handler{Config: cfg} - h.ServeHTTP(resp, req) + s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusNotFound) } } @@ -187,7 +197,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) { URL: u, RequestURI: u.RequestURI(), } - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusNotFound) c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n") } @@ -335,7 +345,7 @@ func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authori func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) { for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} { for _, port := range []string{"80", "443", "8000"} { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port) + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port) u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection)) req := &http.Request{ Method: "GET", @@ -358,7 +368,7 @@ func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) { func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) { resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) if resp.Code != http.StatusSeeOther { return req, resp } @@ -381,7 +391,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", arvadostest.FooCollection+".example.com/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "foo", @@ -392,7 +402,7 @@ func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo", "", - "", + nil, "", http.StatusOK, "foo", @@ -405,7 +415,7 @@ func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo", "", - "", + nil, "", http.StatusNotFound, notFoundMessage+"\n", @@ -413,13 +423,70 @@ func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) { } // Bad token in a cookie (even if it got there via our own -// query-string-to-cookie redirect) is, in principle, retryable at the -// same URL so it's 401 Unauthorized. +// query-string-to-cookie redirect) is, in principle, retryable via +// wb2-login-and-redirect flow. func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) { - s.testVhostRedirectTokenToCookie(c, "GET", + // Inline + resp := s.testVhostRedirectTokenToCookie(c, "GET", arvadostest.FooCollection+".example.com/foo", "?api_token=thisisabogustoken", + http.Header{"Sec-Fetch-Mode": {"navigate"}}, + "", + http.StatusSeeOther, "", + ) + u, err := url.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Logf("redirected to %s", u) + c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host) + c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo") + c.Check(u.Query().Get("redirectToDownload"), check.Equals, "") + + // Download/attachment indicated by ?disposition=attachment + resp = s.testVhostRedirectTokenToCookie(c, "GET", + arvadostest.FooCollection+".example.com/foo", + "?api_token=thisisabogustoken&disposition=attachment", + http.Header{"Sec-Fetch-Mode": {"navigate"}}, + "", + http.StatusSeeOther, + "", + ) + u, err = url.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Logf("redirected to %s", u) + c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host) + c.Check(u.Query().Get("redirectToPreview"), check.Equals, "") + c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo") + + // Download/attachment indicated by vhost + resp = s.testVhostRedirectTokenToCookie(c, "GET", + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo", + "?api_token=thisisabogustoken", + http.Header{"Sec-Fetch-Mode": {"navigate"}}, + "", + http.StatusSeeOther, + "", + ) + u, err = url.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Logf("redirected to %s", u) + c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host) + c.Check(u.Query().Get("redirectToPreview"), check.Equals, "") + c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo") + + // Without "Sec-Fetch-Mode: navigate" header, just 401. + s.testVhostRedirectTokenToCookie(c, "GET", + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo", + "?api_token=thisisabogustoken", + http.Header{"Sec-Fetch-Mode": {"cors"}}, + "", + http.StatusUnauthorized, + unauthorizedMessage+"\n", + ) + s.testVhostRedirectTokenToCookie(c, "GET", + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo", + "?api_token=thisisabogustoken", + nil, "", http.StatusUnauthorized, unauthorizedMessage+"\n", @@ -430,7 +497,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusBadRequest, "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n", @@ -444,7 +511,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check resp := s.testVhostRedirectTokenToCookie(c, "GET", arvadostest.FooCollection+".example.com/foo", "?disposition=attachment&api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "foo", @@ -453,11 +520,11 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check } func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" resp := s.testVhostRedirectTokenToCookie(c, "GET", "download.example.com/by_id/"+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "foo", @@ -466,11 +533,11 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) { } func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" resp := s.testVhostRedirectTokenToCookie(c, "GET", "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "waz", @@ -479,7 +546,7 @@ func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) { resp = s.testVhostRedirectTokenToCookie(c, "GET", "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "waz", @@ -488,11 +555,11 @@ func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) { } func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) { - s.testServer.Config.cluster.Collections.TrustAllContent = true + s.handler.Cluster.Collections.TrustAllContent = true s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "foo", @@ -500,12 +567,12 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C } func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234" s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusBadRequest, "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n", @@ -514,7 +581,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *chec resp := s.testVhostRedirectTokenToCookie(c, "GET", "example.com:1234/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, - "", + nil, "", http.StatusOK, "foo", @@ -526,7 +593,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) { s.testVhostRedirectTokenToCookie(c, "POST", arvadostest.FooCollection+".example.com/foo", "", - "application/x-www-form-urlencoded", + http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(), http.StatusOK, "foo", @@ -537,7 +604,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) s.testVhostRedirectTokenToCookie(c, "POST", arvadostest.FooCollection+".example.com/foo", "", - "application/x-www-form-urlencoded", + http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(), http.StatusNotFound, notFoundMessage+"\n", @@ -545,11 +612,11 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) } func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) { - s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken + s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt", "", - "", + nil, "", http.StatusOK, "Hello world\n", @@ -557,11 +624,11 @@ func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) { } func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) { - s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid" + s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid" s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt", "", - "", + nil, "", http.StatusNotFound, notFoundMessage+"\n", @@ -569,11 +636,11 @@ func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) { } func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" - client := s.testServer.Config.Client + client := arvados.NewClientFromEnv() client.AuthToken = arvadostest.ActiveToken - fs, err := (&arvados.Collection{}).FileSystem(&client, nil) + fs, err := (&arvados.Collection{}).FileSystem(client, nil) c.Assert(err, check.IsNil) f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777) c.Assert(err, check.IsNil) @@ -599,22 +666,22 @@ func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) { }, } resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`) } func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) { arv := arvados.NewClientFromEnv() - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" - s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}" name := "foo/bar/baz" nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1) nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1) - client := s.testServer.Config.Client + client := arvados.NewClientFromEnv() client.AuthToken = arvadostest.ActiveToken - fs, err := (&arvados.Collection{}).FileSystem(&client, nil) + fs, err := (&arvados.Collection{}).FileSystem(client, nil) c.Assert(err, check.IsNil) f, err := fs.OpenFile("filename", os.O_CREATE, 0777) c.Assert(err, check.IsNil) @@ -648,7 +715,7 @@ func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) { }, } resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Matches, expectRegexp) } @@ -676,7 +743,7 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) { }.Encode())), } resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Equals, "foo") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") @@ -695,20 +762,24 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) { }, } resp = httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Equals, "foo") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") } -func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder { +func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder { + if reqHeader == nil { + reqHeader = http.Header{} + } u, _ := url.Parse(`http://` + hostPath + queryString) + c.Logf("requesting %s", u) req := &http.Request{ Method: method, Host: u.Host, URL: u, RequestURI: u.RequestURI(), - Header: http.Header{"Content-Type": {contentType}}, + Header: reqHeader, Body: ioutil.NopCloser(strings.NewReader(reqBody)), } @@ -718,43 +789,49 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho c.Check(resp.Body.String(), check.Equals, expectRespBody) }() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) if resp.Code != http.StatusSeeOther { return resp } c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`) + c.Check(strings.Split(resp.Header().Get("Location"), "?")[0], check.Equals, "http://"+hostPath) cookies := (&http.Response{Header: resp.Header()}).Cookies() - u, _ = u.Parse(resp.Header().Get("Location")) + u, err := u.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Logf("following redirect to %s", u) req = &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), - Header: http.Header{}, + Header: reqHeader, } for _, c := range cookies { req.AddCookie(c) } resp = httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) - c.Check(resp.Header().Get("Location"), check.Equals, "") + s.handler.ServeHTTP(resp, req) + + if resp.Code != http.StatusSeeOther { + c.Check(resp.Header().Get("Location"), check.Equals, "") + } return resp } func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) { - s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken + s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken s.testDirectoryListing(c) } func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) { - s.testServer.Config.cluster.Users.AnonymousUserToken = "" + s.handler.Cluster.Users.AnonymousUserToken = "" s.testDirectoryListing(c) } func (s *IntegrationSuite) testDirectoryListing(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" authHeader := http.Header{ "Authorization": {"OAuth2 " + arvadostest.ActiveToken}, } @@ -901,7 +978,7 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { RequestURI: u.RequestURI(), Header: copyHeader(trial.header), } - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) var cookies []*http.Cookie for resp.Code == http.StatusSeeOther { u, _ := req.URL.Parse(resp.Header().Get("Location")) @@ -917,13 +994,13 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { req.AddCookie(c) } resp = httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) } if trial.redirect != "" { c.Check(req.URL.Path, check.Equals, trial.redirect, comment) } if trial.expect == nil { - if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { + if s.handler.Cluster.Users.AnonymousUserToken == "" { c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) @@ -946,9 +1023,9 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { Body: ioutil.NopCloser(&bytes.Buffer{}), } resp = httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) if trial.expect == nil { - if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { + if s.handler.Cluster.Users.AnonymousUserToken == "" { c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) @@ -966,9 +1043,9 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { Body: ioutil.NopCloser(&bytes.Buffer{}), } resp = httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) if trial.expect == nil { - if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { + if s.handler.Cluster.Users.AnonymousUserToken == "" { c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) @@ -1003,7 +1080,7 @@ func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) { var updated arvados.Collection for _, fnm := range []string{"foo.txt", "bar.txt"} { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com" u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm) req := &http.Request{ Method: "DELETE", @@ -1015,7 +1092,7 @@ func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) { }, } resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusNoContent) updated = arvados.Collection{} @@ -1027,38 +1104,17 @@ func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) { c.Check(updated.ManifestText, check.Equals, "") } -func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) { - s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken - authHeader := http.Header{ - "Authorization": {"Bearer " + arvadostest.ManagementToken}, - } - - resp := httptest.NewRecorder() - u := mustParseURL("http://download.example.com/_health/ping") - req := &http.Request{ - Method: "GET", - Host: u.Host, - URL: u, - RequestURI: u.RequestURI(), - Header: authHeader, - } - s.testServer.Handler.ServeHTTP(resp, req) - - c.Check(resp.Code, check.Equals, http.StatusOK) - c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`) -} - func (s *IntegrationSuite) TestFileContentType(c *check.C) { - s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" - client := s.testServer.Config.Client + client := arvados.NewClientFromEnv() client.AuthToken = arvadostest.ActiveToken - arv, err := arvadosclient.New(&client) + arv, err := arvadosclient.New(client) c.Assert(err, check.Equals, nil) kc, err := keepclient.MakeKeepClient(arv) c.Assert(err, check.Equals, nil) - fs, err := (&arvados.Collection{}).FileSystem(&client, kc) + fs, err := (&arvados.Collection{}).FileSystem(client, kc) c.Assert(err, check.IsNil) trials := []struct { @@ -1101,7 +1157,7 @@ func (s *IntegrationSuite) TestFileContentType(c *check.C) { }, } resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType) c.Check(resp.Body.String(), check.Equals, trial.content) @@ -1109,7 +1165,7 @@ func (s *IntegrationSuite) TestFileContentType(c *check.C) { } func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) { - s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42 + s.handler.Cluster.Collections.WebDAVCache.MaxBlockEntries = 42 c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42) u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo") req := &http.Request{ @@ -1119,7 +1175,7 @@ func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) { RequestURI: u.RequestURI(), } resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42) } @@ -1144,7 +1200,7 @@ func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) { req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example" req.Host = req.URL.Host resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, status) } @@ -1170,7 +1226,7 @@ func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) { reqPut.Host = req.URL.Host reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata")) resp := httptest.NewRecorder() - s.testServer.Handler.ServeHTTP(resp, &reqPut) + s.handler.ServeHTTP(resp, &reqPut) c.Check(resp.Code, check.Equals, http.StatusCreated) // new file should not appear in colls[1] @@ -1188,10 +1244,10 @@ func copyHeader(h http.Header) http.Header { return hc } -func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, h *handler, req *http.Request, +func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request, successCode int, direction string, perm bool, userUuid string, collectionUuid string, filepath string) { - client := s.testServer.Config.Client + client := arvados.NewClientFromEnv() client.AuthToken = arvadostest.AdminToken var logentries arvados.LogList limit1 := 1 @@ -1202,38 +1258,42 @@ func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, h *handler, re c.Check(err, check.IsNil) c.Check(logentries.Items, check.HasLen, 1) lastLogId := logentries.Items[0].ID - nextLogId := lastLogId var logbuf bytes.Buffer logger := logrus.New() logger.Out = &logbuf resp := httptest.NewRecorder() req = req.WithContext(ctxlog.Context(context.Background(), logger)) - h.ServeHTTP(resp, req) + s.handler.ServeHTTP(resp, req) if perm { c.Check(resp.Result().StatusCode, check.Equals, successCode) c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`) c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`) - count := 0 - for ; nextLogId == lastLogId && count < 20; count++ { - time.Sleep(50 * time.Millisecond) + deadline := time.Now().Add(time.Second) + for { + c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry")) err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil, arvados.ResourceListParams{ - Filters: []arvados.Filter{arvados.Filter{Attr: "event_type", Operator: "=", Operand: "file_" + direction}}, - Limit: &limit1, - Order: "created_at desc", + Filters: []arvados.Filter{ + {Attr: "event_type", Operator: "=", Operand: "file_" + direction}, + {Attr: "object_uuid", Operator: "=", Operand: userUuid}, + }, + Limit: &limit1, + Order: "created_at desc", }) - c.Check(err, check.IsNil) - if len(logentries.Items) > 0 { - nextLogId = logentries.Items[0].ID + c.Assert(err, check.IsNil) + if len(logentries.Items) > 0 && + logentries.Items[0].ID > lastLogId && + logentries.Items[0].ObjectUUID == userUuid && + logentries.Items[0].Properties["collection_uuid"] == collectionUuid && + logentries.Items[0].Properties["collection_file_path"] == filepath { + break } + c.Logf("logentries.Items: %+v", logentries.Items) + time.Sleep(50 * time.Millisecond) } - c.Check(count, check.Not(check.Equals), 20) - c.Check(logentries.Items[0].ObjectUUID, check.Equals, userUuid) - c.Check(logentries.Items[0].Properties["collection_uuid"], check.Equals, collectionUuid) - c.Check(logentries.Items[0].Properties["collection_file_path"], check.Equals, filepath) } else { c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden) c.Check(logbuf.String(), check.Equals, "") @@ -1241,16 +1301,14 @@ func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, h *handler, re } func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) { - config := newConfig(ctxlog.TestLogger(c), s.ArvConfig) - h := handler{Config: config} u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo") - config.cluster.Collections.TrustAllContent = true + s.handler.Cluster.Collections.TrustAllContent = true for _, adminperm := range []bool{true, false} { for _, userperm := range []bool{true, false} { - config.cluster.Collections.WebDAVPermission.Admin.Download = adminperm - config.cluster.Collections.WebDAVPermission.User.Download = userperm + s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm + s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm // Test admin permission req := &http.Request{ @@ -1262,7 +1320,7 @@ func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) { "Authorization": {"Bearer " + arvadostest.AdminToken}, }, } - s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", adminperm, + s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm, arvadostest.AdminUserUUID, arvadostest.FooCollection, "foo") // Test user permission @@ -1275,12 +1333,12 @@ func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) { "Authorization": {"Bearer " + arvadostest.ActiveToken}, }, } - s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", userperm, + s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm, arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo") } } - config.cluster.Collections.WebDAVPermission.User.Download = true + s.handler.Cluster.Collections.WebDAVPermission.User.Download = true for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1", "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} { @@ -1295,7 +1353,7 @@ func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) { "Authorization": {"Bearer " + arvadostest.ActiveToken}, }, } - s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", true, + s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true, arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, "dir1/subdir/file1") } @@ -1309,18 +1367,15 @@ func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) { "Authorization": {"Bearer " + arvadostest.ActiveToken}, }, } - s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", true, + s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true, arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo") } func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) { - config := newConfig(ctxlog.TestLogger(c), s.ArvConfig) - h := handler{Config: config} - for _, adminperm := range []bool{true, false} { for _, userperm := range []bool{true, false} { - arv := s.testServer.Config.Client + arv := arvados.NewClientFromEnv() arv.AuthToken = arvadostest.ActiveToken var coll arvados.Collection @@ -1338,8 +1393,8 @@ func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) { u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar") - config.cluster.Collections.WebDAVPermission.Admin.Upload = adminperm - config.cluster.Collections.WebDAVPermission.User.Upload = userperm + s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm + s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm // Test admin permission req := &http.Request{ @@ -1352,7 +1407,7 @@ func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) { }, Body: io.NopCloser(bytes.NewReader([]byte("bar"))), } - s.checkUploadDownloadRequest(c, &h, req, http.StatusCreated, "upload", adminperm, + s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm, arvadostest.AdminUserUUID, coll.UUID, "bar") // Test user permission @@ -1366,7 +1421,7 @@ func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) { }, Body: io.NopCloser(bytes.NewReader([]byte("bar"))), } - s.checkUploadDownloadRequest(c, &h, req, http.StatusCreated, "upload", userperm, + s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm, arvadostest.ActiveUserUUID, coll.UUID, "bar") } }