X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/aa1c0f3049f7b78e7590dde868b915bef9a7ebbe..d6e8bf4f423aa174f38b552161dd7bc2dd1ecb18:/services/keep-web/handler_test.go diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go index 8e2e05c761..8715ab24f3 100644 --- a/services/keep-web/handler_test.go +++ b/services/keep-web/handler_test.go @@ -6,6 +6,7 @@ package main import ( "bytes" + "context" "fmt" "html" "io/ioutil" @@ -16,6 +17,7 @@ import ( "path/filepath" "regexp" "strings" + "time" "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/sdk/go/arvados" @@ -24,11 +26,16 @@ import ( "git.arvados.org/arvados.git/sdk/go/auth" "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/keepclient" + "github.com/sirupsen/logrus" check "gopkg.in/check.v1" ) var _ = check.Suite(&UnitSuite{}) +func init() { + arvados.DebugLocksPanicMode = true +} + type UnitSuite struct { Config *arvados.Config } @@ -72,6 +79,64 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) { c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed) } +func (s *UnitSuite) TestEmptyResponse(c *check.C) { + for _, trial := range []struct { + dataExists bool + sendIMSHeader bool + expectStatus int + logRegexp string + }{ + // If we return no content due to a Keep read error, + // we should emit a log message. + {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`}, + + // If we return no content because the client sent an + // If-Modified-Since header, our response should be + // 304, and we should not emit a log message. + {true, true, http.StatusNotModified, ``}, + } { + c.Logf("trial: %+v", trial) + arvadostest.StartKeep(2, true) + if trial.dataExists { + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, check.IsNil) + arv.ApiToken = arvadostest.ActiveToken + kc, err := keepclient.MakeKeepClient(arv) + c.Assert(err, check.IsNil) + _, _, err = kc.PutB([]byte("foo")) + c.Assert(err, check.IsNil) + } + + h := handler{Config: newConfig(s.Config)} + u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo") + req := &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + }, + } + if trial.sendIMSHeader { + req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1)) + } + + var logbuf bytes.Buffer + logger := logrus.New() + logger.Out = &logbuf + req = req.WithContext(ctxlog.Context(context.Background(), logger)) + + resp := httptest.NewRecorder() + h.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, trial.expectStatus) + c.Check(resp.Body.String(), check.Equals, "") + + c.Log(logbuf.String()) + c.Check(logbuf.String(), check.Matches, trial.logRegexp) + } +} + func (s *UnitSuite) TestInvalidUUID(c *check.C) { bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-" token := arvadostest.ActiveToken @@ -132,11 +197,18 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) { // the token is invalid. type authorizer func(*http.Request, string) int -func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) { - s.doVhostRequests(c, authzViaAuthzHeader) +func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) { + s.doVhostRequests(c, authzViaAuthzHeaderOAuth2) +} +func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int { + r.Header.Add("Authorization", "Bearer "+tok) + return http.StatusUnauthorized +} +func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) { + s.doVhostRequests(c, authzViaAuthzHeaderBearer) } -func authzViaAuthzHeader(r *http.Request, tok string) int { - r.Header.Add("Authorization", "OAuth2 "+tok) +func authzViaAuthzHeaderBearer(r *http.Request, tok string) int { + r.Header.Add("Authorization", "Bearer "+tok) return http.StatusUnauthorized } @@ -237,7 +309,6 @@ func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authori if tok == arvadostest.ActiveToken { c.Check(code, check.Equals, http.StatusOK) c.Check(body, check.Equals, "foo") - } else { c.Check(code >= 400, check.Equals, true) c.Check(code < 500, check.Equals, true) @@ -259,6 +330,30 @@ func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authori } } +func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) { + for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} { + for _, port := range []string{"80", "443", "8000"} { + s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port) + u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection)) + req := &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}}, + } + req, resp := s.doReq(req) + code, _ := resp.Code, resp.Body.String() + + if port == "8000" { + c.Check(code, check.Equals, 401) + } else { + c.Check(code, check.Equals, 200) + } + } + } +} + func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) { resp := httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) @@ -583,6 +678,25 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) { c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Equals, "foo") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") + + // GET + Origin header is representative of both AJAX GET + // requests and inline images via . + u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2) + req = &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Origin": {"https://origin.example"}, + }, + } + resp = httptest.NewRecorder() + s.testServer.Handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusOK) + c.Check(resp.Body.String(), check.Equals, "foo") + c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") } func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder { @@ -664,7 +778,7 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { { // URLs of this form ignore authHeader, and // FooAndBarFilesInDirUUID isn't public, so - // this returns 404. + // this returns 401. uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/", header: authHeader, expect: nil, @@ -807,7 +921,11 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { c.Check(req.URL.Path, check.Equals, trial.redirect, comment) } if trial.expect == nil { - c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) + if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { + c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) + } else { + c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) + } } else { c.Check(resp.Code, check.Equals, http.StatusOK, comment) for _, e := range trial.expect { @@ -828,7 +946,11 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { resp = httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) if trial.expect == nil { - c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) + if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { + c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) + } else { + c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) + } } else { c.Check(resp.Code, check.Equals, http.StatusOK, comment) } @@ -844,7 +966,11 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { resp = httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) if trial.expect == nil { - c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) + if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { + c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) + } else { + c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) + } } else { c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment) for _, e := range trial.expect { @@ -996,6 +1122,62 @@ func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) { c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42) } +// Writing to a collection shouldn't affect its entry in the +// PDH-to-manifest cache. +func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) { + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, check.Equals, nil) + arv.ApiToken = arvadostest.ActiveToken + + u := mustParseURL("http://x.example/testfile") + req := &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{"Authorization": {"Bearer " + arv.ApiToken}}, + } + + checkWithID := func(id string, status int) { + req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example" + req.Host = req.URL.Host + resp := httptest.NewRecorder() + s.testServer.Handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, status) + } + + var colls [2]arvados.Collection + for i := range colls { + err := arv.Create("collections", + map[string]interface{}{ + "ensure_unique_name": true, + "collection": map[string]interface{}{ + "name": "test collection", + }, + }, &colls[i]) + c.Assert(err, check.Equals, nil) + } + + // Populate cache with empty collection + checkWithID(colls[0].PortableDataHash, http.StatusNotFound) + + // write a file to colls[0] + reqPut := *req + reqPut.Method = "PUT" + reqPut.URL.Host = colls[0].UUID + ".example" + reqPut.Host = req.URL.Host + reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata")) + resp := httptest.NewRecorder() + s.testServer.Handler.ServeHTTP(resp, &reqPut) + c.Check(resp.Code, check.Equals, http.StatusCreated) + + // new file should not appear in colls[1] + checkWithID(colls[1].PortableDataHash, http.StatusNotFound) + checkWithID(colls[1].UUID, http.StatusNotFound) + + checkWithID(colls[0].UUID, http.StatusOK) +} + func copyHeader(h http.Header) http.Header { hc := http.Header{} for k, v := range h {