X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/08b07a1a27a19eecd70a09cf4b47727224a9d36d..4b58bfc5b58db18e4816102f9850757f0884a42e:/services/keep-web/handler_test.go diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go index 768013185a..4a76276392 100644 --- a/services/keep-web/handler_test.go +++ b/services/keep-web/handler_test.go @@ -18,6 +18,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "time" "git.arvados.org/arvados.git/lib/config" @@ -83,7 +84,7 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) { c.Check(resp.Body.String(), check.Equals, "") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK") - c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout") + c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control") // Check preflight for a disallowed request resp = httptest.NewRecorder() @@ -93,6 +94,120 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) { c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed) } +func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) { + for _, trial := range []struct { + method string + path string + prefix string + source string + notFound bool + seeOther bool + }{ + { + method: "PROPFIND", + path: "/", + }, + { + method: "PROPFIND", + path: "/dir1", + }, + { + method: "PROPFIND", + path: "/dir1/", + }, + { + method: "PROPFIND", + path: "/dir1/foo", + prefix: "/dir1", + source: "/dir1", + }, + { + method: "PROPFIND", + path: "/prefix/dir1/foo", + prefix: "/prefix/", + source: "", + }, + { + method: "PROPFIND", + path: "/prefix/dir1/foo", + prefix: "/prefix", + source: "", + }, + { + method: "PROPFIND", + path: "/prefix/dir1/foo", + prefix: "/prefix/", + source: "/", + }, + { + method: "PROPFIND", + path: "/prefix/foo", + prefix: "/prefix/", + source: "/dir1/", + }, + { + method: "GET", + path: "/prefix/foo", + prefix: "/prefix/", + source: "/dir1/", + }, + { + method: "PROPFIND", + path: "/prefix/", + prefix: "/prefix", + source: "/dir1", + }, + { + method: "PROPFIND", + path: "/prefix", + prefix: "/prefix", + source: "/dir1/", + }, + { + method: "GET", + path: "/prefix", + prefix: "/prefix", + source: "/dir1", + seeOther: true, + }, + { + method: "PROPFIND", + path: "/dir1/foo", + prefix: "", + source: "/dir1", + notFound: true, + }, + } { + c.Logf("trial %+v", trial) + u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path) + req := &http.Request{ + Method: trial.method, + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveTokenV2}, + "X-Webdav-Prefix": {trial.prefix}, + "X-Webdav-Source": {trial.source}, + }, + Body: ioutil.NopCloser(bytes.NewReader(nil)), + } + + resp := httptest.NewRecorder() + s.handler.ServeHTTP(resp, req) + if trial.notFound { + c.Check(resp.Code, check.Equals, http.StatusNotFound) + } else if trial.method == "PROPFIND" { + c.Check(resp.Code, check.Equals, http.StatusMultiStatus) + c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`) + } else if trial.seeOther { + c.Check(resp.Code, check.Equals, http.StatusSeeOther) + } else { + c.Check(resp.Code, check.Equals, http.StatusOK) + } + } +} + func (s *UnitSuite) TestEmptyResponse(c *check.C) { for _, trial := range []struct { dataExists bool @@ -366,6 +481,24 @@ func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) { } } +func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) { + u := mustParseURL(urlstring) + if hdr == nil && token != "" { + hdr = http.Header{"Authorization": {"Bearer " + token}} + } else if hdr == nil { + hdr = http.Header{} + } else if token != "" { + panic("must not pass both token and hdr") + } + return s.doReq(&http.Request{ + Method: method, + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: hdr, + }) +} + func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) { resp := httptest.NewRecorder() s.handler.ServeHTTP(resp, req) @@ -409,6 +542,26 @@ func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) { ) } +func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) { + s.testVhostRedirectTokenToCookie(c, "GET", + "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo", + "", + nil, + "", + http.StatusOK, + "foo", + ) + // Same valid sharing token, but requesting a different collection + s.testVhostRedirectTokenToCookie(c, "GET", + "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo", + "", + nil, + "", + http.StatusNotFound, + regexp.QuoteMeta(notFoundMessage+"\n"), + ) +} + // Bad token in URL is 404 Not Found because it doesn't make sense to // retry the same URL with different authorization. func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) { @@ -418,7 +571,7 @@ func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) { nil, "", http.StatusNotFound, - notFoundMessage+"\n", + regexp.QuoteMeta(notFoundMessage+"\n"), ) } @@ -481,7 +634,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) http.Header{"Sec-Fetch-Mode": {"cors"}}, "", http.StatusUnauthorized, - unauthorizedMessage+"\n", + regexp.QuoteMeta(unauthorizedMessage+"\n"), ) s.testVhostRedirectTokenToCookie(c, "GET", s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo", @@ -489,10 +642,65 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) nil, "", http.StatusUnauthorized, - unauthorizedMessage+"\n", + regexp.QuoteMeta(unauthorizedMessage+"\n"), ) } +func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) { + resp := s.testVhostRedirectTokenToCookie(c, "GET", + arvadostest.FooCollection+".example.com/foo", + "?api_token=thisisabogustoken", + http.Header{ + "Sec-Fetch-Mode": {"navigate"}, + "Cache-Control": {"no-cache"}, + }, + "", + http.StatusSeeOther, + "", + ) + u, err := url.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Logf("redirected to %s", u) + c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host) + c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo") + c.Check(u.Query().Get("redirectToDownload"), check.Equals, "") +} + +func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) { + for _, trial := range []struct { + anonToken bool + cacheControl string + }{ + {}, + {cacheControl: "no-cache"}, + {anonToken: true}, + {anonToken: true, cacheControl: "no-cache"}, + } { + c.Logf("trial: %+v", trial) + + if trial.anonToken { + s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken + } else { + s.handler.Cluster.Users.AnonymousUserToken = "" + } + req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil) + c.Assert(err, check.IsNil) + req.Header.Set("Sec-Fetch-Mode", "navigate") + if trial.cacheControl != "" { + req.Header.Set("Cache-Control", trial.cacheControl) + } + resp := httptest.NewRecorder() + s.handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusSeeOther) + u, err := url.Parse(resp.Header().Get("Location")) + c.Assert(err, check.IsNil) + c.Logf("redirected to %q", u) + c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host) + c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo") + c.Check(u.Query().Get("redirectToDownload"), check.Equals, "") + } +} + func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", @@ -500,7 +708,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check nil, "", http.StatusBadRequest, - "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n", + regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"), ) } @@ -575,7 +783,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *chec nil, "", http.StatusBadRequest, - "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n", + regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"), ) resp := s.testVhostRedirectTokenToCookie(c, "GET", @@ -607,7 +815,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(), http.StatusNotFound, - notFoundMessage+"\n", + regexp.QuoteMeta(notFoundMessage+"\n"), ) } @@ -630,8 +838,8 @@ func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) { "", nil, "", - http.StatusNotFound, - notFoundMessage+"\n", + http.StatusUnauthorized, + "Authorization tokens are not accepted here: .*\n", ) } @@ -768,7 +976,7 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) { c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") } -func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder { +func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder { if reqHeader == nil { reqHeader = http.Header{} } @@ -786,7 +994,7 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho resp := httptest.NewRecorder() defer func() { c.Check(resp.Code, check.Equals, expectStatus) - c.Check(resp.Body.String(), check.Equals, expectRespBody) + c.Check(resp.Body.String(), check.Matches, matchRespBody) }() s.handler.ServeHTTP(resp, req) @@ -1000,11 +1208,7 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { c.Check(req.URL.Path, check.Equals, trial.redirect, comment) } if trial.expect == nil { - if s.handler.Cluster.Users.AnonymousUserToken == "" { - c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) - } else { - c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) - } + c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusOK, comment) for _, e := range trial.expect { @@ -1025,11 +1229,7 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { resp = httptest.NewRecorder() s.handler.ServeHTTP(resp, req) if trial.expect == nil { - if s.handler.Cluster.Users.AnonymousUserToken == "" { - c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) - } else { - c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) - } + c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusOK, comment) } @@ -1045,11 +1245,7 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) { resp = httptest.NewRecorder() s.handler.ServeHTTP(resp, req) if trial.expect == nil { - if s.handler.Cluster.Users.AnonymousUserToken == "" { - c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) - } else { - c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) - } + c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment) for _, e := range trial.expect { @@ -1245,7 +1441,7 @@ func copyHeader(h http.Header) http.Header { } func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request, - successCode int, direction string, perm bool, userUuid string, collectionUuid string, filepath string) { + successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) { client := arvados.NewClientFromEnv() client.AuthToken = arvadostest.AdminToken @@ -1258,6 +1454,7 @@ func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Requ c.Check(err, check.IsNil) c.Check(logentries.Items, check.HasLen, 1) lastLogId := logentries.Items[0].ID + c.Logf("lastLogId: %d", lastLogId) var logbuf bytes.Buffer logger := logrus.New() @@ -1274,6 +1471,7 @@ func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Requ deadline := time.Now().Add(time.Second) for { c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry")) + logentries = arvados.LogList{} err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil, arvados.ResourceListParams{ Filters: []arvados.Filter{ @@ -1288,6 +1486,7 @@ func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Requ logentries.Items[0].ID > lastLogId && logentries.Items[0].ObjectUUID == userUuid && logentries.Items[0].Properties["collection_uuid"] == collectionUuid && + (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) && logentries.Items[0].Properties["collection_file_path"] == filepath { break } @@ -1321,7 +1520,7 @@ func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) { }, } s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm, - arvadostest.AdminUserUUID, arvadostest.FooCollection, "foo") + arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo") // Test user permission req = &http.Request{ @@ -1334,7 +1533,7 @@ func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) { }, } s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm, - arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo") + arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo") } } @@ -1354,7 +1553,7 @@ func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) { }, } s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true, - arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, "dir1/subdir/file1") + arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1") } u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo") @@ -1368,7 +1567,7 @@ func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) { }, } s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true, - arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo") + arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo") } func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) { @@ -1408,7 +1607,7 @@ func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) { Body: io.NopCloser(bytes.NewReader([]byte("bar"))), } s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm, - arvadostest.AdminUserUUID, coll.UUID, "bar") + arvadostest.AdminUserUUID, coll.UUID, "", "bar") // Test user permission req = &http.Request{ @@ -1422,7 +1621,76 @@ func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) { Body: io.NopCloser(bytes.NewReader([]byte("bar"))), } s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm, - arvadostest.ActiveUserUUID, coll.UUID, "bar") + arvadostest.ActiveUserUUID, coll.UUID, "", "bar") + } + } +} + +func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) { + s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2) + lockTidyInterval = time.Second + client := arvados.NewClientFromEnv() + client.AuthToken = arvadostest.ActiveTokenV2 + // Start small, and increase concurrency (2^2, 4^2, ...) + // only until hitting failure. Avoids unnecessarily long + // failure reports. + for n := 2; n < 16 && !c.Failed(); n = n * 2 { + c.Logf("%s: n=%d", c.TestName(), n) + + var coll arvados.Collection + err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil) + c.Assert(err, check.IsNil) + defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil) + + var wg sync.WaitGroup + for i := 0; i < n && !c.Failed(); i++ { + i := i + wg.Add(1) + go func() { + defer wg.Done() + u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i)) + resp := httptest.NewRecorder() + req, err := http.NewRequest("MKCOL", u.String(), nil) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + s.handler.ServeHTTP(resp, req) + c.Assert(resp.Code, check.Equals, http.StatusCreated) + for j := 0; j < n && !c.Failed(); j++ { + j := j + wg.Add(1) + go func() { + defer wg.Done() + content := fmt.Sprintf("i=%d/j=%d", i, j) + u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content) + + resp := httptest.NewRecorder() + req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content)) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + s.handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusCreated) + + time.Sleep(time.Second) + resp = httptest.NewRecorder() + req, err = http.NewRequest("GET", u.String(), nil) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + s.handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusOK) + c.Check(resp.Body.String(), check.Equals, content) + }() + } + }() + } + wg.Wait() + for i := 0; i < n; i++ { + u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i)) + resp := httptest.NewRecorder() + req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{}) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + s.handler.ServeHTTP(resp, req) + c.Assert(resp.Code, check.Equals, http.StatusMultiStatus) } } }