X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/4164f38ef0f900962e3c365745c59d5c20a63c66..086fdccf436bb68d38f482c4e1418c1290ed7c0c:/services/keep-web/handler_test.go
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 29bcdac1b7..07c7016d3a 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -2,12 +2,14 @@
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
+ "context"
"fmt"
"html"
+ "io"
"io/ioutil"
"net/http"
"net/http/httptest"
@@ -16,50 +18,54 @@ import (
"path/filepath"
"regexp"
"strings"
+ "sync"
+ "time"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/keepclient"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&UnitSuite{})
+func init() {
+ arvados.DebugLocksPanicMode = true
+}
+
type UnitSuite struct {
- Config *arvados.Config
+ cluster *arvados.Cluster
+ handler *handler
}
func (s *UnitSuite) SetUpTest(c *check.C) {
- ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
+ logger := ctxlog.TestLogger(c)
+ ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
ldr.Path = "-"
cfg, err := ldr.Load()
c.Assert(err, check.IsNil)
- s.Config = cfg
-}
-
-func (s *UnitSuite) TestKeepClientBlockCache(c *check.C) {
- cfg := newConfig(s.Config)
- cfg.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
- h := handler{Config: cfg}
- c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
- u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
- req := &http.Request{
- Method: "GET",
- Host: u.Host,
- URL: u,
- RequestURI: u.RequestURI(),
+ cc, err := cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ s.cluster = cc
+ s.handler = &handler{
+ Cluster: cc,
+ Cache: cache{
+ cluster: cc,
+ logger: logger,
+ registry: prometheus.NewRegistry(),
+ },
+ metrics: newMetrics(prometheus.NewRegistry()),
}
- resp := httptest.NewRecorder()
- h.ServeHTTP(resp, req)
- c.Check(resp.Code, check.Equals, http.StatusOK)
- c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
}
func (s *UnitSuite) TestCORSPreflight(c *check.C) {
- h := handler{Config: newConfig(s.Config)}
+ h := s.handler
u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
req := &http.Request{
Method: "OPTIONS",
@@ -79,7 +85,7 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) {
c.Check(resp.Body.String(), check.Equals, "")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
- c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
+ c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
// Check preflight for a disallowed request
resp = httptest.NewRecorder()
@@ -89,6 +95,182 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) {
c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
}
+func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
+ for _, trial := range []struct {
+ method string
+ path string
+ prefix string
+ source string
+ notFound bool
+ seeOther bool
+ }{
+ {
+ method: "PROPFIND",
+ path: "/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1",
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1/foo",
+ prefix: "/dir1",
+ source: "/dir1",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/dir1/foo",
+ prefix: "/prefix/",
+ source: "",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/dir1/foo",
+ prefix: "/prefix",
+ source: "",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/dir1/foo",
+ prefix: "/prefix/",
+ source: "/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/foo",
+ prefix: "/prefix/",
+ source: "/dir1/",
+ },
+ {
+ method: "GET",
+ path: "/prefix/foo",
+ prefix: "/prefix/",
+ source: "/dir1/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/",
+ prefix: "/prefix",
+ source: "/dir1",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix",
+ prefix: "/prefix",
+ source: "/dir1/",
+ },
+ {
+ method: "GET",
+ path: "/prefix",
+ prefix: "/prefix",
+ source: "/dir1",
+ seeOther: true,
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1/foo",
+ prefix: "",
+ source: "/dir1",
+ notFound: true,
+ },
+ } {
+ c.Logf("trial %+v", trial)
+ u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
+ req := &http.Request{
+ Method: trial.method,
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.ActiveTokenV2},
+ "X-Webdav-Prefix": {trial.prefix},
+ "X-Webdav-Source": {trial.source},
+ },
+ Body: ioutil.NopCloser(bytes.NewReader(nil)),
+ }
+
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ if trial.notFound {
+ c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ } else if trial.method == "PROPFIND" {
+ c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
+ } else if trial.seeOther {
+ c.Check(resp.Code, check.Equals, http.StatusSeeOther)
+ } else {
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ }
+ }
+}
+
+func (s *UnitSuite) TestEmptyResponse(c *check.C) {
+ // Ensure we start with an empty cache
+ defer os.Setenv("HOME", os.Getenv("HOME"))
+ os.Setenv("HOME", c.MkDir())
+
+ for _, trial := range []struct {
+ dataExists bool
+ sendIMSHeader bool
+ expectStatus int
+ logRegexp string
+ }{
+ // If we return no content due to a Keep read error,
+ // we should emit a log message.
+ {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
+
+ // If we return no content because the client sent an
+ // If-Modified-Since header, our response should be
+ // 304. We still expect a "File download" log since it
+ // counts as a file access for auditing.
+ {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
+ } {
+ c.Logf("trial: %+v", trial)
+ arvadostest.StartKeep(2, true)
+ if trial.dataExists {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.IsNil)
+ arv.ApiToken = arvadostest.ActiveToken
+ kc, err := keepclient.MakeKeepClient(arv)
+ c.Assert(err, check.IsNil)
+ _, _, err = kc.PutB([]byte("foo"))
+ c.Assert(err, check.IsNil)
+ }
+
+ u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.ActiveToken},
+ },
+ }
+ if trial.sendIMSHeader {
+ req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
+ }
+
+ var logbuf bytes.Buffer
+ logger := logrus.New()
+ logger.Out = &logbuf
+ req = req.WithContext(ctxlog.Context(context.Background(), logger))
+
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, trial.expectStatus)
+ c.Check(resp.Body.String(), check.Equals, "")
+
+ c.Log(logbuf.String())
+ c.Check(logbuf.String(), check.Matches, trial.logRegexp)
+ }
+}
+
func (s *UnitSuite) TestInvalidUUID(c *check.C) {
bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
token := arvadostest.ActiveToken
@@ -109,10 +291,8 @@ func (s *UnitSuite) TestInvalidUUID(c *check.C) {
RequestURI: u.RequestURI(),
}
resp := httptest.NewRecorder()
- cfg := newConfig(s.Config)
- cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
- h := handler{Config: cfg}
- h.ServeHTTP(resp, req)
+ s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
}
}
@@ -137,9 +317,9 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) {
URL: u,
RequestURI: u.RequestURI(),
}
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
- c.Check(resp.Body.String(), check.Equals, "")
+ c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
}
}
@@ -149,14 +329,22 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) {
// the token is invalid.
type authorizer func(*http.Request, string) int
-func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
- s.doVhostRequests(c, authzViaAuthzHeader)
+func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
+ s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
}
-func authzViaAuthzHeader(r *http.Request, tok string) int {
+func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
r.Header.Add("Authorization", "OAuth2 "+tok)
return http.StatusUnauthorized
}
+func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
+ s.doVhostRequests(c, authzViaAuthzHeaderBearer)
+}
+func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
+ r.Header.Add("Authorization", "Bearer "+tok)
+ return http.StatusUnauthorized
+}
+
func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
s.doVhostRequests(c, authzViaCookieValue)
}
@@ -168,6 +356,27 @@ func authzViaCookieValue(r *http.Request, tok string) int {
return http.StatusUnauthorized
}
+func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
+ s.doVhostRequests(c, authzViaHTTPBasicAuth)
+}
+func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
+ r.AddCookie(&http.Cookie{
+ Name: "arvados_api_token",
+ Value: auth.EncodeTokenCookie([]byte(tok)),
+ })
+ return http.StatusUnauthorized
+}
+
+func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
+ s.doVhostRequests(c, func(r *http.Request, tok string) int {
+ r.AddCookie(&http.Cookie{
+ Name: "arvados_api_token",
+ Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
+ })
+ return http.StatusUnauthorized
+ })
+}
+
func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
s.doVhostRequests(c, authzViaPath)
}
@@ -254,7 +463,6 @@ func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authori
if tok == arvadostest.ActiveToken {
c.Check(code, check.Equals, http.StatusOK)
c.Check(body, check.Equals, "foo")
-
} else {
c.Check(code >= 400, check.Equals, true)
c.Check(code < 500, check.Equals, true)
@@ -267,14 +475,60 @@ func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authori
// depending on the authz method.
c.Check(code, check.Equals, failCode)
}
- c.Check(body, check.Equals, "")
+ if code == 404 {
+ c.Check(body, check.Equals, notFoundMessage+"\n")
+ } else {
+ c.Check(body, check.Equals, unauthorizedMessage+"\n")
+ }
+ }
+ }
+}
+
+func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
+ for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
+ for _, port := range []string{"80", "443", "8000"} {
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
+ u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
+ }
+ req, resp := s.doReq(req)
+ code, _ := resp.Code, resp.Body.String()
+
+ if port == "8000" {
+ c.Check(code, check.Equals, 401)
+ } else {
+ c.Check(code, check.Equals, 200)
+ }
}
}
}
+func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
+ u := mustParseURL(urlstring)
+ if hdr == nil && token != "" {
+ hdr = http.Header{"Authorization": {"Bearer " + token}}
+ } else if hdr == nil {
+ hdr = http.Header{}
+ } else if token != "" {
+ panic("must not pass both token and hdr")
+ }
+ return s.doReq(&http.Request{
+ Method: method,
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: hdr,
+ })
+}
+
func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
return req, resp
}
@@ -297,7 +551,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "GET",
arvadostest.FooCollection+".example.com/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"foo",
@@ -308,11 +562,31 @@ func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
"",
+ nil,
+ "",
+ http.StatusOK,
+ "foo",
+ )
+}
+
+func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
"",
+ nil,
"",
http.StatusOK,
"foo",
)
+ // Same valid sharing token, but requesting a different collection
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
+ "",
+ nil,
+ "",
+ http.StatusNotFound,
+ regexp.QuoteMeta(notFoundMessage+"\n"),
+ )
}
// Bad token in URL is 404 Not Found because it doesn't make sense to
@@ -321,35 +595,147 @@ func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
"",
- "",
+ nil,
"",
http.StatusNotFound,
- "",
+ regexp.QuoteMeta(notFoundMessage+"\n"),
)
}
// Bad token in a cookie (even if it got there via our own
-// query-string-to-cookie redirect) is, in principle, retryable at the
-// same URL so it's 401 Unauthorized.
+// query-string-to-cookie redirect) is, in principle, retryable via
+// wb2-login-and-redirect flow.
func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
- s.testVhostRedirectTokenToCookie(c, "GET",
+ // Inline
+ resp := s.testVhostRedirectTokenToCookie(c, "GET",
+ arvadostest.FooCollection+".example.com/foo",
+ "?api_token=thisisabogustoken",
+ http.Header{"Sec-Fetch-Mode": {"navigate"}},
+ "",
+ http.StatusSeeOther,
+ "",
+ )
+ u, err := url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Logf("redirected to %s", u)
+ c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
+ c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
+ c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
+
+ // Download/attachment indicated by ?disposition=attachment
+ resp = s.testVhostRedirectTokenToCookie(c, "GET",
arvadostest.FooCollection+".example.com/foo",
+ "?api_token=thisisabogustoken&disposition=attachment",
+ http.Header{"Sec-Fetch-Mode": {"navigate"}},
+ "",
+ http.StatusSeeOther,
+ "",
+ )
+ u, err = url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Logf("redirected to %s", u)
+ c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
+ c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
+ c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
+
+ // Download/attachment indicated by vhost
+ resp = s.testVhostRedirectTokenToCookie(c, "GET",
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
+ "?api_token=thisisabogustoken",
+ http.Header{"Sec-Fetch-Mode": {"navigate"}},
+ "",
+ http.StatusSeeOther,
+ "",
+ )
+ u, err = url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Logf("redirected to %s", u)
+ c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
+ c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
+ c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
+
+ // Without "Sec-Fetch-Mode: navigate" header, just 401.
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
"?api_token=thisisabogustoken",
+ http.Header{"Sec-Fetch-Mode": {"cors"}},
"",
+ http.StatusUnauthorized,
+ regexp.QuoteMeta(unauthorizedMessage+"\n"),
+ )
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
+ "?api_token=thisisabogustoken",
+ nil,
"",
http.StatusUnauthorized,
+ regexp.QuoteMeta(unauthorizedMessage+"\n"),
+ )
+}
+
+func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
+ resp := s.testVhostRedirectTokenToCookie(c, "GET",
+ arvadostest.FooCollection+".example.com/foo",
+ "?api_token=thisisabogustoken",
+ http.Header{
+ "Sec-Fetch-Mode": {"navigate"},
+ "Cache-Control": {"no-cache"},
+ },
+ "",
+ http.StatusSeeOther,
"",
)
+ u, err := url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Logf("redirected to %s", u)
+ c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
+ c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
+ c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
+}
+
+func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
+ for _, trial := range []struct {
+ anonToken bool
+ cacheControl string
+ }{
+ {},
+ {cacheControl: "no-cache"},
+ {anonToken: true},
+ {anonToken: true, cacheControl: "no-cache"},
+ } {
+ c.Logf("trial: %+v", trial)
+
+ if trial.anonToken {
+ s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ } else {
+ s.handler.Cluster.Users.AnonymousUserToken = ""
+ }
+ req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Sec-Fetch-Mode", "navigate")
+ if trial.cacheControl != "" {
+ req.Header.Set("Cache-Control", trial.cacheControl)
+ }
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusSeeOther)
+ u, err := url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Logf("redirected to %q", u)
+ c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
+ c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
+ c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
+ }
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusBadRequest,
- "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
+ regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
)
}
@@ -360,7 +746,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check
resp := s.testVhostRedirectTokenToCookie(c, "GET",
arvadostest.FooCollection+".example.com/foo",
"?disposition=attachment&api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"foo",
@@ -369,11 +755,11 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
resp := s.testVhostRedirectTokenToCookie(c, "GET",
"download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"foo",
@@ -382,11 +768,11 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
}
func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
resp := s.testVhostRedirectTokenToCookie(c, "GET",
"download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"waz",
@@ -395,7 +781,7 @@ func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
resp = s.testVhostRedirectTokenToCookie(c, "GET",
"download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"waz",
@@ -404,11 +790,11 @@ func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
- s.testServer.Config.cluster.Collections.TrustAllContent = true
+ s.handler.Cluster.Collections.TrustAllContent = true
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"foo",
@@ -416,21 +802,21 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusBadRequest,
- "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
+ regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
)
resp := s.testVhostRedirectTokenToCookie(c, "GET",
"example.com:1234/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"foo",
@@ -438,11 +824,39 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *chec
c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
}
+func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
+ baseUrl := arvadostest.FooCollection + ".example.com/foo"
+ query := url.Values{}
+
+ // The intent of these tests is to check that requests are redirected
+ // correctly in the presence of multiple API tokens. The exact response
+ // codes and content are not closely considered: they're just how
+ // keep-web responded when we made the smallest possible fix. Changing
+ // those responses may be okay, but you should still test all these
+ // different cases and the associated redirect logic.
+ query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
+ query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
+ query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
+ query["api_token"] = []string{"", arvadostest.ActiveToken}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
+
+ expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
+ query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
+ query["api_token"] = []string{arvadostest.AnonymousToken, ""}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
+ query["api_token"] = []string{"", arvadostest.AnonymousToken}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
+}
+
func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "POST",
arvadostest.FooCollection+".example.com/foo",
"",
- "application/x-www-form-urlencoded",
+ http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
http.StatusOK,
"foo",
@@ -453,19 +867,19 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C)
s.testVhostRedirectTokenToCookie(c, "POST",
arvadostest.FooCollection+".example.com/foo",
"",
- "application/x-www-form-urlencoded",
+ http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
http.StatusNotFound,
- "",
+ regexp.QuoteMeta(notFoundMessage+"\n"),
)
}
func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
- "",
+ nil,
"",
http.StatusOK,
"Hello world\n",
@@ -473,23 +887,23 @@ func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
}
func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
+ s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
+ nil,
"",
- "",
- http.StatusNotFound,
- "",
+ http.StatusUnauthorized,
+ "Authorization tokens are not accepted here: .*\n",
)
}
func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
- client := s.testServer.Config.Client
+ client := arvados.NewClientFromEnv()
client.AuthToken = arvadostest.ActiveToken
- fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ fs, err := (&arvados.Collection{}).FileSystem(client, nil)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
c.Assert(err, check.IsNil)
@@ -515,22 +929,22 @@ func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`)
}
func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
arv := arvados.NewClientFromEnv()
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
- s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
name := "foo/bar/baz"
nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
- client := s.testServer.Config.Client
+ client := arvados.NewClientFromEnv()
client.AuthToken = arvadostest.ActiveToken
- fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ fs, err := (&arvados.Collection{}).FileSystem(client, nil)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
c.Assert(err, check.IsNil)
@@ -564,7 +978,7 @@ func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Matches, expectRegexp)
}
@@ -592,66 +1006,122 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
}.Encode())),
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Equals, "foo")
+ c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
+
+ // GET + Origin header is representative of both AJAX GET
+ // requests and inline images via .
+ u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
+ req = &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Origin": {"https://origin.example"},
+ },
+ }
+ resp = httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "foo")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
}
-func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
+func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
+ if reqHeader == nil {
+ reqHeader = http.Header{}
+ }
u, _ := url.Parse(`http://` + hostPath + queryString)
+ c.Logf("requesting %s", u)
req := &http.Request{
Method: method,
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: http.Header{"Content-Type": {contentType}},
+ Header: reqHeader,
Body: ioutil.NopCloser(strings.NewReader(reqBody)),
}
resp := httptest.NewRecorder()
defer func() {
c.Check(resp.Code, check.Equals, expectStatus)
- c.Check(resp.Body.String(), check.Equals, expectRespBody)
+ c.Check(resp.Body.String(), check.Matches, matchRespBody)
}()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
+ attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
+ // Since we're not redirecting, check that any api_token in the URL is
+ // handled safely.
+ // If there is no token in the URL, then we're good.
+ // Otherwise, if the response code is an error, the body is expected to
+ // be static content, and nothing that might maliciously introspect the
+ // URL. It's considered safe and allowed.
+ // Otherwise, if the response content has attachment disposition,
+ // that's considered safe for all the reasons explained in the
+ // safeAttachment comment in handler.go.
+ c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
return resp
}
+
+ loc, err := url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Check(loc.Scheme, check.Equals, u.Scheme)
+ c.Check(loc.Host, check.Equals, u.Host)
+ c.Check(loc.RawPath, check.Equals, u.RawPath)
+ // If the response was a redirect, it should never include an API token.
+ c.Check(loc.Query().Has("api_token"), check.Equals, false)
c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
cookies := (&http.Response{Header: resp.Header()}).Cookies()
- u, _ = u.Parse(resp.Header().Get("Location"))
+ c.Logf("following redirect to %s", u)
req = &http.Request{
Method: "GET",
- Host: u.Host,
- URL: u,
- RequestURI: u.RequestURI(),
- Header: http.Header{},
+ Host: loc.Host,
+ URL: loc,
+ RequestURI: loc.RequestURI(),
+ Header: reqHeader,
}
for _, c := range cookies {
req.AddCookie(c)
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
- c.Check(resp.Header().Get("Location"), check.Equals, "")
+ s.handler.ServeHTTP(resp, req)
+
+ if resp.Code != http.StatusSeeOther {
+ c.Check(resp.Header().Get("Location"), check.Equals, "")
+ }
return resp
}
func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
s.testDirectoryListing(c)
}
func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = ""
+ s.handler.Cluster.Users.AnonymousUserToken = ""
s.testDirectoryListing(c)
}
func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ // The "ownership cycle" test fixtures are reachable from the
+ // "filter group without filters" group, causing webdav's
+ // walkfs to recurse indefinitely. Avoid that by deleting one
+ // of the bogus fixtures.
+ arv := arvados.NewClientFromEnv()
+ err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil)
+ if err != nil {
+ c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{})
+ c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
+ }
+
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
authHeader := http.Header{
"Authorization": {"OAuth2 " + arvadostest.ActiveToken},
}
@@ -677,7 +1147,7 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
{
// URLs of this form ignore authHeader, and
// FooAndBarFilesInDirUUID isn't public, so
- // this returns 404.
+ // this returns 401.
uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
header: authHeader,
expect: nil,
@@ -787,8 +1257,32 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
expect: []string{"waz"},
cutDirs: 2,
},
+ {
+ uri: "download.example.com/users/active/This filter group/",
+ header: authHeader,
+ expect: []string{"A Subproject/"},
+ cutDirs: 3,
+ },
+ {
+ uri: "download.example.com/users/active/This filter group/A Subproject",
+ header: authHeader,
+ expect: []string{"baz_file/"},
+ cutDirs: 4,
+ },
+ {
+ uri: "download.example.com/by_id/" + arvadostest.AFilterGroupUUID,
+ header: authHeader,
+ expect: []string{"A Subproject/"},
+ cutDirs: 2,
+ },
+ {
+ uri: "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject",
+ header: authHeader,
+ expect: []string{"baz_file/"},
+ cutDirs: 3,
+ },
} {
- comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
+ comment := check.Commentf("HTML: %q redir %q => %q", trial.uri, trial.redirect, trial.expect)
resp := httptest.NewRecorder()
u := mustParseURL("//" + trial.uri)
req := &http.Request{
@@ -798,7 +1292,7 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
RequestURI: u.RequestURI(),
Header: copyHeader(trial.header),
}
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
var cookies []*http.Cookie
for resp.Code == http.StatusSeeOther {
u, _ := req.URL.Parse(resp.Header().Get("Location"))
@@ -814,16 +1308,17 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
req.AddCookie(c)
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
}
if trial.redirect != "" {
c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
}
if trial.expect == nil {
- c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
+ c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusOK, comment)
for _, e := range trial.expect {
+ e = strings.Replace(e, " ", "%20", -1)
c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
}
c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
@@ -839,9 +1334,9 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if trial.expect == nil {
- c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
+ c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusOK, comment)
}
@@ -855,9 +1350,15 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
+ // This check avoids logging a big XML document in the
+ // event webdav throws a 500 error after sending
+ // headers for a 207.
+ if !c.Check(strings.HasSuffix(resp.Body.String(), "Internal Server Error"), check.Equals, false) {
+ continue
+ }
if trial.expect == nil {
- c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
+ c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
for _, e := range trial.expect {
@@ -866,6 +1367,7 @@ func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
} else {
e = filepath.Join(u.Path, e)
}
+ e = strings.Replace(e, " ", "%20", -1)
c.Check(resp.Body.String(), check.Matches, `(?ms).*`+e+`.*`, comment)
}
}
@@ -888,7 +1390,7 @@ func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
var updated arvados.Collection
for _, fnm := range []string{"foo.txt", "bar.txt"} {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
req := &http.Request{
Method: "DELETE",
@@ -900,7 +1402,7 @@ func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNoContent)
updated = arvados.Collection{}
@@ -912,25 +1414,130 @@ func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
c.Check(updated.ManifestText, check.Equals, "")
}
-func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
- s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
- authHeader := http.Header{
- "Authorization": {"Bearer " + arvadostest.ManagementToken},
+func (s *IntegrationSuite) TestFileContentType(c *check.C) {
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+
+ client := arvados.NewClientFromEnv()
+ client.AuthToken = arvadostest.ActiveToken
+ arv, err := arvadosclient.New(client)
+ c.Assert(err, check.Equals, nil)
+ kc, err := keepclient.MakeKeepClient(arv)
+ c.Assert(err, check.Equals, nil)
+
+ fs, err := (&arvados.Collection{}).FileSystem(client, kc)
+ c.Assert(err, check.IsNil)
+
+ trials := []struct {
+ filename string
+ content string
+ contentType string
+ }{
+ {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
+ {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
+ {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
+ {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp
+ {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
+ }
+ for _, trial := range trials {
+ f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write([]byte(trial.content))
+ c.Assert(err, check.IsNil)
+ c.Assert(f.Close(), check.IsNil)
}
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ var coll arvados.Collection
+ err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "manifest_text": mtxt,
+ },
+ })
+ c.Assert(err, check.IsNil)
+ for _, trial := range trials {
+ u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + client.AuthToken},
+ },
+ }
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
+ c.Check(resp.Body.String(), check.Equals, trial.content)
+ }
+}
+
+func (s *IntegrationSuite) TestCacheSize(c *check.C) {
+ req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+ c.Assert(err, check.IsNil)
resp := httptest.NewRecorder()
- u := mustParseURL("http://download.example.com/_health/ping")
+ s.handler.ServeHTTP(resp, req)
+ c.Assert(resp.Code, check.Equals, http.StatusOK)
+ c.Check(s.handler.Cache.sessions[arvadostest.ActiveTokenV2].client.DiskCacheSize.Percent(), check.Equals, int64(10))
+}
+
+// Writing to a collection shouldn't affect its entry in the
+// PDH-to-manifest cache.
+func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+ arv.ApiToken = arvadostest.ActiveToken
+
+ u := mustParseURL("http://x.example/testfile")
req := &http.Request{
Method: "GET",
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: authHeader,
+ Header: http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
}
- s.testServer.Handler.ServeHTTP(resp, req)
- c.Check(resp.Code, check.Equals, http.StatusOK)
- c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
+ checkWithID := func(id string, status int) {
+ req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
+ req.Host = req.URL.Host
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, status)
+ }
+
+ var colls [2]arvados.Collection
+ for i := range colls {
+ err := arv.Create("collections",
+ map[string]interface{}{
+ "ensure_unique_name": true,
+ "collection": map[string]interface{}{
+ "name": "test collection",
+ },
+ }, &colls[i])
+ c.Assert(err, check.Equals, nil)
+ }
+
+ // Populate cache with empty collection
+ checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
+
+ // write a file to colls[0]
+ reqPut := *req
+ reqPut.Method = "PUT"
+ reqPut.URL.Host = colls[0].UUID + ".example"
+ reqPut.Host = req.URL.Host
+ reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, &reqPut)
+ c.Check(resp.Code, check.Equals, http.StatusCreated)
+
+ // new file should not appear in colls[1]
+ checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
+ checkWithID(colls[1].UUID, http.StatusNotFound)
+
+ checkWithID(colls[0].UUID, http.StatusOK)
}
func copyHeader(h http.Header) http.Header {
@@ -940,3 +1547,258 @@ func copyHeader(h http.Header) http.Header {
}
return hc
}
+
+func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
+ successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
+
+ client := arvados.NewClientFromEnv()
+ client.AuthToken = arvadostest.AdminToken
+ var logentries arvados.LogList
+ limit1 := 1
+ err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
+ arvados.ResourceListParams{
+ Limit: &limit1,
+ Order: "created_at desc"})
+ c.Check(err, check.IsNil)
+ c.Check(logentries.Items, check.HasLen, 1)
+ lastLogId := logentries.Items[0].ID
+ c.Logf("lastLogId: %d", lastLogId)
+
+ var logbuf bytes.Buffer
+ logger := logrus.New()
+ logger.Out = &logbuf
+ resp := httptest.NewRecorder()
+ req = req.WithContext(ctxlog.Context(context.Background(), logger))
+ s.handler.ServeHTTP(resp, req)
+
+ if perm {
+ c.Check(resp.Result().StatusCode, check.Equals, successCode)
+ c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
+ c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
+
+ deadline := time.Now().Add(time.Second)
+ for {
+ c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
+ logentries = arvados.LogList{}
+ err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
+ arvados.ResourceListParams{
+ Filters: []arvados.Filter{
+ {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
+ {Attr: "object_uuid", Operator: "=", Operand: userUuid},
+ },
+ Limit: &limit1,
+ Order: "created_at desc",
+ })
+ c.Assert(err, check.IsNil)
+ if len(logentries.Items) > 0 &&
+ logentries.Items[0].ID > lastLogId &&
+ logentries.Items[0].ObjectUUID == userUuid &&
+ logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
+ (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
+ logentries.Items[0].Properties["collection_file_path"] == filepath {
+ break
+ }
+ c.Logf("logentries.Items: %+v", logentries.Items)
+ time.Sleep(50 * time.Millisecond)
+ }
+ } else {
+ c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
+ c.Check(logbuf.String(), check.Equals, "")
+ }
+}
+
+func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
+ u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
+
+ s.handler.Cluster.Collections.TrustAllContent = true
+
+ for _, adminperm := range []bool{true, false} {
+ for _, userperm := range []bool{true, false} {
+ s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
+ s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
+
+ // Test admin permission
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.AdminToken},
+ },
+ }
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
+ arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
+
+ // Test user permission
+ req = &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.ActiveToken},
+ },
+ }
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
+ arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
+ }
+ }
+
+ s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
+
+ for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
+ "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
+
+ u = mustParseURL(tryurl)
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.ActiveToken},
+ },
+ }
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
+ arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
+ }
+
+ u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.ActiveToken},
+ },
+ }
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
+ arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
+}
+
+func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
+ for _, adminperm := range []bool{true, false} {
+ for _, userperm := range []bool{true, false} {
+
+ arv := arvados.NewClientFromEnv()
+ arv.AuthToken = arvadostest.ActiveToken
+
+ var coll arvados.Collection
+ err := arv.RequestAndDecode(&coll,
+ "POST",
+ "/arvados/v1/collections",
+ nil,
+ map[string]interface{}{
+ "ensure_unique_name": true,
+ "collection": map[string]interface{}{
+ "name": "test collection",
+ },
+ })
+ c.Assert(err, check.Equals, nil)
+
+ u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
+
+ s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
+ s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
+
+ // Test admin permission
+ req := &http.Request{
+ Method: "PUT",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.AdminToken},
+ },
+ Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
+ }
+ s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
+ arvadostest.AdminUserUUID, coll.UUID, "", "bar")
+
+ // Test user permission
+ req = &http.Request{
+ Method: "PUT",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.ActiveToken},
+ },
+ Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
+ }
+ s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
+ arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
+ }
+ }
+}
+
+func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
+ s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
+ lockTidyInterval = time.Second
+ client := arvados.NewClientFromEnv()
+ client.AuthToken = arvadostest.ActiveTokenV2
+ // Start small, and increase concurrency (2^2, 4^2, ...)
+ // only until hitting failure. Avoids unnecessarily long
+ // failure reports.
+ for n := 2; n < 16 && !c.Failed(); n = n * 2 {
+ c.Logf("%s: n=%d", c.TestName(), n)
+
+ var coll arvados.Collection
+ err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
+ c.Assert(err, check.IsNil)
+ defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
+
+ var wg sync.WaitGroup
+ for i := 0; i < n && !c.Failed(); i++ {
+ i := i
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
+ resp := httptest.NewRecorder()
+ req, err := http.NewRequest("MKCOL", u.String(), nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ s.handler.ServeHTTP(resp, req)
+ c.Assert(resp.Code, check.Equals, http.StatusCreated)
+ for j := 0; j < n && !c.Failed(); j++ {
+ j := j
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ content := fmt.Sprintf("i=%d/j=%d", i, j)
+ u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
+
+ resp := httptest.NewRecorder()
+ req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusCreated)
+
+ time.Sleep(time.Second)
+ resp = httptest.NewRecorder()
+ req, err = http.NewRequest("GET", u.String(), nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Equals, content)
+ }()
+ }
+ }()
+ }
+ wg.Wait()
+ for i := 0; i < n; i++ {
+ u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
+ resp := httptest.NewRecorder()
+ req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ s.handler.ServeHTTP(resp, req)
+ c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
+ }
+ }
+}