X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/9a8b51c6c2468162ee9748514141a94d24e5f663..8041cc091d787764947c56eb5120da11bc2c139b:/services/keep-web/handler_test.go
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 15f32f1be4..5291efeb82 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -12,22 +12,38 @@ import (
"net/http"
"net/http/httptest"
"net/url"
+ "os"
"path/filepath"
"regexp"
"strings"
- "git.curoverse.com/arvados.git/sdk/go/arvadostest"
- "git.curoverse.com/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&UnitSuite{})
-type UnitSuite struct{}
+type UnitSuite struct {
+ Config *arvados.Config
+}
+
+func (s *UnitSuite) SetUpTest(c *check.C) {
+ ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
+ ldr.Path = "-"
+ cfg, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+ s.Config = cfg
+}
func (s *UnitSuite) TestCORSPreflight(c *check.C) {
- h := handler{Config: DefaultConfig()}
- u, _ := url.Parse("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
+ h := handler{Config: newConfig(s.Config)}
+ u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
req := &http.Request{
Method: "OPTIONS",
Host: u.Host,
@@ -45,8 +61,8 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) {
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
- c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PUT, RMCOL")
- c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range")
+ c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
+ c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
// Check preflight for a disallowed request
resp = httptest.NewRecorder()
@@ -57,7 +73,7 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) {
}
func (s *UnitSuite) TestInvalidUUID(c *check.C) {
- bogusID := strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "-"
+ bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
token := arvadostest.ActiveToken
for _, trial := range []string{
"http://keep-web/c=" + bogusID + "/foo",
@@ -68,8 +84,7 @@ func (s *UnitSuite) TestInvalidUUID(c *check.C) {
"http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
} {
c.Log(trial)
- u, err := url.Parse(trial)
- c.Assert(err, check.IsNil)
+ u := mustParseURL(trial)
req := &http.Request{
Method: "GET",
Host: u.Host,
@@ -77,8 +92,8 @@ func (s *UnitSuite) TestInvalidUUID(c *check.C) {
RequestURI: u.RequestURI(),
}
resp := httptest.NewRecorder()
- cfg := DefaultConfig()
- cfg.AnonymousTokens = []string{arvadostest.AnonymousToken}
+ cfg := newConfig(s.Config)
+ cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
h := handler{Config: cfg}
h.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
@@ -107,7 +122,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) {
}
s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
- c.Check(resp.Body.String(), check.Equals, "")
+ c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
}
}
@@ -185,8 +200,8 @@ func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
arvadostest.FooCollection + ".example.com/foo",
arvadostest.FooCollection + "--collections.example.com/foo",
arvadostest.FooCollection + "--collections.example.com/_/foo",
- arvadostest.FooPdh + ".example.com/foo",
- strings.Replace(arvadostest.FooPdh, "+", "-", -1) + "--collections.example.com/foo",
+ arvadostest.FooCollectionPDH + ".example.com/foo",
+ strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
} {
c.Log("doRequests: ", hostPath)
@@ -235,7 +250,11 @@ func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authori
// depending on the authz method.
c.Check(code, check.Equals, failCode)
}
- c.Check(body, check.Equals, "")
+ if code == 404 {
+ c.Check(body, check.Equals, notFoundMessage+"\n")
+ } else {
+ c.Check(body, check.Equals, unauthorizedMessage+"\n")
+ }
}
}
}
@@ -292,7 +311,7 @@ func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
"",
"",
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
@@ -306,7 +325,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C)
"",
"",
http.StatusUnauthorized,
- "",
+ unauthorizedMessage+"\n",
)
}
@@ -317,7 +336,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check
"",
"",
http.StatusBadRequest,
- "",
+ "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
)
}
@@ -333,11 +352,46 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check
http.StatusOK,
"foo",
)
- c.Check(strings.Split(resp.Header().Get("Content-Disposition"), ";")[0], check.Equals, "attachment")
+ c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
+}
+
+func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ resp := s.testVhostRedirectTokenToCookie(c, "GET",
+ "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
+ "?api_token="+arvadostest.ActiveToken,
+ "",
+ "",
+ http.StatusOK,
+ "foo",
+ )
+ c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
+}
+
+func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ resp := s.testVhostRedirectTokenToCookie(c, "GET",
+ "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
+ "?api_token="+arvadostest.ActiveToken,
+ "",
+ "",
+ http.StatusOK,
+ "waz",
+ )
+ c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
+ resp = s.testVhostRedirectTokenToCookie(c, "GET",
+ "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
+ "?api_token="+arvadostest.ActiveToken,
+ "",
+ "",
+ http.StatusOK,
+ "waz",
+ )
+ c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
- s.testServer.Config.TrustAllContent = true
+ s.testServer.Config.cluster.Collections.TrustAllContent = true
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
@@ -349,7 +403,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
- s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
@@ -357,7 +411,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *chec
"",
"",
http.StatusBadRequest,
- "",
+ "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
)
resp := s.testVhostRedirectTokenToCookie(c, "GET",
@@ -389,12 +443,12 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C)
"application/x-www-form-urlencoded",
url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
- s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
+ s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
@@ -406,17 +460,103 @@ func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
}
func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
- s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
+ s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
"",
"",
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
+func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+
+ client := s.testServer.Config.Client
+ client.AuthToken = arvadostest.ActiveToken
+ fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
+ c.Assert(err, check.IsNil)
+ f.Close()
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ var coll arvados.Collection
+ err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "manifest_text": mtxt,
+ },
+ })
+ c.Assert(err, check.IsNil)
+
+ u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + client.AuthToken},
+ },
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`)
+}
+
+func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
+ arv := arvados.NewClientFromEnv()
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
+ name := "foo/bar/baz"
+ nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
+ nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
+
+ client := s.testServer.Config.Client
+ client.AuthToken = arvadostest.ActiveToken
+ fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
+ c.Assert(err, check.IsNil)
+ f.Close()
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ var coll arvados.Collection
+ err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "manifest_text": mtxt,
+ "name": name,
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
+ c.Assert(err, check.IsNil)
+ defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
+
+ base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
+ for tryURL, expectRegexp := range map[string]string{
+ base: `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
+ base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
+ } {
+ u, _ := url.Parse(tryURL)
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + client.AuthToken},
+ },
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Matches, expectRegexp)
+ }
+}
+
// XHRs can't follow redirect-with-cookie so they rely on method=POST
// and disposition=attachment (telling us it's acceptable to respond
// with content instead of a redirect) and an Origin header that gets
@@ -443,6 +583,25 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "foo")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
+
+ // GET + Origin header is representative of both AJAX GET
+ // requests and inline images via 
.
+ u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
+ req = &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Origin": {"https://origin.example"},
+ },
+ }
+ resp = httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Equals, "foo")
+ c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
}
func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
@@ -466,7 +625,7 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
if resp.Code != http.StatusSeeOther {
return resp
}
- c.Check(resp.Body.String(), check.Matches, `.*href="//`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
+ c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
cookies := (&http.Response{Header: resp.Header()}).Cookies()
u, _ = u.Parse(resp.Header().Get("Location"))
@@ -487,8 +646,18 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
return resp
}
-func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
- s.testServer.Config.AttachmentOnlyHost = "download.example.com"
+func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
+ s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.testDirectoryListing(c)
+}
+
+func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
+ s.testServer.Config.cluster.Users.AnonymousUserToken = ""
+ s.testDirectoryListing(c)
+}
+
+func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
authHeader := http.Header{
"Authorization": {"OAuth2 " + arvadostest.ActiveToken},
}
@@ -512,10 +681,12 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 1,
},
{
- uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
- header: authHeader,
- expect: []string{"dir1/foo", "dir1/bar"},
- cutDirs: 2,
+ // URLs of this form ignore authHeader, and
+ // FooAndBarFilesInDirUUID isn't public, so
+ // this returns 404.
+ uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+ header: authHeader,
+ expect: nil,
},
{
uri: "download.example.com/users/active/foo_file_in_dir/",
@@ -610,8 +781,20 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
header: authHeader,
expect: nil,
},
+ {
+ uri: "download.example.com/c=" + arvadostest.WazVersion1Collection,
+ header: authHeader,
+ expect: []string{"waz"},
+ cutDirs: 1,
+ },
+ {
+ uri: "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
+ header: authHeader,
+ expect: []string{"waz"},
+ cutDirs: 2,
+ },
} {
- c.Logf("HTML: %q => %q", trial.uri, trial.expect)
+ comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
resp := httptest.NewRecorder()
u := mustParseURL("//" + trial.uri)
req := &http.Request{
@@ -619,7 +802,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
}
s.testServer.Handler.ServeHTTP(resp, req)
var cookies []*http.Cookie
@@ -630,7 +813,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
}
cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
for _, c := range cookies {
@@ -640,33 +823,33 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
s.testServer.Handler.ServeHTTP(resp, req)
}
if trial.redirect != "" {
- c.Check(req.URL.Path, check.Equals, trial.redirect)
+ c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
}
if trial.expect == nil {
- c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
} else {
- c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Code, check.Equals, http.StatusOK, comment)
for _, e := range trial.expect {
- c.Check(resp.Body.String(), check.Matches, `(?ms).*href="`+e+`".*`)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
}
- c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
}
- c.Logf("WebDAV: %q => %q", trial.uri, trial.expect)
+ comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
req = &http.Request{
Method: "OPTIONS",
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
s.testServer.Handler.ServeHTTP(resp, req)
if trial.expect == nil {
- c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
} else {
- c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Code, check.Equals, http.StatusOK, comment)
}
req = &http.Request{
@@ -674,24 +857,69 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
s.testServer.Handler.ServeHTTP(resp, req)
if trial.expect == nil {
- c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
} else {
- c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
+ c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
for _, e := range trial.expect {
- c.Check(resp.Body.String(), check.Matches, `(?ms).*`+filepath.Join(u.Path, e)+`.*`)
+ if strings.HasSuffix(e, "/") {
+ e = filepath.Join(u.Path, e) + "/"
+ } else {
+ e = filepath.Join(u.Path, e)
+ }
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*`+e+`.*`, comment)
}
}
}
}
+func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
+ arv := arvados.NewClientFromEnv()
+ var newCollection arvados.Collection
+ err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "owner_uuid": arvadostest.ActiveUserUUID,
+ "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
+ "name": "keep-web test collection",
+ },
+ "ensure_unique_name": true,
+ })
+ c.Assert(err, check.IsNil)
+ defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
+
+ var updated arvados.Collection
+ for _, fnm := range []string{"foo.txt", "bar.txt"} {
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
+ u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
+ req := &http.Request{
+ Method: "DELETE",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.ActiveToken},
+ },
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusNoContent)
+
+ updated = arvados.Collection{}
+ err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
+ c.Check(err, check.IsNil)
+ c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
+ c.Logf("updated manifest_text %q", updated.ManifestText)
+ }
+ c.Check(updated.ManifestText, check.Equals, "")
+}
+
func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
- s.testServer.Config.ManagementToken = arvadostest.ManagementToken
+ s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
authHeader := http.Header{
"Authorization": {"Bearer " + arvadostest.ManagementToken},
}
@@ -710,3 +938,87 @@ func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
}
+
+func (s *IntegrationSuite) TestFileContentType(c *check.C) {
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+
+ client := s.testServer.Config.Client
+ client.AuthToken = arvadostest.ActiveToken
+ arv, err := arvadosclient.New(&client)
+ c.Assert(err, check.Equals, nil)
+ kc, err := keepclient.MakeKeepClient(arv)
+ c.Assert(err, check.Equals, nil)
+
+ fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
+ c.Assert(err, check.IsNil)
+
+ trials := []struct {
+ filename string
+ content string
+ contentType string
+ }{
+ {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
+ {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
+ {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
+ {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp
+ {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
+ }
+ for _, trial := range trials {
+ f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write([]byte(trial.content))
+ c.Assert(err, check.IsNil)
+ c.Assert(f.Close(), check.IsNil)
+ }
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ var coll arvados.Collection
+ err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "manifest_text": mtxt,
+ },
+ })
+ c.Assert(err, check.IsNil)
+
+ for _, trial := range trials {
+ u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + client.AuthToken},
+ },
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
+ c.Check(resp.Body.String(), check.Equals, trial.content)
+ }
+}
+
+func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
+ s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
+ c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
+ u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
+}
+
+func copyHeader(h http.Header) http.Header {
+ hc := http.Header{}
+ for k, v := range h {
+ hc[k] = append([]string(nil), v...)
+ }
+ return hc
+}