// Copyright (C) The Arvados Authors. All rights reserved. // // SPDX-License-Identifier: AGPL-3.0 package main import ( "bytes" "context" "fmt" "html" "io" "io/ioutil" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" "regexp" "strings" "time" "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" "git.arvados.org/arvados.git/sdk/go/auth" "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/keepclient" "github.com/sirupsen/logrus" check "gopkg.in/check.v1" ) var _ = check.Suite(&UnitSuite{}) func init() { arvados.DebugLocksPanicMode = true } type UnitSuite struct { Config *arvados.Config } func (s *UnitSuite) SetUpTest(c *check.C) { ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c)) ldr.Path = "-" cfg, err := ldr.Load() c.Assert(err, check.IsNil) s.Config = cfg } func (s *UnitSuite) TestCORSPreflight(c *check.C) { h := handler{Config: newConfig(s.Config)} u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo") req := &http.Request{ Method: "OPTIONS", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Origin": {"https://workbench.example"}, "Access-Control-Request-Method": {"POST"}, }, } // Check preflight for an allowed request resp := httptest.NewRecorder() h.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Equals, "") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK") c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout") // Check preflight for a disallowed request resp = httptest.NewRecorder() req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE") h.ServeHTTP(resp, req) c.Check(resp.Body.String(), check.Equals, "") c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed) } func (s *UnitSuite) TestEmptyResponse(c *check.C) { for _, trial := range []struct { dataExists bool sendIMSHeader bool expectStatus int logRegexp string }{ // If we return no content due to a Keep read error, // we should emit a log message. {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`}, // If we return no content because the client sent an // If-Modified-Since header, our response should be // 304. We still expect a "File download" log since it // counts as a file access for auditing. {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`}, } { c.Logf("trial: %+v", trial) arvadostest.StartKeep(2, true) if trial.dataExists { arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, check.IsNil) arv.ApiToken = arvadostest.ActiveToken kc, err := keepclient.MakeKeepClient(arv) c.Assert(err, check.IsNil) _, _, err = kc.PutB([]byte("foo")) c.Assert(err, check.IsNil) } h := handler{Config: newConfig(s.Config)} u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo") req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Authorization": {"Bearer " + arvadostest.ActiveToken}, }, } if trial.sendIMSHeader { req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1)) } var logbuf bytes.Buffer logger := logrus.New() logger.Out = &logbuf req = req.WithContext(ctxlog.Context(context.Background(), logger)) resp := httptest.NewRecorder() h.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, trial.expectStatus) c.Check(resp.Body.String(), check.Equals, "") c.Log(logbuf.String()) c.Check(logbuf.String(), check.Matches, trial.logRegexp) } } func (s *UnitSuite) TestInvalidUUID(c *check.C) { bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-" token := arvadostest.ActiveToken for _, trial := range []string{ "http://keep-web/c=" + bogusID + "/foo", "http://keep-web/c=" + bogusID + "/t=" + token + "/foo", "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo", "http://keep-web/collections/" + bogusID + "/foo", "http://" + bogusID + ".keep-web/" + bogusID + "/foo", "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo", } { c.Log(trial) u := mustParseURL(trial) req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), } resp := httptest.NewRecorder() cfg := newConfig(s.Config) cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken h := handler{Config: cfg} h.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusNotFound) } } func mustParseURL(s string) *url.URL { r, err := url.Parse(s) if err != nil { panic("parse URL: " + s) } return r } func (s *IntegrationSuite) TestVhost404(c *check.C) { for _, testURL := range []string{ arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy", arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy", } { resp := httptest.NewRecorder() u := mustParseURL(testURL) req := &http.Request{ Method: "GET", URL: u, RequestURI: u.RequestURI(), } s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusNotFound) c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n") } } // An authorizer modifies an HTTP request to make use of the given // token -- by adding it to a header, cookie, query param, or whatever // -- and returns the HTTP status code we should expect from keep-web if // the token is invalid. type authorizer func(*http.Request, string) int func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) { s.doVhostRequests(c, authzViaAuthzHeaderOAuth2) } func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int { r.Header.Add("Authorization", "Bearer "+tok) return http.StatusUnauthorized } func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) { s.doVhostRequests(c, authzViaAuthzHeaderBearer) } func authzViaAuthzHeaderBearer(r *http.Request, tok string) int { r.Header.Add("Authorization", "Bearer "+tok) return http.StatusUnauthorized } func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) { s.doVhostRequests(c, authzViaCookieValue) } func authzViaCookieValue(r *http.Request, tok string) int { r.AddCookie(&http.Cookie{ Name: "arvados_api_token", Value: auth.EncodeTokenCookie([]byte(tok)), }) return http.StatusUnauthorized } func (s *IntegrationSuite) TestVhostViaPath(c *check.C) { s.doVhostRequests(c, authzViaPath) } func authzViaPath(r *http.Request, tok string) int { r.URL.Path = "/t=" + tok + r.URL.Path return http.StatusNotFound } func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) { s.doVhostRequests(c, authzViaQueryString) } func authzViaQueryString(r *http.Request, tok string) int { r.URL.RawQuery = "api_token=" + tok return http.StatusUnauthorized } func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) { s.doVhostRequests(c, authzViaPOST) } func authzViaPOST(r *http.Request, tok string) int { r.Method = "POST" r.Header.Add("Content-Type", "application/x-www-form-urlencoded") r.Body = ioutil.NopCloser(strings.NewReader( url.Values{"api_token": {tok}}.Encode())) return http.StatusUnauthorized } func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) { s.doVhostRequests(c, authzViaPOST) } func authzViaXHRPOST(r *http.Request, tok string) int { r.Method = "POST" r.Header.Add("Content-Type", "application/x-www-form-urlencoded") r.Header.Add("Origin", "https://origin.example") r.Body = ioutil.NopCloser(strings.NewReader( url.Values{ "api_token": {tok}, "disposition": {"attachment"}, }.Encode())) return http.StatusUnauthorized } // Try some combinations of {url, token} using the given authorization // mechanism, and verify the result is correct. func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) { for _, hostPath := range []string{ arvadostest.FooCollection + ".example.com/foo", arvadostest.FooCollection + "--collections.example.com/foo", arvadostest.FooCollection + "--collections.example.com/_/foo", arvadostest.FooCollectionPDH + ".example.com/foo", strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo", arvadostest.FooBarDirCollection + ".example.com/dir1/foo", } { c.Log("doRequests: ", hostPath) s.doVhostRequestsWithHostPath(c, authz, hostPath) } } func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) { for _, tok := range []string{ arvadostest.ActiveToken, arvadostest.ActiveToken[:15], arvadostest.SpectatorToken, "bogus", "", } { u := mustParseURL("http://" + hostPath) req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{}, } failCode := authz(req, tok) req, resp := s.doReq(req) code, body := resp.Code, resp.Body.String() // If the initial request had a (non-empty) token // showing in the query string, we should have been // redirected in order to hide it in a cookie. c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`) if tok == arvadostest.ActiveToken { c.Check(code, check.Equals, http.StatusOK) c.Check(body, check.Equals, "foo") } else { c.Check(code >= 400, check.Equals, true) c.Check(code < 500, check.Equals, true) if tok == arvadostest.SpectatorToken { // Valid token never offers to retry // with different credentials. c.Check(code, check.Equals, http.StatusNotFound) } else { // Invalid token can ask to retry // depending on the authz method. c.Check(code, check.Equals, failCode) } if code == 404 { c.Check(body, check.Equals, notFoundMessage+"\n") } else { c.Check(body, check.Equals, unauthorizedMessage+"\n") } } } } func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) { for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} { for _, port := range []string{"80", "443", "8000"} { s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port) u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection)) req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}}, } req, resp := s.doReq(req) code, _ := resp.Code, resp.Body.String() if port == "8000" { c.Check(code, check.Equals, 401) } else { c.Check(code, check.Equals, 200) } } } } func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) { resp := httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) if resp.Code != http.StatusSeeOther { return req, resp } cookies := (&http.Response{Header: resp.Header()}).Cookies() u, _ := req.URL.Parse(resp.Header().Get("Location")) req = &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{}, } for _, c := range cookies { req.AddCookie(c) } return s.doReq(req) } func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", arvadostest.FooCollection+".example.com/foo", "?api_token="+arvadostest.ActiveToken, "", "", http.StatusOK, "foo", ) } func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo", "", "", "", http.StatusOK, "foo", ) } // Bad token in URL is 404 Not Found because it doesn't make sense to // retry the same URL with different authorization. func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo", "", "", "", http.StatusNotFound, notFoundMessage+"\n", ) } // Bad token in a cookie (even if it got there via our own // query-string-to-cookie redirect) is, in principle, retryable at the // same URL so it's 401 Unauthorized. func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", arvadostest.FooCollection+".example.com/foo", "?api_token=thisisabogustoken", "", "", http.StatusUnauthorized, unauthorizedMessage+"\n", ) } func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) { s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, "", "", http.StatusBadRequest, "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n", ) } // If client requests an attachment by putting ?disposition=attachment // in the query string, and gets redirected, the redirect target // should respond with an attachment. func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) { resp := s.testVhostRedirectTokenToCookie(c, "GET", arvadostest.FooCollection+".example.com/foo", "?disposition=attachment&api_token="+arvadostest.ActiveToken, "", "", http.StatusOK, "foo", ) c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?") } func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) { s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" resp := s.testVhostRedirectTokenToCookie(c, "GET", "download.example.com/by_id/"+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, "", "", http.StatusOK, "foo", ) c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?") } func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) { s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" resp := s.testVhostRedirectTokenToCookie(c, "GET", "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz", "?api_token="+arvadostest.ActiveToken, "", "", http.StatusOK, "waz", ) c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?") resp = s.testVhostRedirectTokenToCookie(c, "GET", "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz", "?api_token="+arvadostest.ActiveToken, "", "", http.StatusOK, "waz", ) c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?") } func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) { s.testServer.Config.cluster.Collections.TrustAllContent = true s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, "", "", http.StatusOK, "foo", ) } func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) { s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234" s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, "", "", http.StatusBadRequest, "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n", ) resp := s.testVhostRedirectTokenToCookie(c, "GET", "example.com:1234/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, "", "", http.StatusOK, "foo", ) c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment") } func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) { s.testVhostRedirectTokenToCookie(c, "POST", arvadostest.FooCollection+".example.com/foo", "", "application/x-www-form-urlencoded", url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(), http.StatusOK, "foo", ) } func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) { s.testVhostRedirectTokenToCookie(c, "POST", arvadostest.FooCollection+".example.com/foo", "", "application/x-www-form-urlencoded", url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(), http.StatusNotFound, notFoundMessage+"\n", ) } func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) { s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt", "", "", "", http.StatusOK, "Hello world\n", ) } func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) { s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid" s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt", "", "", "", http.StatusNotFound, notFoundMessage+"\n", ) } func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) { s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" client := s.testServer.Config.Client client.AuthToken = arvadostest.ActiveToken fs, err := (&arvados.Collection{}).FileSystem(&client, nil) c.Assert(err, check.IsNil) f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777) c.Assert(err, check.IsNil) f.Close() mtxt, err := fs.MarshalManifest(".") c.Assert(err, check.IsNil) var coll arvados.Collection err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{ "collection": map[string]string{ "manifest_text": mtxt, }, }) c.Assert(err, check.IsNil) u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/") req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Authorization": {"Bearer " + client.AuthToken}, }, } resp := httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`) } func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) { arv := arvados.NewClientFromEnv() s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}" name := "foo/bar/baz" nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1) nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1) client := s.testServer.Config.Client client.AuthToken = arvadostest.ActiveToken fs, err := (&arvados.Collection{}).FileSystem(&client, nil) c.Assert(err, check.IsNil) f, err := fs.OpenFile("filename", os.O_CREATE, 0777) c.Assert(err, check.IsNil) f.Close() mtxt, err := fs.MarshalManifest(".") c.Assert(err, check.IsNil) var coll arvados.Collection err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{ "collection": map[string]string{ "manifest_text": mtxt, "name": name, "owner_uuid": arvadostest.AProjectUUID, }, }) c.Assert(err, check.IsNil) defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil) base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/" for tryURL, expectRegexp := range map[string]string{ base: `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`, base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`, } { u, _ := url.Parse(tryURL) req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Authorization": {"Bearer " + client.AuthToken}, }, } resp := httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Matches, expectRegexp) } } // XHRs can't follow redirect-with-cookie so they rely on method=POST // and disposition=attachment (telling us it's acceptable to respond // with content instead of a redirect) and an Origin header that gets // added automatically by the browser (telling us it's desirable to do // so). func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) { u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo") req := &http.Request{ Method: "POST", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Origin": {"https://origin.example"}, "Content-Type": {"application/x-www-form-urlencoded"}, }, Body: ioutil.NopCloser(strings.NewReader(url.Values{ "api_token": {arvadostest.ActiveToken}, "disposition": {"attachment"}, }.Encode())), } resp := httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Equals, "foo") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") // GET + Origin header is representative of both AJAX GET // requests and inline images via . u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2) req = &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Origin": {"https://origin.example"}, }, } resp = httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Equals, "foo") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") } func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder { u, _ := url.Parse(`http://` + hostPath + queryString) req := &http.Request{ Method: method, Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{"Content-Type": {contentType}}, Body: ioutil.NopCloser(strings.NewReader(reqBody)), } resp := httptest.NewRecorder() defer func() { c.Check(resp.Code, check.Equals, expectStatus) c.Check(resp.Body.String(), check.Equals, expectRespBody) }() s.testServer.Handler.ServeHTTP(resp, req) if resp.Code != http.StatusSeeOther { return resp } c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`) cookies := (&http.Response{Header: resp.Header()}).Cookies() u, _ = u.Parse(resp.Header().Get("Location")) req = &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{}, } for _, c := range cookies { req.AddCookie(c) } resp = httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Header().Get("Location"), check.Equals, "") return resp } func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) { s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken s.testDirectoryListing(c) } func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) { s.testServer.Config.cluster.Users.AnonymousUserToken = "" s.testDirectoryListing(c) } func (s *IntegrationSuite) testDirectoryListing(c *check.C) { s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" authHeader := http.Header{ "Authorization": {"OAuth2 " + arvadostest.ActiveToken}, } for _, trial := range []struct { uri string header http.Header expect []string redirect string cutDirs int }{ { uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/", header: authHeader, expect: []string{"dir1/foo", "dir1/bar"}, cutDirs: 0, }, { uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/", header: authHeader, expect: []string{"foo", "bar"}, cutDirs: 1, }, { // URLs of this form ignore authHeader, and // FooAndBarFilesInDirUUID isn't public, so // this returns 401. uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/", header: authHeader, expect: nil, }, { uri: "download.example.com/users/active/foo_file_in_dir/", header: authHeader, expect: []string{"dir1/"}, cutDirs: 3, }, { uri: "download.example.com/users/active/foo_file_in_dir/dir1/", header: authHeader, expect: []string{"bar"}, cutDirs: 4, }, { uri: "download.example.com/", header: authHeader, expect: []string{"users/"}, cutDirs: 0, }, { uri: "download.example.com/users", header: authHeader, redirect: "/users/", expect: []string{"active/"}, cutDirs: 1, }, { uri: "download.example.com/users/", header: authHeader, expect: []string{"active/"}, cutDirs: 1, }, { uri: "download.example.com/users/active", header: authHeader, redirect: "/users/active/", expect: []string{"foo_file_in_dir/"}, cutDirs: 2, }, { uri: "download.example.com/users/active/", header: authHeader, expect: []string{"foo_file_in_dir/"}, cutDirs: 2, }, { uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/", header: nil, expect: []string{"dir1/foo", "dir1/bar"}, cutDirs: 4, }, { uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/", header: nil, expect: []string{"dir1/foo", "dir1/bar"}, cutDirs: 2, }, { uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken, header: nil, expect: []string{"dir1/foo", "dir1/bar"}, cutDirs: 2, }, { uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID, header: authHeader, expect: []string{"dir1/foo", "dir1/bar"}, cutDirs: 1, }, { uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1", header: authHeader, redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/", expect: []string{"foo", "bar"}, cutDirs: 2, }, { uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/", header: authHeader, expect: []string{"foo", "bar"}, cutDirs: 3, }, { uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken, header: authHeader, redirect: "/dir1/", expect: []string{"foo", "bar"}, cutDirs: 1, }, { uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/", header: authHeader, expect: nil, }, { uri: "download.example.com/c=" + arvadostest.WazVersion1Collection, header: authHeader, expect: []string{"waz"}, cutDirs: 1, }, { uri: "download.example.com/by_id/" + arvadostest.WazVersion1Collection, header: authHeader, expect: []string{"waz"}, cutDirs: 2, }, } { comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect) resp := httptest.NewRecorder() u := mustParseURL("//" + trial.uri) req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: copyHeader(trial.header), } s.testServer.Handler.ServeHTTP(resp, req) var cookies []*http.Cookie for resp.Code == http.StatusSeeOther { u, _ := req.URL.Parse(resp.Header().Get("Location")) req = &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: copyHeader(trial.header), } cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...) for _, c := range cookies { req.AddCookie(c) } resp = httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) } if trial.redirect != "" { c.Check(req.URL.Path, check.Equals, trial.redirect, comment) } if trial.expect == nil { if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) } } else { c.Check(resp.Code, check.Equals, http.StatusOK, comment) for _, e := range trial.expect { c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment) } c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment) } comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect) req = &http.Request{ Method: "OPTIONS", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: copyHeader(trial.header), Body: ioutil.NopCloser(&bytes.Buffer{}), } resp = httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) if trial.expect == nil { if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) } } else { c.Check(resp.Code, check.Equals, http.StatusOK, comment) } req = &http.Request{ Method: "PROPFIND", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: copyHeader(trial.header), Body: ioutil.NopCloser(&bytes.Buffer{}), } resp = httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) if trial.expect == nil { if s.testServer.Config.cluster.Users.AnonymousUserToken == "" { c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment) } else { c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) } } else { c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment) for _, e := range trial.expect { if strings.HasSuffix(e, "/") { e = filepath.Join(u.Path, e) + "/" } else { e = filepath.Join(u.Path, e) } c.Check(resp.Body.String(), check.Matches, `(?ms).*`+e+`.*`, comment) } } } } func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) { arv := arvados.NewClientFromEnv() var newCollection arvados.Collection err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{ "collection": map[string]string{ "owner_uuid": arvadostest.ActiveUserUUID, "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n", "name": "keep-web test collection", }, "ensure_unique_name": true, }) c.Assert(err, check.IsNil) defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil) var updated arvados.Collection for _, fnm := range []string{"foo.txt", "bar.txt"} { s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com" u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm) req := &http.Request{ Method: "DELETE", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Authorization": {"Bearer " + arvadostest.ActiveToken}, }, } resp := httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusNoContent) updated = arvados.Collection{} err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil) c.Check(err, check.IsNil) c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`) c.Logf("updated manifest_text %q", updated.ManifestText) } c.Check(updated.ManifestText, check.Equals, "") } func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) { s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken authHeader := http.Header{ "Authorization": {"Bearer " + arvadostest.ManagementToken}, } resp := httptest.NewRecorder() u := mustParseURL("http://download.example.com/_health/ping") req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: authHeader, } s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`) } func (s *IntegrationSuite) TestFileContentType(c *check.C) { s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" client := s.testServer.Config.Client client.AuthToken = arvadostest.ActiveToken arv, err := arvadosclient.New(&client) c.Assert(err, check.Equals, nil) kc, err := keepclient.MakeKeepClient(arv) c.Assert(err, check.Equals, nil) fs, err := (&arvados.Collection{}).FileSystem(&client, kc) c.Assert(err, check.IsNil) trials := []struct { filename string content string contentType string }{ {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"}, {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"}, {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"}, {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff } for _, trial := range trials { f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777) c.Assert(err, check.IsNil) _, err = f.Write([]byte(trial.content)) c.Assert(err, check.IsNil) c.Assert(f.Close(), check.IsNil) } mtxt, err := fs.MarshalManifest(".") c.Assert(err, check.IsNil) var coll arvados.Collection err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{ "collection": map[string]string{ "manifest_text": mtxt, }, }) c.Assert(err, check.IsNil) for _, trial := range trials { u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename) req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Authorization": {"Bearer " + client.AuthToken}, }, } resp := httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType) c.Check(resp.Body.String(), check.Equals, trial.content) } } func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) { s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42 c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42) u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo") req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), } resp := httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42) } // Writing to a collection shouldn't affect its entry in the // PDH-to-manifest cache. func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) { arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, check.Equals, nil) arv.ApiToken = arvadostest.ActiveToken u := mustParseURL("http://x.example/testfile") req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{"Authorization": {"Bearer " + arv.ApiToken}}, } checkWithID := func(id string, status int) { req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example" req.Host = req.URL.Host resp := httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, status) } var colls [2]arvados.Collection for i := range colls { err := arv.Create("collections", map[string]interface{}{ "ensure_unique_name": true, "collection": map[string]interface{}{ "name": "test collection", }, }, &colls[i]) c.Assert(err, check.Equals, nil) } // Populate cache with empty collection checkWithID(colls[0].PortableDataHash, http.StatusNotFound) // write a file to colls[0] reqPut := *req reqPut.Method = "PUT" reqPut.URL.Host = colls[0].UUID + ".example" reqPut.Host = req.URL.Host reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata")) resp := httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, &reqPut) c.Check(resp.Code, check.Equals, http.StatusCreated) // new file should not appear in colls[1] checkWithID(colls[1].PortableDataHash, http.StatusNotFound) checkWithID(colls[1].UUID, http.StatusNotFound) checkWithID(colls[0].UUID, http.StatusOK) } func copyHeader(h http.Header) http.Header { hc := http.Header{} for k, v := range h { hc[k] = append([]string(nil), v...) } return hc } func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, h *handler, req *http.Request, successCode int, direction string, perm bool, userUuid string, collectionUuid string, filepath string) { client := s.testServer.Config.Client client.AuthToken = arvadostest.AdminToken var logentries arvados.LogList limit1 := 1 err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil, arvados.ResourceListParams{ Limit: &limit1, Order: "created_at desc"}) c.Check(err, check.IsNil) c.Check(logentries.Items, check.HasLen, 1) lastLogId := logentries.Items[0].ID nextLogId := lastLogId var logbuf bytes.Buffer logger := logrus.New() logger.Out = &logbuf resp := httptest.NewRecorder() req = req.WithContext(ctxlog.Context(context.Background(), logger)) h.ServeHTTP(resp, req) if perm { c.Check(resp.Result().StatusCode, check.Equals, successCode) c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`) c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`) count := 0 for ; nextLogId == lastLogId && count < 20; count++ { time.Sleep(50 * time.Millisecond) err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil, arvados.ResourceListParams{ Filters: []arvados.Filter{arvados.Filter{Attr: "event_type", Operator: "=", Operand: "file_" + direction}}, Limit: &limit1, Order: "created_at desc", }) c.Check(err, check.IsNil) if len(logentries.Items) > 0 { nextLogId = logentries.Items[0].ID } } c.Check(count, check.Not(check.Equals), 20) c.Check(logentries.Items[0].ObjectUUID, check.Equals, userUuid) c.Check(logentries.Items[0].Properties["collection_uuid"], check.Equals, collectionUuid) c.Check(logentries.Items[0].Properties["collection_file_path"], check.Equals, filepath) } else { c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden) c.Check(logbuf.String(), check.Equals, "") } } func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) { config := newConfig(s.ArvConfig) h := handler{Config: config} u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo") config.cluster.Collections.TrustAllContent = true for _, adminperm := range []bool{true, false} { for _, userperm := range []bool{true, false} { config.cluster.Collections.WebDAVPermission.Admin.Download = adminperm config.cluster.Collections.WebDAVPermission.User.Download = userperm // Test admin permission req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Authorization": {"Bearer " + arvadostest.AdminToken}, }, } s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", adminperm, arvadostest.AdminUserUUID, arvadostest.FooCollection, "foo") // Test user permission req = &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Authorization": {"Bearer " + arvadostest.ActiveToken}, }, } s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", userperm, arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo") } } config.cluster.Collections.WebDAVPermission.User.Download = true for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1", "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} { u = mustParseURL(tryurl) req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Authorization": {"Bearer " + arvadostest.ActiveToken}, }, } s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", true, arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, "dir1/subdir/file1") } u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo") req := &http.Request{ Method: "GET", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Authorization": {"Bearer " + arvadostest.ActiveToken}, }, } s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", true, arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo") } func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) { config := newConfig(s.ArvConfig) h := handler{Config: config} for _, adminperm := range []bool{true, false} { for _, userperm := range []bool{true, false} { arv := s.testServer.Config.Client arv.AuthToken = arvadostest.ActiveToken var coll arvados.Collection err := arv.RequestAndDecode(&coll, "POST", "/arvados/v1/collections", nil, map[string]interface{}{ "ensure_unique_name": true, "collection": map[string]interface{}{ "name": "test collection", }, }) c.Assert(err, check.Equals, nil) u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar") config.cluster.Collections.WebDAVPermission.Admin.Upload = adminperm config.cluster.Collections.WebDAVPermission.User.Upload = userperm // Test admin permission req := &http.Request{ Method: "PUT", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Authorization": {"Bearer " + arvadostest.AdminToken}, }, Body: io.NopCloser(bytes.NewReader([]byte("bar"))), } s.checkUploadDownloadRequest(c, &h, req, http.StatusCreated, "upload", adminperm, arvadostest.AdminUserUUID, coll.UUID, "bar") // Test user permission req = &http.Request{ Method: "PUT", Host: u.Host, URL: u, RequestURI: u.RequestURI(), Header: http.Header{ "Authorization": {"Bearer " + arvadostest.ActiveToken}, }, Body: io.NopCloser(bytes.NewReader([]byte("bar"))), } s.checkUploadDownloadRequest(c, &h, req, http.StatusCreated, "upload", userperm, arvadostest.ActiveUserUUID, coll.UUID, "bar") } } }