X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/9e37d22cf961167df7ce5f7ac934059749363500..acb1d98cdfa6e33bfd3e1bb7510b78a4fd9384e2:/services/keep-web/handler_test.go diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go index 4894ceb70b..fe8e767c4c 100644 --- a/services/keep-web/handler_test.go +++ b/services/keep-web/handler_test.go @@ -12,10 +12,13 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" "path/filepath" "regexp" "strings" + "git.curoverse.com/arvados.git/lib/config" + "git.curoverse.com/arvados.git/sdk/go/arvados" "git.curoverse.com/arvados.git/sdk/go/arvadostest" "git.curoverse.com/arvados.git/sdk/go/auth" check "gopkg.in/check.v1" @@ -23,11 +26,20 @@ import ( var _ = check.Suite(&UnitSuite{}) -type UnitSuite struct{} +type UnitSuite struct { + Config *arvados.Config +} + +func (s *UnitSuite) SetUpTest(c *check.C) { + ldr := config.NewLoader(nil, nil) + cfg, err := ldr.LoadDefaults() + c.Assert(err, check.IsNil) + s.Config = cfg +} func (s *UnitSuite) TestCORSPreflight(c *check.C) { - h := handler{Config: DefaultConfig()} - u, _ := url.Parse("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo") + h := handler{Config: DefaultConfig(s.Config)} + u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo") req := &http.Request{ Method: "OPTIONS", Host: u.Host, @@ -45,8 +57,8 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) { c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Equals, "") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") - c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PUT, RMCOL") - c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range") + c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK") + c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout") // Check preflight for a disallowed request resp = httptest.NewRecorder() @@ -57,7 +69,7 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) { } func (s *UnitSuite) TestInvalidUUID(c *check.C) { - bogusID := strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "-" + bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-" token := arvadostest.ActiveToken for _, trial := range []string{ "http://keep-web/c=" + bogusID + "/foo", @@ -68,8 +80,7 @@ func (s *UnitSuite) TestInvalidUUID(c *check.C) { "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo", } { c.Log(trial) - u, err := url.Parse(trial) - c.Assert(err, check.IsNil) + u := mustParseURL(trial) req := &http.Request{ Method: "GET", Host: u.Host, @@ -77,8 +88,8 @@ func (s *UnitSuite) TestInvalidUUID(c *check.C) { RequestURI: u.RequestURI(), } resp := httptest.NewRecorder() - cfg := DefaultConfig() - cfg.AnonymousTokens = []string{arvadostest.AnonymousToken} + cfg := DefaultConfig(s.Config) + cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken h := handler{Config: cfg} h.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusNotFound) @@ -185,8 +196,8 @@ func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) { arvadostest.FooCollection + ".example.com/foo", arvadostest.FooCollection + "--collections.example.com/foo", arvadostest.FooCollection + "--collections.example.com/_/foo", - arvadostest.FooPdh + ".example.com/foo", - strings.Replace(arvadostest.FooPdh, "+", "-", -1) + "--collections.example.com/foo", + arvadostest.FooCollectionPDH + ".example.com/foo", + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo", arvadostest.FooBarDirCollection + ".example.com/dir1/foo", } { c.Log("doRequests: ", hostPath) @@ -337,7 +348,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check } func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) { - s.testServer.Config.AttachmentOnlyHost = "download.example.com" + s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" resp := s.testVhostRedirectTokenToCookie(c, "GET", "download.example.com/by_id/"+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, @@ -349,8 +360,30 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) { c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?") } +func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) { + s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + resp := s.testVhostRedirectTokenToCookie(c, "GET", + "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz", + "?api_token="+arvadostest.ActiveToken, + "", + "", + http.StatusOK, + "waz", + ) + c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?") + resp = s.testVhostRedirectTokenToCookie(c, "GET", + "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz", + "?api_token="+arvadostest.ActiveToken, + "", + "", + http.StatusOK, + "waz", + ) + c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?") +} + func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) { - s.testServer.Config.TrustAllContent = true + s.testServer.Config.cluster.Collections.TrustAllContent = true s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", "?api_token="+arvadostest.ActiveToken, @@ -362,7 +395,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C } func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) { - s.testServer.Config.AttachmentOnlyHost = "example.com:1234" + s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234" s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.FooCollection+"/foo", @@ -407,7 +440,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) } func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) { - s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken} + s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt", "", @@ -419,7 +452,7 @@ func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) { } func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) { - s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"} + s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid" s.testVhostRedirectTokenToCookie(c, "GET", "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt", "", @@ -430,6 +463,42 @@ func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) { ) } +func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) { + s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + + client := s.testServer.Config.Client + client.AuthToken = arvadostest.ActiveToken + fs, err := (&arvados.Collection{}).FileSystem(&client, nil) + c.Assert(err, check.IsNil) + f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777) + c.Assert(err, check.IsNil) + f.Close() + mtxt, err := fs.MarshalManifest(".") + c.Assert(err, check.IsNil) + var coll arvados.Collection + err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{ + "collection": map[string]string{ + "manifest_text": mtxt, + }, + }) + c.Assert(err, check.IsNil) + + u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/") + req := &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + client.AuthToken}, + }, + } + resp := httptest.NewRecorder() + s.testServer.Handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusOK) + c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`) +} + // XHRs can't follow redirect-with-cookie so they rely on method=POST // and disposition=attachment (telling us it's acceptable to respond // with content instead of a redirect) and an Origin header that gets @@ -479,7 +548,7 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho if resp.Code != http.StatusSeeOther { return resp } - c.Check(resp.Body.String(), check.Matches, `.*href="//`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`) + c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`) cookies := (&http.Response{Header: resp.Header()}).Cookies() u, _ = u.Parse(resp.Header().Get("Location")) @@ -500,8 +569,18 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho return resp } -func (s *IntegrationSuite) TestDirectoryListing(c *check.C) { - s.testServer.Config.AttachmentOnlyHost = "download.example.com" +func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) { + s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken + s.testDirectoryListing(c) +} + +func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) { + s.testServer.Config.cluster.Users.AnonymousUserToken = "" + s.testDirectoryListing(c) +} + +func (s *IntegrationSuite) testDirectoryListing(c *check.C) { + s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" authHeader := http.Header{ "Authorization": {"OAuth2 " + arvadostest.ActiveToken}, } @@ -525,10 +604,12 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) { cutDirs: 1, }, { - uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/", - header: authHeader, - expect: []string{"dir1/foo", "dir1/bar"}, - cutDirs: 2, + // URLs of this form ignore authHeader, and + // FooAndBarFilesInDirUUID isn't public, so + // this returns 404. + uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/", + header: authHeader, + expect: nil, }, { uri: "download.example.com/users/active/foo_file_in_dir/", @@ -623,8 +704,20 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) { header: authHeader, expect: nil, }, + { + uri: "download.example.com/c=" + arvadostest.WazVersion1Collection, + header: authHeader, + expect: []string{"waz"}, + cutDirs: 1, + }, + { + uri: "download.example.com/by_id/" + arvadostest.WazVersion1Collection, + header: authHeader, + expect: []string{"waz"}, + cutDirs: 2, + }, } { - c.Logf("HTML: %q => %q", trial.uri, trial.expect) + comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect) resp := httptest.NewRecorder() u := mustParseURL("//" + trial.uri) req := &http.Request{ @@ -632,7 +725,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) { Host: u.Host, URL: u, RequestURI: u.RequestURI(), - Header: trial.header, + Header: copyHeader(trial.header), } s.testServer.Handler.ServeHTTP(resp, req) var cookies []*http.Cookie @@ -643,7 +736,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) { Host: u.Host, URL: u, RequestURI: u.RequestURI(), - Header: trial.header, + Header: copyHeader(trial.header), } cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...) for _, c := range cookies { @@ -653,33 +746,33 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) { s.testServer.Handler.ServeHTTP(resp, req) } if trial.redirect != "" { - c.Check(req.URL.Path, check.Equals, trial.redirect) + c.Check(req.URL.Path, check.Equals, trial.redirect, comment) } if trial.expect == nil { - c.Check(resp.Code, check.Equals, http.StatusNotFound) + c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) } else { - c.Check(resp.Code, check.Equals, http.StatusOK) + c.Check(resp.Code, check.Equals, http.StatusOK, comment) for _, e := range trial.expect { - c.Check(resp.Body.String(), check.Matches, `(?ms).*href="`+e+`".*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment) } - c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment) } - c.Logf("WebDAV: %q => %q", trial.uri, trial.expect) + comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect) req = &http.Request{ Method: "OPTIONS", Host: u.Host, URL: u, RequestURI: u.RequestURI(), - Header: trial.header, + Header: copyHeader(trial.header), Body: ioutil.NopCloser(&bytes.Buffer{}), } resp = httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) if trial.expect == nil { - c.Check(resp.Code, check.Equals, http.StatusNotFound) + c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) } else { - c.Check(resp.Code, check.Equals, http.StatusOK) + c.Check(resp.Code, check.Equals, http.StatusOK, comment) } req = &http.Request{ @@ -687,24 +780,64 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) { Host: u.Host, URL: u, RequestURI: u.RequestURI(), - Header: trial.header, + Header: copyHeader(trial.header), Body: ioutil.NopCloser(&bytes.Buffer{}), } resp = httptest.NewRecorder() s.testServer.Handler.ServeHTTP(resp, req) if trial.expect == nil { - c.Check(resp.Code, check.Equals, http.StatusNotFound) + c.Check(resp.Code, check.Equals, http.StatusNotFound, comment) } else { - c.Check(resp.Code, check.Equals, http.StatusMultiStatus) + c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment) for _, e := range trial.expect { - c.Check(resp.Body.String(), check.Matches, `(?ms).*`+filepath.Join(u.Path, e)+`.*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*`+filepath.Join(u.Path, e)+`.*`, comment) } } } } +func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) { + arv := arvados.NewClientFromEnv() + var newCollection arvados.Collection + err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{ + "collection": map[string]string{ + "owner_uuid": arvadostest.ActiveUserUUID, + "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n", + "name": "keep-web test collection", + }, + "ensure_unique_name": true, + }) + c.Assert(err, check.IsNil) + defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil) + + var updated arvados.Collection + for _, fnm := range []string{"foo.txt", "bar.txt"} { + s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com" + u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm) + req := &http.Request{ + Method: "DELETE", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + }, + } + resp := httptest.NewRecorder() + s.testServer.Handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusNoContent) + + updated = arvados.Collection{} + err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil) + c.Check(err, check.IsNil) + c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`) + c.Logf("updated manifest_text %q", updated.ManifestText) + } + c.Check(updated.ManifestText, check.Equals, "") +} + func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) { - s.testServer.Config.ManagementToken = arvadostest.ManagementToken + s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken authHeader := http.Header{ "Authorization": {"Bearer " + arvadostest.ManagementToken}, } @@ -723,3 +856,11 @@ func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) { c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`) } + +func copyHeader(h http.Header) http.Header { + hc := http.Header{} + for k, v := range h { + hc[k] = append([]string(nil), v...) + } + return hc +}