From: Tom Clegg Date: Mon, 17 Feb 2020 16:36:58 +0000 (-0500) Subject: Merge branch '16100-mime-types' X-Git-Tag: 2.1.0~295 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/0a415b6c80c3bf39bb753274aae857eadde2f590?hp=b9e15eb4acf18db8bc409d23dfdbe2cf9cf986d4 Merge branch '16100-mime-types' fixes #16100 Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/sdk/go/httpserver/logger.go b/sdk/go/httpserver/logger.go index 30f5e2612b..8886f9517d 100644 --- a/sdk/go/httpserver/logger.go +++ b/sdk/go/httpserver/logger.go @@ -68,10 +68,16 @@ func logRequest(w *responseTimer, req *http.Request, lgr *logrus.Entry) { func logResponse(w *responseTimer, req *http.Request, lgr *logrus.Entry) { if tStart, ok := req.Context().Value(&requestTimeContextKey).(time.Time); ok { tDone := time.Now() + writeTime := w.writeTime + if !w.wrote { + // Empty response body. Header was sent when + // handler exited. + writeTime = tDone + } lgr = lgr.WithFields(logrus.Fields{ "timeTotal": stats.Duration(tDone.Sub(tStart)), - "timeToStatus": stats.Duration(w.writeTime.Sub(tStart)), - "timeWriteBody": stats.Duration(tDone.Sub(w.writeTime)), + "timeToStatus": stats.Duration(writeTime.Sub(tStart)), + "timeWriteBody": stats.Duration(tDone.Sub(writeTime)), }) } respCode := w.WroteStatus() diff --git a/services/keep-web/fpm-info.sh b/services/keep-web/fpm-info.sh new file mode 100644 index 0000000000..6bcbf67fe0 --- /dev/null +++ b/services/keep-web/fpm-info.sh @@ -0,0 +1,12 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +case "$TARGET" in + centos*) + fpm_depends+=(mailcap) + ;; + debian* | ubuntu*) + fpm_depends+=(mime-support) + ;; +esac diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go index 29bcdac1b7..f6f3de8877 100644 --- a/services/keep-web/handler_test.go +++ b/services/keep-web/handler_test.go @@ -19,6 +19,7 @@ import ( "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" "git.arvados.org/arvados.git/sdk/go/auth" "git.arvados.org/arvados.git/sdk/go/ctxlog" @@ -40,24 +41,6 @@ func (s *UnitSuite) SetUpTest(c *check.C) { s.Config = cfg } -func (s *UnitSuite) TestKeepClientBlockCache(c *check.C) { - cfg := newConfig(s.Config) - cfg.cluster.Collections.WebDAVCache.MaxBlockEntries = 42 - h := handler{Config: cfg} - c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), cfg.cluster.Collections.WebDAVCache.MaxBlockEntries) - u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo") - req := &http.Request{ - Method: "GET", - Host: u.Host, - URL: u, - RequestURI: u.RequestURI(), - } - resp := httptest.NewRecorder() - h.ServeHTTP(resp, req) - c.Check(resp.Code, check.Equals, http.StatusOK) - c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, cfg.cluster.Collections.WebDAVCache.MaxBlockEntries) -} - func (s *UnitSuite) TestCORSPreflight(c *check.C) { h := handler{Config: newConfig(s.Config)} u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo") @@ -933,6 +916,82 @@ func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) { c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`) } +func (s *IntegrationSuite) TestFileContentType(c *check.C) { + s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com" + + client := s.testServer.Config.Client + client.AuthToken = arvadostest.ActiveToken + arv, err := arvadosclient.New(&client) + c.Assert(err, check.Equals, nil) + kc, err := keepclient.MakeKeepClient(arv) + c.Assert(err, check.Equals, nil) + + fs, err := (&arvados.Collection{}).FileSystem(&client, kc) + c.Assert(err, check.IsNil) + + trials := []struct { + filename string + content string + contentType string + }{ + {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"}, + {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"}, + {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"}, + {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp + {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff + } + for _, trial := range trials { + f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777) + c.Assert(err, check.IsNil) + _, err = f.Write([]byte(trial.content)) + c.Assert(err, check.IsNil) + c.Assert(f.Close(), check.IsNil) + } + mtxt, err := fs.MarshalManifest(".") + c.Assert(err, check.IsNil) + var coll arvados.Collection + err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{ + "collection": map[string]string{ + "manifest_text": mtxt, + }, + }) + c.Assert(err, check.IsNil) + + for _, trial := range trials { + u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename) + req := &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + client.AuthToken}, + }, + } + resp := httptest.NewRecorder() + s.testServer.Handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusOK) + c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType) + c.Check(resp.Body.String(), check.Equals, trial.content) + } +} + +func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) { + s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42 + c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42) + u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo") + req := &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + } + resp := httptest.NewRecorder() + s.testServer.Handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusOK) + c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42) +} + func copyHeader(h http.Header) http.Header { hc := http.Header{} for k, v := range h { diff --git a/services/keep-web/main.go b/services/keep-web/main.go index f028cca223..e4028842f0 100644 --- a/services/keep-web/main.go +++ b/services/keep-web/main.go @@ -7,6 +7,7 @@ package main import ( "flag" "fmt" + "mime" "os" "git.arvados.org/arvados.git/lib/config" @@ -104,6 +105,10 @@ func main() { log.Printf("keep-web %s started", version) + if ext := ".txt"; mime.TypeByExtension(ext) == "" { + log.Warnf("cannot look up MIME type for %q -- this probably means /etc/mime.types is missing -- clients will see incorrect content types", ext) + } + os.Setenv("ARVADOS_API_HOST", cfg.cluster.Services.Controller.ExternalURL.Host) srv := &server{Config: cfg} if err := srv.Start(); err != nil {