Merge branch '16100-mime-types'

fixes #16100

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

diff --git a/sdk/go/httpserver/logger.go b/sdk/go/httpserver/logger.go
index 30f5e2612b91dfdeff2a82f9082c4934aca0c6e3..8886f9517dfd5983032235e713a000f5615880b7 100644 (file)
--- a/sdk/go/httpserver/logger.go
+++ b/sdk/go/httpserver/logger.go
@@ -68,10 +68,16 @@ func logRequest(w *responseTimer, req *http.Request, lgr *logrus.Entry) {
 func logResponse(w *responseTimer, req *http.Request, lgr *logrus.Entry) {
        if tStart, ok := req.Context().Value(&requestTimeContextKey).(time.Time); ok {
                tDone := time.Now()
+               writeTime := w.writeTime
+               if !w.wrote {
+                       // Empty response body. Header was sent when
+                       // handler exited.
+                       writeTime = tDone
+               }
                lgr = lgr.WithFields(logrus.Fields{
                        "timeTotal":     stats.Duration(tDone.Sub(tStart)),
-                       "timeToStatus":  stats.Duration(w.writeTime.Sub(tStart)),
-                       "timeWriteBody": stats.Duration(tDone.Sub(w.writeTime)),
+                       "timeToStatus":  stats.Duration(writeTime.Sub(tStart)),
+                       "timeWriteBody": stats.Duration(tDone.Sub(writeTime)),
                })
        }
        respCode := w.WroteStatus()

diff --git a/services/keep-web/fpm-info.sh b/services/keep-web/fpm-info.sh
new file mode 100644 (file)
index 0000000..6bcbf67
--- /dev/null
+++ b/services/keep-web/fpm-info.sh
@@ -0,0 +1,12 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+case "$TARGET" in
+    centos*)
+        fpm_depends+=(mailcap)
+        ;;
+    debian* | ubuntu*)
+        fpm_depends+=(mime-support)
+        ;;
+esac

diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 29bcdac1b72a9ee23dea8d6828d94fc53f8f3b8d..f6f3de8877fa14abca9fc479216f13d4cb85a308 100644 (file)
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -19,6 +19,7 @@ import (
 
        "git.arvados.org/arvados.git/lib/config"
        "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadosclient"
        "git.arvados.org/arvados.git/sdk/go/arvadostest"
        "git.arvados.org/arvados.git/sdk/go/auth"
        "git.arvados.org/arvados.git/sdk/go/ctxlog"
@@ -40,24 +41,6 @@ func (s *UnitSuite) SetUpTest(c *check.C) {
        s.Config = cfg
 }
 
-func (s *UnitSuite) TestKeepClientBlockCache(c *check.C) {
-       cfg := newConfig(s.Config)
-       cfg.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
-       h := handler{Config: cfg}
-       c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
-       u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
-       req := &http.Request{
-               Method:     "GET",
-               Host:       u.Host,
-               URL:        u,
-               RequestURI: u.RequestURI(),
-       }
-       resp := httptest.NewRecorder()
-       h.ServeHTTP(resp, req)
-       c.Check(resp.Code, check.Equals, http.StatusOK)
-       c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
-}
-
 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
        h := handler{Config: newConfig(s.Config)}
        u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
@@ -933,6 +916,82 @@ func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
        c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 }
 
+func (s *IntegrationSuite) TestFileContentType(c *check.C) {
+       s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+
+       client := s.testServer.Config.Client
+       client.AuthToken = arvadostest.ActiveToken
+       arv, err := arvadosclient.New(&client)
+       c.Assert(err, check.Equals, nil)
+       kc, err := keepclient.MakeKeepClient(arv)
+       c.Assert(err, check.Equals, nil)
+
+       fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
+       c.Assert(err, check.IsNil)
+
+       trials := []struct {
+               filename    string
+               content     string
+               contentType string
+       }{
+               {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
+               {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
+               {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
+               {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
+               {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
+       }
+       for _, trial := range trials {
+               f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
+               c.Assert(err, check.IsNil)
+               _, err = f.Write([]byte(trial.content))
+               c.Assert(err, check.IsNil)
+               c.Assert(f.Close(), check.IsNil)
+       }
+       mtxt, err := fs.MarshalManifest(".")
+       c.Assert(err, check.IsNil)
+       var coll arvados.Collection
+       err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+               "collection": map[string]string{
+                       "manifest_text": mtxt,
+               },
+       })
+       c.Assert(err, check.IsNil)
+
+       for _, trial := range trials {
+               u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
+               req := &http.Request{
+                       Method:     "GET",
+                       Host:       u.Host,
+                       URL:        u,
+                       RequestURI: u.RequestURI(),
+                       Header: http.Header{
+                               "Authorization": {"Bearer " + client.AuthToken},
+                       },
+               }
+               resp := httptest.NewRecorder()
+               s.testServer.Handler.ServeHTTP(resp, req)
+               c.Check(resp.Code, check.Equals, http.StatusOK)
+               c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
+               c.Check(resp.Body.String(), check.Equals, trial.content)
+       }
+}
+
+func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
+       s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
+       c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
+       u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
+       req := &http.Request{
+               Method:     "GET",
+               Host:       u.Host,
+               URL:        u,
+               RequestURI: u.RequestURI(),
+       }
+       resp := httptest.NewRecorder()
+       s.testServer.Handler.ServeHTTP(resp, req)
+       c.Check(resp.Code, check.Equals, http.StatusOK)
+       c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
+}
+
 func copyHeader(h http.Header) http.Header {
        hc := http.Header{}
        for k, v := range h {

diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index f028cca223f555d22afff883bec61423896d61fd..e4028842f0c6b9390715a93c836846f2d9ba753b 100644 (file)
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -7,6 +7,7 @@ package main
 import (
        "flag"
        "fmt"
+       "mime"
        "os"
 
        "git.arvados.org/arvados.git/lib/config"
@@ -104,6 +105,10 @@ func main() {
 
        log.Printf("keep-web %s started", version)
 
+       if ext := ".txt"; mime.TypeByExtension(ext) == "" {
+               log.Warnf("cannot look up MIME type for %q -- this probably means /etc/mime.types is missing -- clients will see incorrect content types", ext)
+       }
+
        os.Setenv("ARVADOS_API_HOST", cfg.cluster.Services.Controller.ExternalURL.Host)
        srv := &server{Config: cfg}
        if err := srv.Start(); err != nil {