Merge branch '21666-provision-test-improvement'

[arvados.git] / services / keep-web / handler.go

diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 123c4fe34da3d1dba8ae504b9867a410cd5022b5..b9250efec76b8b45f599c30dc8961cbc3e279474 100644 (file)
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -27,15 +27,14 @@ import (
        "git.arvados.org/arvados.git/sdk/go/auth"
        "git.arvados.org/arvados.git/sdk/go/ctxlog"
        "git.arvados.org/arvados.git/sdk/go/httpserver"
-       "git.arvados.org/arvados.git/sdk/go/keepclient"
        "github.com/sirupsen/logrus"
        "golang.org/x/net/webdav"
 )
 
 type handler struct {
-       Cache     cache
-       Cluster   *arvados.Cluster
-       setupOnce sync.Once
+       Cache   cache
+       Cluster *arvados.Cluster
+       metrics *metrics
 
        lockMtx    sync.Mutex
        lock       map[string]*sync.RWMutex
@@ -60,10 +59,6 @@ func parseCollectionIDFromURL(s string) string {
        return ""
 }
 
-func (h *handler) setup() {
-       keepclient.DefaultBlockCache.MaxBlocks = h.Cluster.Collections.WebDAVCache.MaxBlockEntries
-}
-
 func (h *handler) serveStatus(w http.ResponseWriter, r *http.Request) {
        json.NewEncoder(w).Encode(struct{ Version string }{cmd.Version.String()})
 }
@@ -179,13 +174,16 @@ func (h *handler) Done() <-chan struct{} {
 
 // ServeHTTP implements http.Handler.
 func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
-       h.setupOnce.Do(h.setup)
-
        if xfp := r.Header.Get("X-Forwarded-Proto"); xfp != "" && xfp != "http" {
                r.URL.Scheme = xfp
        }
 
-       w := httpserver.WrapResponseWriter(wOrig)
+       wbuffer := newWriteBuffer(wOrig, int(h.Cluster.Collections.WebDAVOutputBuffer))
+       defer wbuffer.Close()
+       w := httpserver.WrapResponseWriter(responseWriter{
+               Writer:         wbuffer,
+               ResponseWriter: wOrig,
+       })
 
        if r.Method == "OPTIONS" && ServeCORSPreflight(w, r.Header) {
                return
@@ -432,11 +430,26 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
                        return
                }
                defer f.Close()
-               defer sess.Release()
 
                collectionDir, sessionFS, session, tokenUser = f, fs, sess, user
                break
        }
+
+       // releaseSession() is equivalent to session.Release() except
+       // that it's a no-op if (1) session is nil, or (2) it has
+       // already been called.
+       //
+       // This way, we can do a defer call here to ensure it gets
+       // called in all code paths, and also call it inline (see
+       // below) in the cases where we want to release the lock
+       // before returning.
+       releaseSession := func() {}
+       if session != nil {
+               var releaseSessionOnce sync.Once
+               releaseSession = func() { releaseSessionOnce.Do(func() { session.Release() }) }
+       }
+       defer releaseSession()
+
        if forceReload && collectionDir != nil {
                err := collectionDir.Sync()
                if err != nil {
@@ -524,6 +537,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
        if r.Method == http.MethodGet || r.Method == http.MethodHead {
                targetfnm := fsprefix + strings.Join(pathParts[stripParts:], "/")
                if fi, err := sessionFS.Stat(targetfnm); err == nil && fi.IsDir() {
+                       releaseSession() // because we won't be writing anything
                        if !strings.HasSuffix(r.URL.Path, "/") {
                                h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
                        } else {
@@ -593,6 +607,15 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
                                collectionDir.Splice(snap)
                                return nil
                        }}
+       } else {
+               // When writing, we need to block session renewal
+               // until we're finished, in order to guarantee the
+               // effect of the write is visible in future responses.
+               // But if we're not writing, we can release the lock
+               // early.  This enables us to keep renewing sessions
+               // and processing more requests even if a slow client
+               // takes a long time to download a large file.
+               releaseSession()
        }
        if r.Method == http.MethodGet {
                applyContentDispositionHdr(w, r, basename, attachment)
@@ -600,7 +623,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
        if webdavPrefix == "" {
                webdavPrefix = "/" + strings.Join(pathParts[:stripParts], "/")
        }
-       wh := webdav.Handler{
+       wh := &webdav.Handler{
                Prefix: webdavPrefix,
                FileSystem: &webdavfs.FS{
                        FileSystem:    sessionFS,
@@ -615,7 +638,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
                        }
                },
        }
-       wh.ServeHTTP(w, r)
+       h.metrics.track(wh, w, r)
        if r.Method == http.MethodGet && w.WroteStatus() == http.StatusOK {
                wrote := int64(w.WroteBodyBytes())
                fnm := strings.Join(pathParts[stripParts:], "/")