X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/6b0d4ac8df4b5b4255eb56b1d76865f06089ca2a..4b58bfc5b58db18e4816102f9850757f0884a42e:/services/keep-web/handler.go diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go index 7ac8bc02db..3af326a1ad 100644 --- a/services/keep-web/handler.go +++ b/services/keep-web/handler.go @@ -14,12 +14,14 @@ import ( "net/http" "net/url" "os" - "path/filepath" "sort" "strconv" "strings" "sync" + "time" + "git.arvados.org/arvados.git/lib/cmd" + "git.arvados.org/arvados.git/lib/webdavfs" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/auth" @@ -31,17 +33,19 @@ import ( ) type handler struct { - Cache cache - Cluster *arvados.Cluster - clientPool *arvadosclient.ClientPool - setupOnce sync.Once - webdavLS webdav.LockSystem + Cache cache + Cluster *arvados.Cluster + setupOnce sync.Once + + lockMtx sync.Mutex + lock map[string]*sync.RWMutex + lockTidied time.Time } var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+") var notFoundMessage = "Not Found" -var unauthorizedMessage = "401 Unauthorized\r\n\r\nA valid Arvados token must be provided to access this resource.\r\n" +var unauthorizedMessage = "401 Unauthorized\n\nA valid Arvados token must be provided to access this resource." // parseCollectionIDFromURL returns a UUID or PDH if s is a UUID or a // PDH (even if it is a PDH with "+" replaced by " " or "-"); @@ -57,19 +61,15 @@ func parseCollectionIDFromURL(s string) string { } func (h *handler) setup() { - // Errors will be handled at the client pool. - arv, _ := arvados.NewClientFromConfig(h.Cluster) - h.clientPool = arvadosclient.MakeClientPoolWith(arv) - keepclient.DefaultBlockCache.MaxBlocks = h.Cluster.Collections.WebDAVCache.MaxBlockEntries - - // Even though we don't accept LOCK requests, every webdav - // handler must have a non-nil LockSystem. - h.webdavLS = &noLockSystem{} } func (h *handler) serveStatus(w http.ResponseWriter, r *http.Request) { - json.NewEncoder(w).Encode(struct{ Version string }{version}) + json.NewEncoder(w).Encode(struct{ Version string }{cmd.Version.String()}) +} + +type errorWithHTTPStatus interface { + HTTPStatus() int } // updateOnSuccess wraps httpserver.ResponseWriter. If the handler @@ -102,8 +102,7 @@ func (uos *updateOnSuccess) WriteHeader(code int) { if code >= 200 && code < 400 { if uos.err = uos.update(); uos.err != nil { code := http.StatusInternalServerError - var he interface{ HTTPStatus() int } - if errors.As(uos.err, &he) { + if he := errorWithHTTPStatus(nil); errors.As(uos.err, &he) { code = he.HTTPStatus() } uos.logger.WithError(uos.err).Errorf("update() returned %T error, changing response to HTTP %d", uos.err, code) @@ -119,7 +118,7 @@ var ( corsAllowHeadersHeader = strings.Join([]string{ "Authorization", "Content-Type", "Range", // WebDAV request headers: - "Depth", "Destination", "If", "Lock-Token", "Overwrite", "Timeout", + "Depth", "Destination", "If", "Lock-Token", "Overwrite", "Timeout", "Cache-Control", }, ", ") writeMethod = map[string]bool{ "COPY": true, @@ -188,15 +187,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { w := httpserver.WrapResponseWriter(wOrig) - if method := r.Header.Get("Access-Control-Request-Method"); method != "" && r.Method == "OPTIONS" { - if !browserMethod[method] && !webdavMethod[method] { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - w.Header().Set("Access-Control-Allow-Headers", corsAllowHeadersHeader) - w.Header().Set("Access-Control-Allow-Methods", "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK") - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Max-Age", "86400") + if r.Method == "OPTIONS" && ServeCORSPreflight(w, r.Header) { return } @@ -219,7 +210,26 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { return } - pathParts := strings.Split(r.URL.Path[1:], "/") + webdavPrefix := "" + arvPath := r.URL.Path + if prefix := r.Header.Get("X-Webdav-Prefix"); prefix != "" { + // Enable a proxy (e.g., container log handler in + // controller) to satisfy a request for path + // "/foo/bar/baz.txt" using content from + // "//abc123-4.internal/bar/baz.txt", by adding a + // request header "X-Webdav-Prefix: /foo" + if !strings.HasPrefix(arvPath, prefix) { + http.Error(w, "X-Webdav-Prefix header is not a prefix of the requested path", http.StatusBadRequest) + return + } + arvPath = r.URL.Path[len(prefix):] + if arvPath == "" { + arvPath = "/" + } + w.Header().Set("Vary", "X-Webdav-Prefix, "+w.Header().Get("Vary")) + webdavPrefix = prefix + } + pathParts := strings.Split(arvPath[1:], "/") var stripParts int var collectionID string @@ -274,11 +284,6 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { } } - if collectionID == "" && !useSiteFS { - http.Error(w, notFoundMessage, http.StatusNotFound) - return - } - forceReload := false if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") { forceReload = true @@ -319,11 +324,6 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { return } - if useSiteFS { - h.serveSiteFS(w, r, reqTokens, credentialsOK, attachment) - return - } - targetPath := pathParts[stripParts:] if tokens == nil && len(targetPath) > 0 && strings.HasPrefix(targetPath[0], "t=") { // http://ID.example/t=TOKEN/PATH... @@ -338,20 +338,34 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { stripParts++ } - if tokens == nil { - tokens = reqTokens - if h.Cluster.Users.AnonymousUserToken != "" { - tokens = append(tokens, h.Cluster.Users.AnonymousUserToken) + fsprefix := "" + if useSiteFS { + if writeMethod[r.Method] { + http.Error(w, webdavfs.ErrReadOnly.Error(), http.StatusMethodNotAllowed) + return + } + if len(reqTokens) == 0 { + w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"") + http.Error(w, unauthorizedMessage, http.StatusUnauthorized) + return } + tokens = reqTokens + } else if collectionID == "" { + http.Error(w, notFoundMessage, http.StatusNotFound) + return + } else { + fsprefix = "by_id/" + collectionID + "/" + } + + if src := r.Header.Get("X-Webdav-Source"); strings.HasPrefix(src, "/") && !strings.Contains(src, "//") && !strings.Contains(src, "/../") { + fsprefix += src[1:] } if tokens == nil { - if !credentialsOK { - http.Error(w, fmt.Sprintf("Authorization tokens are not accepted here: %v, and no anonymous user token is configured.", reasonNotAcceptingCredentials), http.StatusUnauthorized) - } else { - http.Error(w, fmt.Sprintf("No authorization token in request, and no anonymous user token is configured."), http.StatusUnauthorized) + tokens = reqTokens + if h.Cluster.Users.AnonymousUserToken != "" { + tokens = append(tokens, h.Cluster.Users.AnonymousUserToken) } - return } if len(targetPath) > 0 && targetPath[0] == "_" { @@ -365,26 +379,20 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { stripParts++ } - arv := h.clientPool.Get() - if arv == nil { - http.Error(w, "client pool error: "+h.clientPool.Err().Error(), http.StatusInternalServerError) - return - } - defer h.clientPool.Put(arv) - dirOpenMode := os.O_RDONLY if writeMethod[r.Method] { dirOpenMode = os.O_RDWR } - validToken := make(map[string]bool) + var tokenValid bool + var tokenScopeProblem bool var token string var tokenUser *arvados.User var sessionFS arvados.CustomFileSystem var session *cachedSession var collectionDir arvados.File for _, token = range tokens { - var statusErr interface{ HTTPStatus() int } + var statusErr errorWithHTTPStatus fs, sess, user, err := h.Cache.GetSession(token) if errors.As(err, &statusErr) && statusErr.HTTPStatus() == http.StatusUnauthorized { // bad token @@ -393,32 +401,39 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { http.Error(w, "cache error: "+err.Error(), http.StatusInternalServerError) return } - f, err := fs.OpenFile("by_id/"+collectionID, dirOpenMode, 0) - if errors.As(err, &statusErr) && statusErr.HTTPStatus() == http.StatusForbidden { - // collection id is outside token scope - validToken[token] = true - continue + if token != h.Cluster.Users.AnonymousUserToken { + tokenValid = true } - validToken[token] = true - if os.IsNotExist(err) { + f, err := fs.OpenFile(fsprefix, dirOpenMode, 0) + if errors.As(err, &statusErr) && + statusErr.HTTPStatus() == http.StatusForbidden && + token != h.Cluster.Users.AnonymousUserToken { + // collection id is outside scope of supplied + // token + tokenScopeProblem = true + sess.Release() + continue + } else if os.IsNotExist(err) { // collection does not exist or is not // readable using this token + sess.Release() continue } else if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + sess.Release() return } defer f.Close() + defer sess.Release() collectionDir, sessionFS, session, tokenUser = f, fs, sess, user break } - if forceReload { + if forceReload && collectionDir != nil { err := collectionDir.Sync() if err != nil { - var statusErr interface{ HTTPStatus() int } - if errors.As(err, &statusErr) { - http.Error(w, err.Error(), statusErr.HTTPStatus()) + if he := errorWithHTTPStatus(nil); errors.As(err, &he) { + http.Error(w, err.Error(), he.HTTPStatus()) } else { http.Error(w, err.Error(), http.StatusInternalServerError) } @@ -426,22 +441,27 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { } } if session == nil { - if pathToken || !credentialsOK { - // Either the URL is a "secret sharing link" - // that didn't work out (and asking the client - // for additional credentials would just be - // confusing), or we don't even accept - // credentials at this path. + if pathToken { + // The URL is a "secret sharing link" that + // didn't work out. Asking the client for + // additional credentials would just be + // confusing. http.Error(w, notFoundMessage, http.StatusNotFound) return } - for _, t := range reqTokens { - if validToken[t] { - // The client provided valid token(s), - // but the collection was not found. - http.Error(w, notFoundMessage, http.StatusNotFound) - return - } + if tokenValid { + // The client provided valid token(s), but the + // collection was not found. + http.Error(w, notFoundMessage, http.StatusNotFound) + return + } + if tokenScopeProblem { + // The client provided a valid token but + // fetching a collection returned 401, which + // means the token scope doesn't permit + // fetching that collection. + http.Error(w, notFoundMessage, http.StatusForbidden) + return } // The client's token was invalid (e.g., expired), or // the client didn't even provide one. Redirect to @@ -478,30 +498,52 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { target.RawQuery = redirkey + "=" + callback w.Header().Add("Location", target.String()) w.WriteHeader(http.StatusSeeOther) - } else { - w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"") - http.Error(w, unauthorizedMessage, http.StatusUnauthorized) + return } + if !credentialsOK { + http.Error(w, fmt.Sprintf("Authorization tokens are not accepted here: %v, and no anonymous user token is configured.", reasonNotAcceptingCredentials), http.StatusUnauthorized) + return + } + // If none of the above cases apply, suggest the + // user-agent (which is either a non-browser agent + // like wget, or a browser that can't redirect through + // a login flow) prompt the user for credentials. + w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"") + http.Error(w, unauthorizedMessage, http.StatusUnauthorized) return } + if r.Method == http.MethodGet || r.Method == http.MethodHead { + targetfnm := fsprefix + strings.Join(pathParts[stripParts:], "/") + if fi, err := sessionFS.Stat(targetfnm); err == nil && fi.IsDir() { + if !strings.HasSuffix(r.URL.Path, "/") { + h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK) + } else { + h.serveDirectory(w, r, fi.Name(), sessionFS, targetfnm, !useSiteFS) + } + return + } + } + var basename string if len(targetPath) > 0 { basename = targetPath[len(targetPath)-1] } - applyContentDispositionHdr(w, r, basename, attachment) - if arvadosclient.PDHMatch(collectionID) && writeMethod[r.Method] { - http.Error(w, errReadOnly.Error(), http.StatusMethodNotAllowed) + http.Error(w, webdavfs.ErrReadOnly.Error(), http.StatusMethodNotAllowed) return } if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) { http.Error(w, "Not permitted", http.StatusForbidden) return } - h.logUploadOrDownload(r, session.arvadosclient, sessionFS, "by_id/"+collectionID+"/"+strings.Join(targetPath, "/"), nil, tokenUser) + h.logUploadOrDownload(r, session.arvadosclient, sessionFS, fsprefix+strings.Join(targetPath, "/"), nil, tokenUser) - if writeMethod[r.Method] { + writing := writeMethod[r.Method] + locker := h.collectionLock(collectionID, writing) + defer locker.Unlock() + + if writing { // Save the collection only if/when all // webdav->filesystem operations succeed -- // and send a 500 error if the modified @@ -516,7 +558,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { // subsequent read requests. client := session.client.WithRequestID(r.Header.Get("X-Request-Id")) sessionFS = client.SiteFileSystem(session.keepclient) - writingDir, err := sessionFS.OpenFile("by_id/"+collectionID, os.O_RDONLY, 0) + writingDir, err := sessionFS.OpenFile(fsprefix, os.O_RDONLY, 0) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -544,32 +586,27 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { return nil }} } + if r.Method == http.MethodGet { + applyContentDispositionHdr(w, r, basename, attachment) + } + if webdavPrefix == "" { + webdavPrefix = "/" + strings.Join(pathParts[:stripParts], "/") + } wh := webdav.Handler{ - Prefix: "/" + strings.Join(pathParts[:stripParts], "/"), - FileSystem: &webdavFS{ - collfs: sessionFS, - prefix: "by_id/" + collectionID + "/", - writing: writeMethod[r.Method], - alwaysReadEOF: r.Method == "PROPFIND", + Prefix: webdavPrefix, + FileSystem: &webdavfs.FS{ + FileSystem: sessionFS, + Prefix: fsprefix, + Writing: writeMethod[r.Method], + AlwaysReadEOF: r.Method == "PROPFIND", }, - LockSystem: h.webdavLS, + LockSystem: webdavfs.NoLockSystem, Logger: func(r *http.Request, err error) { - if err != nil { + if err != nil && !os.IsNotExist(err) { ctxlog.FromContext(r.Context()).WithError(err).Error("error reported by webdav handler") } }, } - if r.Method == http.MethodGet || r.Method == http.MethodHead { - targetfnm := "by_id/" + collectionID + "/" + strings.Join(pathParts[stripParts:], "/") - if fi, err := sessionFS.Stat(targetfnm); err == nil && fi.IsDir() { - if !strings.HasSuffix(r.URL.Path, "/") { - h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK) - } else { - h.serveDirectory(w, r, fi.Name(), sessionFS, targetfnm, true) - } - return - } - } wh.ServeHTTP(w, r) if r.Method == http.MethodGet && w.WroteStatus() == http.StatusOK { wrote := int64(w.WroteBodyBytes()) @@ -587,88 +624,6 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { } } -func (h *handler) getClients(reqID, token string) (arv *arvadosclient.ArvadosClient, kc *keepclient.KeepClient, client *arvados.Client, release func(), err error) { - arv = h.clientPool.Get() - if arv == nil { - err = h.clientPool.Err() - return - } - release = func() { h.clientPool.Put(arv) } - arv.ApiToken = token - kc, err = keepclient.MakeKeepClient(arv) - if err != nil { - release() - return - } - kc.RequestID = reqID - client = (&arvados.Client{ - APIHost: arv.ApiServer, - AuthToken: arv.ApiToken, - Insecure: arv.ApiInsecure, - }).WithRequestID(reqID) - return -} - -func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK, attachment bool) { - if len(tokens) == 0 { - w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"") - http.Error(w, unauthorizedMessage, http.StatusUnauthorized) - return - } - if writeMethod[r.Method] { - http.Error(w, errReadOnly.Error(), http.StatusMethodNotAllowed) - return - } - - fs, sess, user, err := h.Cache.GetSession(tokens[0]) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - f, err := fs.Open(r.URL.Path) - if os.IsNotExist(err) { - http.Error(w, err.Error(), http.StatusNotFound) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - defer f.Close() - if fi, err := f.Stat(); err == nil && fi.IsDir() && r.Method == "GET" { - if !strings.HasSuffix(r.URL.Path, "/") { - h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK) - } else { - h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false) - } - return - } - - if !h.userPermittedToUploadOrDownload(r.Method, user) { - http.Error(w, "Not permitted", http.StatusForbidden) - return - } - h.logUploadOrDownload(r, sess.arvadosclient, fs, r.URL.Path, nil, user) - - if r.Method == "GET" { - _, basename := filepath.Split(r.URL.Path) - applyContentDispositionHdr(w, r, basename, attachment) - } - wh := webdav.Handler{ - FileSystem: &webdavFS{ - collfs: fs, - writing: writeMethod[r.Method], - alwaysReadEOF: r.Method == "PROPFIND", - }, - LockSystem: h.webdavLS, - Logger: func(_ *http.Request, err error) { - if err != nil { - ctxlog.FromContext(r.Context()).WithError(err).Error("error reported by webdav handler") - } - }, - } - wh.ServeHTTP(w, r) -} - var dirListingTemplate = ` @@ -999,3 +954,54 @@ func (h *handler) determineCollection(fs arvados.CustomFileSystem, path string) } return nil, "" } + +var lockTidyInterval = time.Minute * 10 + +// Lock the specified collection for reading or writing. Caller must +// call Unlock() on the returned Locker when the operation is +// finished. +func (h *handler) collectionLock(collectionID string, writing bool) sync.Locker { + h.lockMtx.Lock() + defer h.lockMtx.Unlock() + if time.Since(h.lockTidied) > lockTidyInterval { + // Periodically delete all locks that aren't in use. + h.lockTidied = time.Now() + for id, locker := range h.lock { + if locker.TryLock() { + locker.Unlock() + delete(h.lock, id) + } + } + } + locker := h.lock[collectionID] + if locker == nil { + locker = new(sync.RWMutex) + if h.lock == nil { + h.lock = map[string]*sync.RWMutex{} + } + h.lock[collectionID] = locker + } + if writing { + locker.Lock() + return locker + } else { + locker.RLock() + return locker.RLocker() + } +} + +func ServeCORSPreflight(w http.ResponseWriter, header http.Header) bool { + method := header.Get("Access-Control-Request-Method") + if method == "" { + return false + } + if !browserMethod[method] && !webdavMethod[method] { + w.WriteHeader(http.StatusMethodNotAllowed) + return true + } + w.Header().Set("Access-Control-Allow-Headers", corsAllowHeadersHeader) + w.Header().Set("Access-Control-Allow-Methods", "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK") + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Max-Age", "86400") + return true +}