17009: Fix s3 ListObjects endpoint with vhost-style requests.

[arvados.git] / services / keep-web / s3.go

diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 362d09ecd6f4ca9a045693c4d87fd6cc6a67a02a..97201d2922116bf7db8cd8368557b4247093a074 100644 (file)
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -16,6 +16,7 @@ import (
        "net/url"
        "os"
        "path/filepath"
+       "regexp"
        "sort"
        "strconv"
        "strings"
@@ -111,6 +112,21 @@ func s3signature(secretKey, scope, signedHeaders, stringToSign string) (string,
        return hashdigest(hmac.New(sha256.New, key), stringToSign), nil
 }
 
+var v2tokenUnderscore = regexp.MustCompile(`^v2_[a-z0-9]{5}-gj3su-[a-z0-9]{15}_`)
+
+func unescapeKey(key string) string {
+       if v2tokenUnderscore.MatchString(key) {
+               // Entire Arvados token, with "/" replaced by "_" to
+               // avoid colliding with the Authorization header
+               // format.
+               return strings.Replace(key, "_", "/", -1)
+       } else if s, err := url.PathUnescape(key); err == nil {
+               return s
+       } else {
+               return key
+       }
+}
+
 // checks3signature verifies the given S3 V4 signature and returns the
 // Arvados token that corresponds to the given accessKey. An error is
 // returned if accessKey is not a valid token UUID or the signature
@@ -152,7 +168,7 @@ func (h *handler) checks3signature(r *http.Request) (string, error) {
        } else {
                // Access key and secret key are both an entire
                // Arvados token or OIDC access token.
-               ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+key)
+               ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+unescapeKey(key))
                err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/current", nil, nil)
                secret = key
        }
@@ -170,20 +186,26 @@ func (h *handler) checks3signature(r *http.Request) (string, error) {
        } else if expect != signature {
                return "", fmt.Errorf("signature does not match (scope %q signedHeaders %q stringToSign %q)", scope, signedHeaders, stringToSign)
        }
-       return secret, nil
+       return aca.TokenV2(), nil
 }
 
 func s3ErrorResponse(w http.ResponseWriter, s3code string, message string, resource string, code int) {
        w.Header().Set("Content-Type", "application/xml")
        w.Header().Set("X-Content-Type-Options", "nosniff")
        w.WriteHeader(code)
-       fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
-<Error>
-  <Code>%v</Code>
-  <Message>%v</Message>
-  <Resource>%v</Resource>
-  <RequestId></RequestId>
-</Error>`, code, message, resource)
+       var errstruct struct {
+               Code      string
+               Message   string
+               Resource  string
+               RequestId string
+       }
+       errstruct.Code = s3code
+       errstruct.Message = message
+       errstruct.Resource = resource
+       errstruct.RequestId = ""
+       enc := xml.NewEncoder(w)
+       fmt.Fprint(w, xml.Header)
+       enc.EncodeElement(errstruct, xml.StartElement{Name: xml.Name{Local: "Error"}})
 }
 
 var NoSuchKey = "NoSuchKey"
@@ -204,7 +226,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
                        s3ErrorResponse(w, InvalidRequest, "malformed Authorization header", r.URL.Path, http.StatusUnauthorized)
                        return true
                }
-               token = split[0]
+               token = unescapeKey(split[0])
        } else if strings.HasPrefix(auth, s3SignAlgorithm+" ") {
                t, err := h.checks3signature(r)
                if err != nil {
@@ -226,7 +248,18 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
        fs := client.SiteFileSystem(kc)
        fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
 
-       objectNameGiven := strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+       var objectNameGiven bool
+       var bucketName string
+       fspath := "/by_id"
+       if id := parseCollectionIDFromDNSName(r.Host); id != "" {
+               fspath += "/" + id
+               bucketName = id
+               objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
+       } else {
+               bucketName = strings.SplitN(strings.TrimPrefix(r.URL.Path, "/"), "/", 2)[0]
+               objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+       }
+       fspath += r.URL.Path
 
        switch {
        case r.Method == http.MethodGet && !objectNameGiven:
@@ -238,11 +271,10 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
                        fmt.Fprintln(w, `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`)
                } else {
                        // ListObjects
-                       h.s3list(w, r, fs)
+                       h.s3list(bucketName, w, r, fs)
                }
                return true
        case r.Method == http.MethodGet || r.Method == http.MethodHead:
-               fspath := "/by_id" + r.URL.Path
                fi, err := fs.Stat(fspath)
                if r.Method == "HEAD" && !objectNameGiven {
                        // HeadBucket
@@ -276,7 +308,6 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
                        s3ErrorResponse(w, InvalidArgument, "Missing object name in PUT request.", r.URL.Path, http.StatusBadRequest)
                        return true
                }
-               fspath := "by_id" + r.URL.Path
                var objectIsDir bool
                if strings.HasSuffix(fspath, "/") {
                        if !h.Config.cluster.Collections.S3FolderObjects {
@@ -371,7 +402,6 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
                        s3ErrorResponse(w, InvalidArgument, "missing object name in DELETE request", r.URL.Path, http.StatusBadRequest)
                        return true
                }
-               fspath := "by_id" + r.URL.Path
                if strings.HasSuffix(fspath, "/") {
                        fspath = strings.TrimSuffix(fspath, "/")
                        fi, err := fs.Stat(fspath)
@@ -477,15 +507,13 @@ func walkFS(fs arvados.CustomFileSystem, path string, isRoot bool, fn func(path
 
 var errDone = errors.New("done")
 
-func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
+func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
        var params struct {
-               bucket    string
                delimiter string
                marker    string
                maxKeys   int
                prefix    string
        }
-       params.bucket = strings.SplitN(r.URL.Path[1:], "/", 2)[0]
        params.delimiter = r.FormValue("delimiter")
        params.marker = r.FormValue("marker")
        if mk, _ := strconv.ParseInt(r.FormValue("max-keys"), 10, 64); mk > 0 && mk < s3MaxKeys {
@@ -495,7 +523,7 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
        }
        params.prefix = r.FormValue("prefix")
 
-       bucketdir := "by_id/" + params.bucket
+       bucketdir := "by_id/" + bucket
        // walkpath is the directory (relative to bucketdir) we need
        // to walk: the innermost directory that is guaranteed to
        // contain all paths that have the requested prefix. Examples:
@@ -530,7 +558,7 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
        }
        resp := listResp{
                ListResp: s3.ListResp{
-                       Name:      strings.SplitN(r.URL.Path[1:], "/", 2)[0],
+                       Name:      bucket,
                        Prefix:    params.prefix,
                        Delimiter: params.delimiter,
                        Marker:    params.marker,