16306: Merge branch 'master'

[arvados.git] / services / keep-web / s3.go

diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 7fb90789a55b164bd2465bf8d2454cf2d30e4f52..1c84976d2b1bf26622ba67619438f4536e6551f0 100644 (file)
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -75,6 +75,8 @@ func s3querystring(u *url.URL) string {
        return strings.Join(keys, "&")
 }
 
+var reMultipleSlashChars = regexp.MustCompile(`//+`)
+
 func s3stringToSign(alg, scope, signedHeaders string, r *http.Request) (string, error) {
        timefmt, timestr := "20060102T150405Z", r.Header.Get("X-Amz-Date")
        if timestr == "" {
@@ -97,7 +99,10 @@ func s3stringToSign(alg, scope, signedHeaders string, r *http.Request) (string,
                }
        }
 
-       canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, r.URL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256"))
+       normalizedURL := *r.URL
+       normalizedURL.RawPath = ""
+       normalizedURL.Path = reMultipleSlashChars.ReplaceAllString(normalizedURL.Path, "/")
+       canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, normalizedURL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256"))
        ctxlog.FromContext(r.Context()).Debugf("s3stringToSign: canonicalRequest %s", canonicalRequest)
        return fmt.Sprintf("%s\n%s\n%s\n%s", alg, r.Header.Get("X-Amz-Date"), scope, hashdigest(sha256.New(), canonicalRequest)), nil
 }
@@ -249,14 +254,17 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
        fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
 
        var objectNameGiven bool
+       var bucketName string
        fspath := "/by_id"
        if id := parseCollectionIDFromDNSName(r.Host); id != "" {
                fspath += "/" + id
+               bucketName = id
                objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
        } else {
+               bucketName = strings.SplitN(strings.TrimPrefix(r.URL.Path, "/"), "/", 2)[0]
                objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
        }
-       fspath += r.URL.Path
+       fspath += reMultipleSlashChars.ReplaceAllString(r.URL.Path, "/")
 
        switch {
        case r.Method == http.MethodGet && !objectNameGiven:
@@ -268,7 +276,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
                        fmt.Fprintln(w, `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`)
                } else {
                        // ListObjects
-                       h.s3list(w, r, fs)
+                       h.s3list(bucketName, w, r, fs)
                }
                return true
        case r.Method == http.MethodGet || r.Method == http.MethodHead:
@@ -504,15 +512,13 @@ func walkFS(fs arvados.CustomFileSystem, path string, isRoot bool, fn func(path
 
 var errDone = errors.New("done")
 
-func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
+func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
        var params struct {
-               bucket    string
                delimiter string
                marker    string
                maxKeys   int
                prefix    string
        }
-       params.bucket = strings.SplitN(r.URL.Path[1:], "/", 2)[0]
        params.delimiter = r.FormValue("delimiter")
        params.marker = r.FormValue("marker")
        if mk, _ := strconv.ParseInt(r.FormValue("max-keys"), 10, 64); mk > 0 && mk < s3MaxKeys {
@@ -522,7 +528,7 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
        }
        params.prefix = r.FormValue("prefix")
 
-       bucketdir := "by_id/" + params.bucket
+       bucketdir := "by_id/" + bucket
        // walkpath is the directory (relative to bucketdir) we need
        // to walk: the innermost directory that is guaranteed to
        // contain all paths that have the requested prefix. Examples:
@@ -557,7 +563,7 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
        }
        resp := listResp{
                ListResp: s3.ListResp{
-                       Name:      strings.SplitN(r.URL.Path[1:], "/", 2)[0],
+                       Name:      bucket,
                        Prefix:    params.prefix,
                        Delimiter: params.delimiter,
                        Marker:    params.marker,