X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a3eefdae693e4195ca4f46b7873f3055f4a0d377..5513ae5fa4abed9b3f871173f08d4a0aa52418af:/services/keep-web/s3.go diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go index 7cd1b1a896..4ee69f277d 100644 --- a/services/keep-web/s3.go +++ b/services/keep-web/s3.go @@ -5,23 +5,189 @@ package main import ( + "crypto/hmac" + "crypto/sha256" "encoding/xml" "errors" "fmt" + "hash" "io" "net/http" + "net/url" "os" "path/filepath" + "regexp" "sort" "strconv" "strings" + "time" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" "github.com/AdRoll/goamz/s3" ) -const s3MaxKeys = 1000 +const ( + s3MaxKeys = 1000 + s3SignAlgorithm = "AWS4-HMAC-SHA256" + s3MaxClockSkew = 5 * time.Minute +) + +func hmacstring(msg string, key []byte) []byte { + h := hmac.New(sha256.New, key) + io.WriteString(h, msg) + return h.Sum(nil) +} + +func hashdigest(h hash.Hash, payload string) string { + io.WriteString(h, payload) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +// Signing key for given secret key and request attrs. +func s3signatureKey(key, datestamp, regionName, serviceName string) []byte { + return hmacstring("aws4_request", + hmacstring(serviceName, + hmacstring(regionName, + hmacstring(datestamp, []byte("AWS4"+key))))) +} + +// Canonical query string for S3 V4 signature: sorted keys, spaces +// escaped as %20 instead of +, keyvalues joined with &. +func s3querystring(u *url.URL) string { + keys := make([]string, 0, len(u.Query())) + values := make(map[string]string, len(u.Query())) + for k, vs := range u.Query() { + k = strings.Replace(url.QueryEscape(k), "+", "%20", -1) + keys = append(keys, k) + for _, v := range vs { + v = strings.Replace(url.QueryEscape(v), "+", "%20", -1) + if values[k] != "" { + values[k] += "&" + } + values[k] += k + "=" + v + } + } + sort.Strings(keys) + for i, k := range keys { + keys[i] = values[k] + } + return strings.Join(keys, "&") +} + +func s3stringToSign(alg, scope, signedHeaders string, r *http.Request) (string, error) { + timefmt, timestr := "20060102T150405Z", r.Header.Get("X-Amz-Date") + if timestr == "" { + timefmt, timestr = time.RFC1123, r.Header.Get("Date") + } + t, err := time.Parse(timefmt, timestr) + if err != nil { + return "", fmt.Errorf("invalid timestamp %q: %s", timestr, err) + } + if skew := time.Now().Sub(t); skew < -s3MaxClockSkew || skew > s3MaxClockSkew { + return "", errors.New("exceeded max clock skew") + } + + var canonicalHeaders string + for _, h := range strings.Split(signedHeaders, ";") { + if h == "host" { + canonicalHeaders += h + ":" + r.Host + "\n" + } else { + canonicalHeaders += h + ":" + r.Header.Get(h) + "\n" + } + } + + canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, r.URL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256")) + ctxlog.FromContext(r.Context()).Debugf("s3stringToSign: canonicalRequest %s", canonicalRequest) + return fmt.Sprintf("%s\n%s\n%s\n%s", alg, r.Header.Get("X-Amz-Date"), scope, hashdigest(sha256.New(), canonicalRequest)), nil +} + +func s3signature(secretKey, scope, signedHeaders, stringToSign string) (string, error) { + // scope is {datestamp}/{region}/{service}/aws4_request + drs := strings.Split(scope, "/") + if len(drs) != 4 { + return "", fmt.Errorf("invalid scope %q", scope) + } + key := s3signatureKey(secretKey, drs[0], drs[1], drs[2]) + return hashdigest(hmac.New(sha256.New, key), stringToSign), nil +} + +var v2tokenUnderscore = regexp.MustCompile(`^v2_[a-z0-9]{5}-gj3su-[a-z0-9]{15}_`) + +func unescapeKey(key string) string { + if v2tokenUnderscore.MatchString(key) { + // Entire Arvados token, with "/" replaced by "_" to + // avoid colliding with the Authorization header + // format. + return strings.Replace(key, "_", "/", -1) + } else if s, err := url.PathUnescape(key); err == nil { + return s + } else { + return key + } +} + +// checks3signature verifies the given S3 V4 signature and returns the +// Arvados token that corresponds to the given accessKey. An error is +// returned if accessKey is not a valid token UUID or the signature +// does not match. +func (h *handler) checks3signature(r *http.Request) (string, error) { + var key, scope, signedHeaders, signature string + authstring := strings.TrimPrefix(r.Header.Get("Authorization"), s3SignAlgorithm+" ") + for _, cmpt := range strings.Split(authstring, ",") { + cmpt = strings.TrimSpace(cmpt) + split := strings.SplitN(cmpt, "=", 2) + switch { + case len(split) != 2: + // (?) ignore + case split[0] == "Credential": + keyandscope := strings.SplitN(split[1], "/", 2) + if len(keyandscope) == 2 { + key, scope = keyandscope[0], keyandscope[1] + } + case split[0] == "SignedHeaders": + signedHeaders = split[1] + case split[0] == "Signature": + signature = split[1] + } + } + + client := (&arvados.Client{ + APIHost: h.Config.cluster.Services.Controller.ExternalURL.Host, + Insecure: h.Config.cluster.TLS.Insecure, + }).WithRequestID(r.Header.Get("X-Request-Id")) + var aca arvados.APIClientAuthorization + var secret string + var err error + if len(key) == 27 && key[5:12] == "-gj3su-" { + // Access key is the UUID of an Arvados token, secret + // key is the secret part. + ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+h.Config.cluster.SystemRootToken) + err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/"+key, nil, nil) + secret = aca.APIToken + } else { + // Access key and secret key are both an entire + // Arvados token or OIDC access token. + ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+unescapeKey(key)) + err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/current", nil, nil) + secret = key + } + if err != nil { + ctxlog.FromContext(r.Context()).WithError(err).WithField("UUID", key).Info("token lookup failed") + return "", errors.New("invalid access key") + } + stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, r) + if err != nil { + return "", err + } + expect, err := s3signature(secret, scope, signedHeaders, stringToSign) + if err != nil { + return "", err + } else if expect != signature { + return "", fmt.Errorf("signature does not match (scope %q signedHeaders %q stringToSign %q)", scope, signedHeaders, stringToSign) + } + return aca.TokenV2(), nil +} // serveS3 handles r and returns true if r is a request from an S3 // client, otherwise it returns false. @@ -30,27 +196,17 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { if auth := r.Header.Get("Authorization"); strings.HasPrefix(auth, "AWS ") { split := strings.SplitN(auth[4:], ":", 2) if len(split) < 2 { - w.WriteHeader(http.StatusUnauthorized) + http.Error(w, "malformed Authorization header", http.StatusUnauthorized) return true } - token = split[0] - } else if strings.HasPrefix(auth, "AWS4-HMAC-SHA256 ") { - for _, cmpt := range strings.Split(auth[17:], ",") { - cmpt = strings.TrimSpace(cmpt) - split := strings.SplitN(cmpt, "=", 2) - if len(split) == 2 && split[0] == "Credential" { - keyandscope := strings.Split(split[1], "/") - if len(keyandscope[0]) > 0 { - token = keyandscope[0] - break - } - } - } - if token == "" { - w.WriteHeader(http.StatusBadRequest) - fmt.Println(w, "invalid V4 signature") + token = unescapeKey(split[0]) + } else if strings.HasPrefix(auth, s3SignAlgorithm+" ") { + t, err := h.checks3signature(r) + if err != nil { + http.Error(w, "signature verification failed: "+err.Error(), http.StatusForbidden) return true } + token = t } else { return false } @@ -65,7 +221,15 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { fs := client.SiteFileSystem(kc) fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution) - objectNameGiven := strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1 + var objectNameGiven bool + fspath := "/by_id" + if id := parseCollectionIDFromDNSName(r.Host); id != "" { + fspath += "/" + id + objectNameGiven = true + } else { + objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1 + } + fspath += r.URL.Path switch { case r.Method == http.MethodGet && !objectNameGiven: @@ -81,7 +245,6 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { } return true case r.Method == http.MethodGet || r.Method == http.MethodHead: - fspath := "/by_id" + r.URL.Path fi, err := fs.Stat(fspath) if r.Method == "HEAD" && !objectNameGiven { // HeadBucket @@ -115,7 +278,6 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { http.Error(w, "missing object name in PUT request", http.StatusBadRequest) return true } - fspath := "by_id" + r.URL.Path var objectIsDir bool if strings.HasSuffix(fspath, "/") { if !h.Config.cluster.Collections.S3FolderObjects { @@ -210,7 +372,6 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { http.Error(w, "missing object name in DELETE request", http.StatusBadRequest) return true } - fspath := "by_id" + r.URL.Path if strings.HasSuffix(fspath, "/") { fspath = strings.TrimSuffix(fspath, "/") fi, err := fs.Stat(fspath)