"encoding/xml"
"errors"
"fmt"
+ "hash"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
+ "regexp"
"sort"
"strconv"
"strings"
return h.Sum(nil)
}
+func hashdigest(h hash.Hash, payload string) string {
+ io.WriteString(h, payload)
+ return fmt.Sprintf("%x", h.Sum(nil))
+}
+
// Signing key for given secret key and request attrs.
func s3signatureKey(key, datestamp, regionName, serviceName string) []byte {
return hmacstring("aws4_request",
return strings.Join(keys, "&")
}
-func s3signature(alg, secretKey, scope, signedHeaders string, r *http.Request) (string, error) {
+func s3stringToSign(alg, scope, signedHeaders string, r *http.Request) (string, error) {
timefmt, timestr := "20060102T150405Z", r.Header.Get("X-Amz-Date")
if timestr == "" {
timefmt, timestr = time.RFC1123, r.Header.Get("Date")
var canonicalHeaders string
for _, h := range strings.Split(signedHeaders, ";") {
if h == "host" {
- canonicalHeaders += h + ":" + r.URL.Host + "\n"
+ canonicalHeaders += h + ":" + r.Host + "\n"
} else {
canonicalHeaders += h + ":" + r.Header.Get(h) + "\n"
}
}
- crhash := sha256.New()
- fmt.Fprintf(crhash, "%s\n%s\n%s\n%s\n%s\n%s", r.Method, r.URL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256"))
- crdigest := fmt.Sprintf("%x", crhash.Sum(nil))
-
- payload := fmt.Sprintf("%s\n%s\n%s\n%s", alg, r.Header.Get("X-Amz-Date"), scope, crdigest)
+ canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, r.URL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256"))
+ ctxlog.FromContext(r.Context()).Debugf("s3stringToSign: canonicalRequest %s", canonicalRequest)
+ return fmt.Sprintf("%s\n%s\n%s\n%s", alg, r.Header.Get("X-Amz-Date"), scope, hashdigest(sha256.New(), canonicalRequest)), nil
+}
+func s3signature(secretKey, scope, signedHeaders, stringToSign string) (string, error) {
// scope is {datestamp}/{region}/{service}/aws4_request
drs := strings.Split(scope, "/")
if len(drs) != 4 {
return "", fmt.Errorf("invalid scope %q", scope)
}
-
key := s3signatureKey(secretKey, drs[0], drs[1], drs[2])
- h := hmac.New(sha256.New, key)
- h.Write([]byte(payload))
- return fmt.Sprintf("%x", h.Sum(nil)), nil
+ return hashdigest(hmac.New(sha256.New, key), stringToSign), nil
+}
+
+var v2tokenUnderscore = regexp.MustCompile(`^v2_[a-z0-9]{5}-gj3su-[a-z0-9]{15}_`)
+
+func unescapeKey(key string) string {
+ if v2tokenUnderscore.MatchString(key) {
+ // Entire Arvados token, with "/" replaced by "_" to
+ // avoid colliding with the Authorization header
+ // format.
+ return strings.Replace(key, "_", "/", -1)
+ } else if s, err := url.PathUnescape(key); err == nil {
+ return s
+ } else {
+ return key
+ }
}
// checks3signature verifies the given S3 V4 signature and returns the
} else {
// Access key and secret key are both an entire
// Arvados token or OIDC access token.
- ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+key)
+ ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+unescapeKey(key))
err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/current", nil, nil)
secret = key
}
ctxlog.FromContext(r.Context()).WithError(err).WithField("UUID", key).Info("token lookup failed")
return "", errors.New("invalid access key")
}
- expect, err := s3signature(s3SignAlgorithm, secret, scope, signedHeaders, r)
+ stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, r)
+ if err != nil {
+ return "", err
+ }
+ expect, err := s3signature(secret, scope, signedHeaders, stringToSign)
if err != nil {
return "", err
} else if expect != signature {
- return "", errors.New("signature does not match")
+ return "", fmt.Errorf("signature does not match (scope %q signedHeaders %q stringToSign %q)", scope, signedHeaders, stringToSign)
}
- return secret, nil
+ return aca.TokenV2(), nil
}
// serveS3 handles r and returns true if r is a request from an S3
http.Error(w, "malformed Authorization header", http.StatusUnauthorized)
return true
}
- token = split[0]
+ token = unescapeKey(split[0])
} else if strings.HasPrefix(auth, s3SignAlgorithm+" ") {
t, err := h.checks3signature(r)
if err != nil {
fs := client.SiteFileSystem(kc)
fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
- objectNameGiven := strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+ var objectNameGiven bool
+ fspath := "/by_id"
+ if id := parseCollectionIDFromDNSName(r.Host); id != "" {
+ fspath += "/" + id
+ objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
+ } else {
+ objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+ }
+ fspath += r.URL.Path
switch {
case r.Method == http.MethodGet && !objectNameGiven:
}
return true
case r.Method == http.MethodGet || r.Method == http.MethodHead:
- fspath := "/by_id" + r.URL.Path
fi, err := fs.Stat(fspath)
if r.Method == "HEAD" && !objectNameGiven {
// HeadBucket
http.Error(w, "missing object name in PUT request", http.StatusBadRequest)
return true
}
- fspath := "by_id" + r.URL.Path
var objectIsDir bool
if strings.HasSuffix(fspath, "/") {
if !h.Config.cluster.Collections.S3FolderObjects {
http.Error(w, "missing object name in DELETE request", http.StatusBadRequest)
return true
}
- fspath := "by_id" + r.URL.Path
if strings.HasSuffix(fspath, "/") {
fspath = strings.TrimSuffix(fspath, "/")
fi, err := fs.Stat(fspath)
// github.com/aws/aws-sdk-net never terminates its
// paging loop).
NextMarker string `xml:"NextMarker,omitempty"`
+ // ListObjectsV2 has a KeyCount response field.
+ KeyCount int
}
resp := listResp{
ListResp: s3.ListResp{
}
sort.Slice(resp.CommonPrefixes, func(i, j int) bool { return resp.CommonPrefixes[i].Prefix < resp.CommonPrefixes[j].Prefix })
}
+ resp.KeyCount = len(resp.Contents)
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, xml.Header)
if err := xml.NewEncoder(w).Encode(resp); err != nil {
projects / arvados.git / blobdiff