X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e627df2797dae0d6fa95da61f1a58bb9fafe8240..8a27fe370239ecb8e50d53f46b45ed61203a35ca:/services/keep-web/s3.go diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go index 6ea9bf9f7a..59ab3cd438 100644 --- a/services/keep-web/s3.go +++ b/services/keep-web/s3.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepweb import ( "crypto/hmac" @@ -24,7 +24,9 @@ import ( "time" "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/ctxlog" + "git.arvados.org/arvados.git/sdk/go/keepclient" "github.com/AdRoll/goamz/s3" ) @@ -220,8 +222,8 @@ func (h *handler) checks3signature(r *http.Request) (string, error) { } client := (&arvados.Client{ - APIHost: h.Config.cluster.Services.Controller.ExternalURL.Host, - Insecure: h.Config.cluster.TLS.Insecure, + APIHost: h.Cluster.Services.Controller.ExternalURL.Host, + Insecure: h.Cluster.TLS.Insecure, }).WithRequestID(r.Header.Get("X-Request-Id")) var aca arvados.APIClientAuthorization var secret string @@ -229,7 +231,7 @@ func (h *handler) checks3signature(r *http.Request) (string, error) { if len(key) == 27 && key[5:12] == "-gj3su-" { // Access key is the UUID of an Arvados token, secret // key is the secret part. - ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+h.Config.cluster.SystemRootToken) + ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+h.Cluster.SystemRootToken) err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/"+key, nil, nil) secret = aca.APIToken } else { @@ -309,32 +311,38 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { var err error var fs arvados.CustomFileSystem + var arvclient *arvadosclient.ArvadosClient if r.Method == http.MethodGet || r.Method == http.MethodHead { // Use a single session (cached FileSystem) across // multiple read requests. - fs, err = h.Config.Cache.GetSession(token) + var sess *cachedSession + fs, sess, err = h.Cache.GetSession(token) if err != nil { s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError) return true } + arvclient = sess.arvadosclient } else { // Create a FileSystem for this request, to avoid // exposing incomplete write operations to concurrent // requests. - _, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), token) + var kc *keepclient.KeepClient + var release func() + var client *arvados.Client + arvclient, kc, client, release, err = h.getClients(r.Header.Get("X-Request-Id"), token) if err != nil { s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError) return true } defer release() fs = client.SiteFileSystem(kc) - fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution) + fs.ForwardSlashNameSubstitution(h.Cluster.Collections.ForwardSlashNameSubstitution) } var objectNameGiven bool var bucketName string fspath := "/by_id" - if id := parseCollectionIDFromDNSName(r.Host); id != "" { + if id := arvados.CollectionIDFromDNSName(r.Host); id != "" { fspath += "/" + id bucketName = id objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0 @@ -357,7 +365,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { w.Header().Set("Content-Type", "application/xml") io.WriteString(w, xml.Header) fmt.Fprintln(w, ``+ - h.Config.cluster.ClusterID+ + h.Cluster.ClusterID+ ``) } else if reRawQueryIndicatesAPI.MatchString(r.URL.RawQuery) { // GetBucketWebsite ("GET /bucketid/?website"), GetBucketTagging, etc. @@ -385,7 +393,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { } return true } - if err == nil && fi.IsDir() && objectNameGiven && strings.HasSuffix(fspath, "/") && h.Config.cluster.Collections.S3FolderObjects { + if err == nil && fi.IsDir() && objectNameGiven && strings.HasSuffix(fspath, "/") && h.Cluster.Collections.S3FolderObjects { w.Header().Set("Content-Type", "application/x-directory") w.WriteHeader(http.StatusOK) return true @@ -396,6 +404,14 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { s3ErrorResponse(w, NoSuchKey, "The specified key does not exist.", r.URL.Path, http.StatusNotFound) return true } + + tokenUser, err := h.Cache.GetTokenUser(token) + if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) { + http.Error(w, "Not permitted", http.StatusForbidden) + return true + } + h.logUploadOrDownload(r, arvclient, fs, fspath, nil, tokenUser) + // shallow copy r, and change URL path r := *r r.URL.Path = fspath @@ -413,7 +429,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { } var objectIsDir bool if strings.HasSuffix(fspath, "/") { - if !h.Config.cluster.Collections.S3FolderObjects { + if !h.Cluster.Collections.S3FolderObjects { s3ErrorResponse(w, InvalidArgument, "invalid object name: trailing slash", r.URL.Path, http.StatusBadRequest) return true } @@ -455,7 +471,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { return true } err = fs.Mkdir(dir, 0755) - if err == arvados.ErrInvalidArgument { + if errors.Is(err, arvados.ErrInvalidArgument) || errors.Is(err, arvados.ErrInvalidOperation) { // Cannot create a directory // here. err = fmt.Errorf("mkdir %q failed: %w", dir, err) @@ -479,6 +495,14 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { return true } defer f.Close() + + tokenUser, err := h.Cache.GetTokenUser(token) + if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) { + http.Error(w, "Not permitted", http.StatusForbidden) + return true + } + h.logUploadOrDownload(r, arvclient, fs, fspath, nil, tokenUser) + _, err = io.Copy(f, r.Body) if err != nil { err = fmt.Errorf("write to %q failed: %w", r.URL.Path, err) @@ -499,7 +523,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { return true } // Ensure a subsequent read operation will see the changes. - h.Config.Cache.ResetSession(token) + h.Cache.ResetSession(token) w.WriteHeader(http.StatusOK) return true case r.Method == http.MethodDelete: @@ -553,7 +577,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { return true } // Ensure a subsequent read operation will see the changes. - h.Config.Cache.ResetSession(token) + h.Cache.ResetSession(token) w.WriteHeader(http.StatusNoContent) return true default: @@ -732,7 +756,7 @@ func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request, if path < params.marker || path < params.prefix || path <= params.startAfter { return nil } - if fi.IsDir() && !h.Config.cluster.Collections.S3FolderObjects { + if fi.IsDir() && !h.Cluster.Collections.S3FolderObjects { // Note we don't add anything to // commonPrefixes here even if delimiter is // "/". We descend into the directory, and