X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/cd9e72bd95b32203231f6ccc96ddce9325b30a95..f5c6eb8491fb0d0a94bc1699ad7d84b054e0fec0:/services/keep-web/s3.go diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go index 90b75f8a30..38428cdab1 100644 --- a/services/keep-web/s3.go +++ b/services/keep-web/s3.go @@ -27,10 +27,7 @@ import ( "time" "git.arvados.org/arvados.git/sdk/go/arvados" - "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/ctxlog" - "git.arvados.org/arvados.git/sdk/go/keepclient" - "github.com/AdRoll/goamz/s3" ) const ( @@ -44,11 +41,17 @@ type commonPrefix struct { } type listV1Resp struct { - XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"` - s3.ListResp - // s3.ListResp marshals an empty tag when - // CommonPrefixes is nil, which confuses some clients. - // Fix by using this nested struct instead. + XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"` + Name string + Prefix string + Delimiter string + Marker string + MaxKeys int + IsTruncated bool + Contents []s3Key + // If we use a []string here, xml marshals an empty tag when + // CommonPrefixes is nil, which confuses some clients. Fix by + // using this nested struct instead. CommonPrefixes []commonPrefix // Similarly, we need omitempty here, because an empty // tag confuses some clients (e.g., @@ -62,7 +65,7 @@ type listV1Resp struct { type listV2Resp struct { XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"` IsTruncated bool - Contents []s3.Key + Contents []s3Key Name string Prefix string Delimiter string @@ -75,6 +78,21 @@ type listV2Resp struct { StartAfter string `xml:",omitempty"` } +type s3Key struct { + Key string + LastModified string + Size int64 + // The following fields are not populated, but are here in + // case clients rely on the keys being present in xml + // responses. + ETag string + StorageClass string + Owner struct { + ID string + DisplayName string + } +} + func hmacstring(msg string, key []byte) []byte { h := hmac.New(sha256.New, key) io.WriteString(h, msg) @@ -312,33 +330,18 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { return false } - var err error - var fs arvados.CustomFileSystem - var arvclient *arvadosclient.ArvadosClient - if r.Method == http.MethodGet || r.Method == http.MethodHead { - // Use a single session (cached FileSystem) across - // multiple read requests. - var sess *cachedSession - fs, sess, err = h.Cache.GetSession(token) - if err != nil { - s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError) - return true - } - arvclient = sess.arvadosclient - } else { + fs, sess, tokenUser, err := h.Cache.GetSession(token) + if err != nil { + s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError) + return true + } + readfs := fs + if writeMethod[r.Method] { // Create a FileSystem for this request, to avoid // exposing incomplete write operations to concurrent // requests. - var kc *keepclient.KeepClient - var release func() - var client *arvados.Client - arvclient, kc, client, release, err = h.getClients(r.Header.Get("X-Request-Id"), token) - if err != nil { - s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError) - return true - } - defer release() - fs = client.SiteFileSystem(kc) + client := sess.client.WithRequestID(r.Header.Get("X-Request-Id")) + fs = client.SiteFileSystem(sess.keepclient) fs.ForwardSlashNameSubstitution(h.Cluster.Collections.ForwardSlashNameSubstitution) } @@ -418,12 +421,11 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { return true } - tokenUser, err := h.Cache.GetTokenUser(token) if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) { http.Error(w, "Not permitted", http.StatusForbidden) return true } - h.logUploadOrDownload(r, arvclient, fs, fspath, nil, tokenUser) + h.logUploadOrDownload(r, sess.arvadosclient, fs, fspath, nil, tokenUser) // shallow copy r, and change URL path r := *r @@ -514,12 +516,11 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { } defer f.Close() - tokenUser, err := h.Cache.GetTokenUser(token) if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) { http.Error(w, "Not permitted", http.StatusForbidden) return true } - h.logUploadOrDownload(r, arvclient, fs, fspath, nil, tokenUser) + h.logUploadOrDownload(r, sess.arvadosclient, fs, fspath, nil, tokenUser) _, err = io.Copy(f, r.Body) if err != nil { @@ -534,14 +535,12 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { return true } } - err = fs.Sync() + err = h.syncCollection(fs, readfs, fspath) if err != nil { err = fmt.Errorf("sync failed: %w", err) s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError) return true } - // Ensure a subsequent read operation will see the changes. - h.Cache.ResetSession(token) w.WriteHeader(http.StatusOK) return true case r.Method == http.MethodDelete: @@ -588,14 +587,12 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest) return true } - err = fs.Sync() + err = h.syncCollection(fs, readfs, fspath) if err != nil { err = fmt.Errorf("sync failed: %w", err) s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError) return true } - // Ensure a subsequent read operation will see the changes. - h.Cache.ResetSession(token) w.WriteHeader(http.StatusNoContent) return true default: @@ -604,10 +601,38 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { } } +// Save modifications to the indicated collection in srcfs, then (if +// successful) ensure they are also reflected in dstfs. +func (h *handler) syncCollection(srcfs, dstfs arvados.CustomFileSystem, path string) error { + coll, _ := h.determineCollection(srcfs, path) + if coll == nil || coll.UUID == "" { + return errors.New("could not determine collection to sync") + } + d, err := srcfs.OpenFile("by_id/"+coll.UUID, os.O_RDWR, 0777) + if err != nil { + return err + } + defer d.Close() + err = d.Sync() + if err != nil { + return err + } + snap, err := d.Snapshot() + if err != nil { + return err + } + dstd, err := dstfs.OpenFile("by_id/"+coll.UUID, os.O_RDWR, 0777) + if err != nil { + return err + } + defer dstd.Close() + return dstd.Splice(snap) +} + func setFileInfoHeaders(header http.Header, fs arvados.CustomFileSystem, path string) error { maybeEncode := func(s string) string { for _, c := range s { - if c > '\u007f' { + if c > '\u007f' || c < ' ' { return mime.BEncoding.Encode("UTF-8", s) } } @@ -854,7 +879,7 @@ func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request, return filepath.SkipDir } } - resp.Contents = append(resp.Contents, s3.Key{ + resp.Contents = append(resp.Contents, s3Key{ Key: path, LastModified: fi.ModTime().UTC().Format("2006-01-02T15:04:05.999") + "Z", Size: filesize, @@ -918,15 +943,13 @@ func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request, CommonPrefixes: resp.CommonPrefixes, NextMarker: nextMarker, KeyCount: resp.KeyCount, - ListResp: s3.ListResp{ - IsTruncated: resp.IsTruncated, - Name: bucket, - Prefix: params.prefix, - Delimiter: params.delimiter, - Marker: params.marker, - MaxKeys: params.maxKeys, - Contents: resp.Contents, - }, + IsTruncated: resp.IsTruncated, + Name: bucket, + Prefix: params.prefix, + Delimiter: params.delimiter, + Marker: params.marker, + MaxKeys: params.maxKeys, + Contents: resp.Contents, } }