Merge branch '15888-remove-py2-from-test' into master

[arvados.git] / services / keep-web / s3.go

diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 12e294d9339710f861a491518c5dcb4d541f2f10..52cfede46642120dce520868d2d38758fb97cc12 100644 (file)
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -205,6 +205,54 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
                }
                w.WriteHeader(http.StatusOK)
                return true
+       case r.Method == http.MethodDelete:
+               if !objectNameGiven || r.URL.Path == "/" {
+                       http.Error(w, "missing object name in DELETE request", http.StatusBadRequest)
+                       return true
+               }
+               fspath := "by_id" + r.URL.Path
+               if strings.HasSuffix(fspath, "/") {
+                       fspath = strings.TrimSuffix(fspath, "/")
+                       fi, err := fs.Stat(fspath)
+                       if os.IsNotExist(err) {
+                               w.WriteHeader(http.StatusNoContent)
+                               return true
+                       } else if err != nil {
+                               http.Error(w, err.Error(), http.StatusInternalServerError)
+                               return true
+                       } else if !fi.IsDir() {
+                               // if "foo" exists and is a file, then
+                               // "foo/" doesn't exist, so we say
+                               // delete was successful.
+                               w.WriteHeader(http.StatusNoContent)
+                               return true
+                       }
+               } else if fi, err := fs.Stat(fspath); err == nil && fi.IsDir() {
+                       // if "foo" is a dir, it is visible via S3
+                       // only as "foo/", not "foo" -- so we leave
+                       // the dir alone and return 204 to indicate
+                       // that "foo" does not exist.
+                       w.WriteHeader(http.StatusNoContent)
+                       return true
+               }
+               err = fs.Remove(fspath)
+               if os.IsNotExist(err) {
+                       w.WriteHeader(http.StatusNoContent)
+                       return true
+               }
+               if err != nil {
+                       err = fmt.Errorf("rm failed: %w", err)
+                       http.Error(w, err.Error(), http.StatusBadRequest)
+                       return true
+               }
+               err = fs.Sync()
+               if err != nil {
+                       err = fmt.Errorf("sync failed: %w", err)
+                       http.Error(w, err.Error(), http.StatusInternalServerError)
+                       return true
+               }
+               w.WriteHeader(http.StatusNoContent)
+               return true
        default:
                http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
                return true
@@ -300,12 +348,30 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
                walkpath = ""
        }
 
-       resp := s3.ListResp{
-               Name:      strings.SplitN(r.URL.Path[1:], "/", 2)[0],
-               Prefix:    params.prefix,
-               Delimiter: params.delimiter,
-               Marker:    params.marker,
-               MaxKeys:   params.maxKeys,
+       type commonPrefix struct {
+               Prefix string
+       }
+       type listResp struct {
+               XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
+               s3.ListResp
+               // s3.ListResp marshals an empty tag when
+               // CommonPrefixes is nil, which confuses some clients.
+               // Fix by using this nested struct instead.
+               CommonPrefixes []commonPrefix
+               // Similarly, we need omitempty here, because an empty
+               // tag confuses some clients (e.g.,
+               // github.com/aws/aws-sdk-net never terminates its
+               // paging loop).
+               NextMarker string `xml:"NextMarker,omitempty"`
+       }
+       resp := listResp{
+               ListResp: s3.ListResp{
+                       Name:      strings.SplitN(r.URL.Path[1:], "/", 2)[0],
+                       Prefix:    params.prefix,
+                       Delimiter: params.delimiter,
+                       Marker:    params.marker,
+                       MaxKeys:   params.maxKeys,
+               },
        }
        commonPrefixes := map[string]bool{}
        err := walkFS(fs, strings.TrimSuffix(bucketdir+"/"+walkpath, "/"), true, func(path string, fi os.FileInfo) error {
@@ -387,18 +453,15 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
                return
        }
        if params.delimiter != "" {
+               resp.CommonPrefixes = make([]commonPrefix, 0, len(commonPrefixes))
                for prefix := range commonPrefixes {
-                       resp.CommonPrefixes = append(resp.CommonPrefixes, prefix)
-                       sort.Strings(resp.CommonPrefixes)
+                       resp.CommonPrefixes = append(resp.CommonPrefixes, commonPrefix{prefix})
                }
+               sort.Slice(resp.CommonPrefixes, func(i, j int) bool { return resp.CommonPrefixes[i].Prefix < resp.CommonPrefixes[j].Prefix })
        }
-       wrappedResp := struct {
-               XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
-               s3.ListResp
-       }{"", resp}
        w.Header().Set("Content-Type", "application/xml")
        io.WriteString(w, xml.Header)
-       if err := xml.NewEncoder(w).Encode(wrappedResp); err != nil {
+       if err := xml.NewEncoder(w).Encode(resp); err != nil {
                ctxlog.FromContext(r.Context()).WithError(err).Error("error writing xml response")
        }
 }