Merge branch '17009-s3-vhost-list'

refs #17009

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 7fb90789a55b164bd2465bf8d2454cf2d30e4f52..97201d2922116bf7db8cd8368557b4247093a074 100644 (file)
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -249,11 +249,14 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
        fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
 
        var objectNameGiven bool
+       var bucketName string
        fspath := "/by_id"
        if id := parseCollectionIDFromDNSName(r.Host); id != "" {
                fspath += "/" + id
+               bucketName = id
                objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
        } else {
+               bucketName = strings.SplitN(strings.TrimPrefix(r.URL.Path, "/"), "/", 2)[0]
                objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
        }
        fspath += r.URL.Path
@@ -268,7 +271,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
                        fmt.Fprintln(w, `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`)
                } else {
                        // ListObjects
-                       h.s3list(w, r, fs)
+                       h.s3list(bucketName, w, r, fs)
                }
                return true
        case r.Method == http.MethodGet || r.Method == http.MethodHead:
@@ -504,15 +507,13 @@ func walkFS(fs arvados.CustomFileSystem, path string, isRoot bool, fn func(path
 
 var errDone = errors.New("done")
 
-func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
+func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
        var params struct {
-               bucket    string
                delimiter string
                marker    string
                maxKeys   int
                prefix    string
        }
-       params.bucket = strings.SplitN(r.URL.Path[1:], "/", 2)[0]
        params.delimiter = r.FormValue("delimiter")
        params.marker = r.FormValue("marker")
        if mk, _ := strconv.ParseInt(r.FormValue("max-keys"), 10, 64); mk > 0 && mk < s3MaxKeys {
@@ -522,7 +523,7 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
        }
        params.prefix = r.FormValue("prefix")
 
-       bucketdir := "by_id/" + params.bucket
+       bucketdir := "by_id/" + bucket
        // walkpath is the directory (relative to bucketdir) we need
        // to walk: the innermost directory that is guaranteed to
        // contain all paths that have the requested prefix. Examples:
@@ -557,7 +558,7 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
        }
        resp := listResp{
                ListResp: s3.ListResp{
-                       Name:      strings.SplitN(r.URL.Path[1:], "/", 2)[0],
+                       Name:      bucket,
                        Prefix:    params.prefix,
                        Delimiter: params.delimiter,
                        Marker:    params.marker,

diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index 562d5296b8738a5a7761e45969bb4fa5a22ecf89..a6aab357e301e4b4703f2c72ef4a1a490ab65766 100644 (file)
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -10,6 +10,7 @@ import (
        "fmt"
        "io/ioutil"
        "net/http"
+       "net/http/httptest"
        "net/url"
        "os"
        "os/exec"
@@ -440,6 +441,70 @@ func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
        c.Assert(fs.Sync(), check.IsNil)
 }
 
+func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
+       stage := s.s3setup(c)
+       defer stage.teardown(c)
+       for _, trial := range []struct {
+               url            string
+               method         string
+               body           string
+               responseCode   int
+               responseRegexp []string
+       }{
+               {
+                       url:            "https://" + stage.collbucket.Name + ".example.com/",
+                       method:         "GET",
+                       responseCode:   http.StatusOK,
+                       responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+               },
+               {
+                       url:            "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
+                       method:         "GET",
+                       responseCode:   http.StatusOK,
+                       responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+               },
+               {
+                       url:            "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
+                       method:         "GET",
+                       responseCode:   http.StatusOK,
+                       responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+               },
+               {
+                       url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
+                       method:         "GET",
+                       responseCode:   http.StatusOK,
+                       responseRegexp: []string{`⛵\n`},
+               },
+               {
+                       url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
+                       method:       "PUT",
+                       body:         "boop",
+                       responseCode: http.StatusOK,
+               },
+               {
+                       url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
+                       method:         "GET",
+                       responseCode:   http.StatusOK,
+                       responseRegexp: []string{`boop`},
+               },
+       } {
+               url, err := url.Parse(trial.url)
+               c.Assert(err, check.IsNil)
+               req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
+               c.Assert(err, check.IsNil)
+               req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+               rr := httptest.NewRecorder()
+               s.testServer.Server.Handler.ServeHTTP(rr, req)
+               resp := rr.Result()
+               c.Check(resp.StatusCode, check.Equals, trial.responseCode)
+               body, err := ioutil.ReadAll(resp.Body)
+               c.Assert(err, check.IsNil)
+               for _, re := range trial.responseRegexp {
+                       c.Check(string(body), check.Matches, re)
+               }
+       }
+}
+
 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
        stage := s.s3setup(c)
        defer stage.teardown(c)