X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c75f2e9b8a29bcdadcb092122f6d30e2930c08a3..2fe25e2b32042098106acead136fd3064bab30e3:/services/keep-web/s3_test.go diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go index b9a6d85ecb..e60b55c935 100644 --- a/services/keep-web/s3_test.go +++ b/services/keep-web/s3_test.go @@ -7,9 +7,11 @@ package main import ( "bytes" "crypto/rand" + "crypto/sha256" "fmt" "io/ioutil" "net/http" + "net/http/httptest" "net/url" "os" "os/exec" @@ -74,7 +76,7 @@ func (s *IntegrationSuite) s3setup(c *check.C) s3stage { auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour)) region := aws.Region{ - Name: s.testServer.Addr, + Name: "zzzzz", S3Endpoint: "http://" + s.testServer.Addr, } client := s3.New(*auth, region) @@ -201,6 +203,11 @@ func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix c.Check(err, check.IsNil) c.Check(resp.StatusCode, check.Equals, http.StatusOK) c.Check(resp.ContentLength, check.Equals, int64(4)) + + // HeadObject with superfluous leading slashes + exists, err = bucket.Exists(prefix + "//sailboat.txt") + c.Check(err, check.IsNil) + c.Check(exists, check.Equals, true) } func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) { @@ -227,6 +234,18 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, path: "newdir/newfile", size: 1 << 26, contentType: "application/octet-stream", + }, { + path: "/aaa", + size: 2, + contentType: "application/octet-stream", + }, { + path: "//bbb", + size: 2, + contentType: "application/octet-stream", + }, { + path: "ccc//", + size: 0, + contentType: "application/x-directory", }, { path: "newdir1/newdir2/newfile", size: 0, @@ -242,9 +261,14 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, objname := prefix + trial.path _, err := bucket.GetReader(objname) + if !c.Check(err, check.NotNil) { + continue + } c.Check(err.(*s3.Error).StatusCode, check.Equals, 404) c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`) - c.Assert(err, check.ErrorMatches, `The specified key does not exist.`) + if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) { + continue + } buf := make([]byte, trial.size) rand.Read(buf) @@ -303,7 +327,7 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) { err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{}) c.Check(err.(*s3.Error).StatusCode, check.Equals, 400) c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`) - c.Check(err, check.ErrorMatches, `(mkdir "by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`) + c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`) _, err = bucket.GetReader(trial.path) c.Check(err.(*s3.Error).StatusCode, check.Equals, 404) @@ -362,14 +386,6 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) { func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) { s.testServer.Config.cluster.Collections.S3FolderObjects = false - // Can't use V4 signature for these tests, because - // double-slash is incorrectly cleaned by the aws.V4Signature, - // resulting in a "bad signature" error. (Cleaning the path is - // appropriate for other services, but not in S3 where object - // names "foo//bar" and "foo/bar" are semantically different.) - bucket.S3.Auth = *(aws.NewAuth(arvadostest.ActiveToken, "none", "", time.Now().Add(time.Hour))) - bucket.S3.Signature = aws.V2Signature - var wg sync.WaitGroup for _, trial := range []struct { path string @@ -392,8 +408,6 @@ func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, path: "/", }, { path: "//", - }, { - path: "foo//bar", }, { path: "", }, @@ -440,6 +454,148 @@ func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) { c.Assert(fs.Sync(), check.IsNil) } +func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) { + scope := "20200202/zzzzz/service/aws4_request" + signedHeaders := "date" + req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123)) + stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req) + c.Assert(err, check.IsNil) + sig, err := s3signature(secret, scope, signedHeaders, stringToSign) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig) +} + +func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + for _, trial := range []struct { + url string + method string + body string + responseCode int + responseRegexp []string + }{ + { + url: "https://" + stage.collbucket.Name + ".example.com/", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`(?ms).*sailboat\.txt.*`}, + }, + { + url: "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`(?ms).*sailboat\.txt.*`}, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`(?ms).*sailboat\.txt.*`}, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`⛵\n`}, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep", + method: "PUT", + body: "boop", + responseCode: http.StatusOK, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`boop`}, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop", + method: "GET", + responseCode: http.StatusNotFound, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop", + method: "PUT", + body: "boop", + responseCode: http.StatusOK, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`boop`}, + }, + } { + url, err := url.Parse(trial.url) + c.Assert(err, check.IsNil) + req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body))) + c.Assert(err, check.IsNil) + s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken) + rr := httptest.NewRecorder() + s.testServer.Server.Handler.ServeHTTP(rr, req) + resp := rr.Result() + c.Check(resp.StatusCode, check.Equals, trial.responseCode) + body, err := ioutil.ReadAll(resp.Body) + c.Assert(err, check.IsNil) + for _, re := range trial.responseRegexp { + c.Check(string(body), check.Matches, re) + } + } +} + +func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + for _, trial := range []struct { + rawPath string + normalizedPath string + }{ + {"/foo", "/foo"}, // boring case + {"/foo%5fbar", "/foo_bar"}, // _ must not be escaped + {"/foo%2fbar", "/foo/bar"}, // / must not be escaped + {"/(foo)", "/%28foo%29"}, // () must be escaped + {"/foo%5bbar", "/foo%5Bbar"}, // %XX must be uppercase + } { + date := time.Now().UTC().Format("20060102T150405Z") + scope := "20200202/zzzzz/S3/aws4_request" + canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "") + c.Logf("canonicalRequest %q", canonicalRequest) + expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest)) + c.Logf("expected stringToSign %q", expect) + + req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil) + req.Header.Set("X-Amz-Date", date) + req.Host = "host.example.com" + c.Assert(err, check.IsNil) + + obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req) + if !c.Check(err, check.IsNil) { + continue + } + c.Check(obtained, check.Equals, expect) + } +} + +func (s *IntegrationSuite) TestS3GetBucketLocation(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} { + req, err := http.NewRequest("GET", bucket.URL("/"), nil) + c.Check(err, check.IsNil) + req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none") + req.URL.RawQuery = "location" + resp, err := http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml") + buf, err := ioutil.ReadAll(resp.Body) + c.Assert(err, check.IsNil) + c.Check(string(buf), check.Equals, "\nzzzzz\n") + } +} + func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) { stage := s.s3setup(c) defer stage.teardown(c) @@ -457,6 +613,37 @@ func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) { } } +func (s *IntegrationSuite) TestS3UnsupportedAPIs(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + for _, trial := range []struct { + method string + path string + rawquery string + }{ + {"GET", "/", "acl&versionId=1234"}, // GetBucketAcl + {"GET", "/foo", "acl&versionId=1234"}, // GetObjectAcl + {"PUT", "/", "acl"}, // PutBucketAcl + {"PUT", "/foo", "acl"}, // PutObjectAcl + {"DELETE", "/", "tagging"}, // DeleteBucketTagging + {"DELETE", "/foo", "tagging"}, // DeleteObjectTagging + } { + for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} { + c.Logf("trial %v bucket %v", trial, bucket) + req, err := http.NewRequest(trial.method, bucket.URL(trial.path), nil) + c.Check(err, check.IsNil) + req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none") + req.URL.RawQuery = trial.rawquery + resp, err := http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml") + buf, err := ioutil.ReadAll(resp.Body) + c.Assert(err, check.IsNil) + c.Check(string(buf), check.Matches, "(?ms).*InvalidRequest.*API not supported.*") + } + } +} + // If there are no CommonPrefixes entries, the CommonPrefixes XML tag // should not appear at all. func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {