X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b521d2c5254e439e23cd750f86d55eadffb3e4b9..2af7b0336b2b92e38f6966b8bbc233c05704815d:/services/keep-web/s3_test.go diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go index c6d53238e8..786e68afec 100644 --- a/services/keep-web/s3_test.go +++ b/services/keep-web/s3_test.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "net/http" "os" + "os/exec" "strings" "sync" "time" @@ -70,12 +71,13 @@ func (s *IntegrationSuite) s3setup(c *check.C) s3stage { err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil) c.Assert(err, check.IsNil) - auth := aws.NewAuth(arvadostest.ActiveTokenV2, arvadostest.ActiveTokenV2, "", time.Now().Add(time.Hour)) + auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour)) region := aws.Region{ Name: s.testServer.Addr, S3Endpoint: "http://" + s.testServer.Addr, } client := s3.New(*auth, region) + client.Signature = aws.V4Signature return s3stage{ arv: arv, ac: ac, @@ -104,6 +106,40 @@ func (stage s3stage) teardown(c *check.C) { } } +func (s *IntegrationSuite) TestS3Signatures(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + + bucket := stage.collbucket + for _, trial := range []struct { + success bool + signature int + accesskey string + secretkey string + }{ + {true, aws.V2Signature, arvadostest.ActiveToken, "none"}, + {false, aws.V2Signature, "none", "none"}, + {false, aws.V2Signature, "none", arvadostest.ActiveToken}, + + {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken}, + {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken}, + {false, aws.V4Signature, arvadostest.ActiveToken, ""}, + {false, aws.V4Signature, arvadostest.ActiveToken, "none"}, + {false, aws.V4Signature, "none", arvadostest.ActiveToken}, + {false, aws.V4Signature, "none", "none"}, + } { + c.Logf("%#v", trial) + bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour))) + bucket.S3.Signature = trial.signature + _, err := bucket.GetReader("emptyfile") + if trial.success { + c.Check(err, check.IsNil) + } else { + c.Check(err, check.NotNil) + } + } +} + func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) { stage := s.s3setup(c) defer stage.teardown(c) @@ -154,9 +190,10 @@ func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix c.Check(err, check.IsNil) // HeadObject - exists, err = bucket.Exists(prefix + "sailboat.txt") + resp, err := bucket.Head(prefix+"sailboat.txt", nil) c.Check(err, check.IsNil) - c.Check(exists, check.Equals, true) + c.Check(resp.StatusCode, check.Equals, http.StatusOK) + c.Check(resp.ContentLength, check.Equals, int64(4)) } func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) { @@ -309,6 +346,15 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) { } func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) { s.testServer.Config.cluster.Collections.S3FolderObjects = false + + // Can't use V4 signature for these tests, because + // double-slash is incorrectly cleaned by the aws.V4Signature, + // resulting in a "bad signature" error. (Cleaning the path is + // appropriate for other services, but not in S3 where object + // names "foo//bar" and "foo/bar" are semantically different.) + bucket.S3.Auth = *(aws.NewAuth(arvadostest.ActiveToken, "none", "", time.Now().Add(time.Hour))) + bucket.S3.Signature = aws.V2Signature + var wg sync.WaitGroup for _, trial := range []struct { path string @@ -411,6 +457,42 @@ func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) { c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`) } +// If there is no delimiter in the request, or the results are not +// truncated, the NextMarker XML tag should not appear in the response +// body. +func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + + for _, query := range []string{"prefix=e&delimiter=/", ""} { + req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none") + req.URL.RawQuery = query + resp, err := http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + buf, err := ioutil.ReadAll(resp.Body) + c.Assert(err, check.IsNil) + c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`) + } +} + +// List response should include KeyCount field. +func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + + req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none") + req.URL.RawQuery = "prefix=&delimiter=/" + resp, err := http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + buf, err := ioutil.ReadAll(resp.Body) + c.Assert(err, check.IsNil) + c.Check(string(buf), check.Matches, `(?ms).*2.*`) +} + func (s *IntegrationSuite) TestS3CollectionList(c *check.C) { stage := s.s3setup(c) defer stage.teardown(c) @@ -599,3 +681,22 @@ func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) { c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker) } } + +// TestS3cmd checks compatibility with the s3cmd command line tool, if +// it's installed. As of Debian buster, s3cmd is only in backports, so +// `arvados-server install` don't install it, and this test skips if +// it's not installed. +func (s *IntegrationSuite) TestS3cmd(c *check.C) { + if _, err := exec.LookPath("s3cmd"); err != nil { + c.Skip("s3cmd not found") + return + } + + stage := s.s3setup(c) + defer stage.teardown(c) + + cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.Addr, "--host-bucket="+s.testServer.Addr, "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection) + buf, err := cmd.CombinedOutput() + c.Check(err, check.IsNil) + c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`) +}