16809: Test S3 API using Amazon client library.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

diff --git a/lib/install/deps.go b/lib/install/deps.go
index dceb8f3ff78ff1c7a67b1cf3b4eae59483489582..b16d5bfc272b3ba3d3ddcfb8ad817e89ba316f9d 100644 (file)
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -146,7 +146,6 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
                        "python3-virtualenv",
                        "r-base",
                        "r-cran-testthat",
-                       "s3cmd",
                        "sudo",
                        "wget",
                        "xvfb",

diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index 8a476c9abbeaffbd4522bba3ee891214a9992463..51ac5ba98f2c72a49909dc11dd74d00691b46396 100644 (file)
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -666,6 +666,10 @@ func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
        }
 }
 
+// TestS3cmd checks compatibility with the s3cmd command line tool, if
+// it's installed. As of Debian buster, s3cmd is only in backports, so
+// `arvados-server install` don't install it, and this test skips if
+// it's not installed.
 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
        if _, err := exec.LookPath("s3cmd"); err != nil {
                c.Skip("s3cmd not found")

diff --git a/services/keep-web/s3aws_test.go b/services/keep-web/s3aws_test.go
new file mode 100644 (file)
index 0000000..d3db94e
--- /dev/null
+++ b/services/keep-web/s3aws_test.go
@@ -0,0 +1,78 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+       "bytes"
+       "context"
+       "io/ioutil"
+
+       "git.arvados.org/arvados.git/sdk/go/arvadostest"
+       "github.com/aws/aws-sdk-go-v2/aws"
+       "github.com/aws/aws-sdk-go-v2/aws/defaults"
+       "github.com/aws/aws-sdk-go-v2/aws/ec2metadata"
+       "github.com/aws/aws-sdk-go-v2/aws/ec2rolecreds"
+       "github.com/aws/aws-sdk-go-v2/aws/endpoints"
+       "github.com/aws/aws-sdk-go-v2/service/s3"
+       check "gopkg.in/check.v1"
+)
+
+func (s *IntegrationSuite) TestS3AWSSDK(c *check.C) {
+       stage := s.s3setup(c)
+       defer stage.teardown(c)
+
+       cfg := defaults.Config()
+       cfg.Credentials = aws.NewChainProvider([]aws.CredentialsProvider{
+               aws.NewStaticCredentialsProvider(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, ""),
+               ec2rolecreds.New(ec2metadata.New(cfg)),
+       })
+       cfg.EndpointResolver = aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
+               if service == "s3" {
+                       return aws.Endpoint{
+                               URL:           "http://" + s.testServer.Addr,
+                               SigningRegion: "custom-signing-region",
+                       }, nil
+               } else {
+                       return endpoints.NewDefaultResolver().ResolveEndpoint(service, region)
+               }
+       })
+       client := s3.New(cfg)
+       client.ForcePathStyle = true
+       listreq := client.ListObjectsV2Request(&s3.ListObjectsV2Input{
+               Bucket:            aws.String(arvadostest.FooCollection),
+               MaxKeys:           aws.Int64(100),
+               Prefix:            aws.String(""),
+               ContinuationToken: nil,
+       })
+       resp, err := listreq.Send(context.Background())
+       c.Assert(err, check.IsNil)
+       c.Check(resp.Contents, check.HasLen, 1)
+       for _, key := range resp.Contents {
+               c.Check(*key.Key, check.Equals, "foo")
+       }
+
+       p := make([]byte, 100000000)
+       for i := range p {
+               p[i] = byte('a')
+       }
+       putreq := client.PutObjectRequest(&s3.PutObjectInput{
+               Body:        bytes.NewReader(p),
+               Bucket:      aws.String(stage.collbucket.Name),
+               ContentType: aws.String("application/octet-stream"),
+               Key:         aws.String("aaaa"),
+       })
+       _, err = putreq.Send(context.Background())
+       c.Assert(err, check.IsNil)
+
+       getreq := client.GetObjectRequest(&s3.GetObjectInput{
+               Bucket: aws.String(stage.collbucket.Name),
+               Key:    aws.String("aaaa"),
+       })
+       getresp, err := getreq.Send(context.Background())
+       c.Assert(err, check.IsNil)
+       getdata, err := ioutil.ReadAll(getresp.Body)
+       c.Assert(err, check.IsNil)
+       c.Check(bytes.Equal(getdata, p), check.Equals, true)
+}