Merge branch '16774-keep-web-errors' refs #16774

[arvados.git] / services / keep-web / s3_test.go

diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index 8a476c9abbeaffbd4522bba3ee891214a9992463..b9a6d85ecb1a6bacc75b2a9b94b3ebc54c851f7e 100644 (file)
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -10,6 +10,7 @@ import (
        "fmt"
        "io/ioutil"
        "net/http"
+       "net/url"
        "os"
        "os/exec"
        "strings"
@@ -118,11 +119,15 @@ func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
                secretkey string
        }{
                {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
+               {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
+               {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
                {false, aws.V2Signature, "none", "none"},
                {false, aws.V2Signature, "none", arvadostest.ActiveToken},
 
                {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
                {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
+               {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
+               {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
                {false, aws.V4Signature, arvadostest.ActiveToken, ""},
                {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
                {false, aws.V4Signature, "none", arvadostest.ActiveToken},
@@ -173,7 +178,9 @@ func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix
 
        // GetObject
        rdr, err = bucket.GetReader(prefix + "missingfile")
-       c.Check(err, check.ErrorMatches, `404 Not Found`)
+       c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+       c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+       c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
 
        // HeadObject
        exists, err := bucket.Exists(prefix + "missingfile")
@@ -235,7 +242,9 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket,
                objname := prefix + trial.path
 
                _, err := bucket.GetReader(objname)
-               c.Assert(err, check.ErrorMatches, `404 Not Found`)
+               c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+               c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+               c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 
                buf := make([]byte, trial.size)
                rand.Read(buf)
@@ -284,16 +293,22 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
                c.Logf("=== %v", trial)
 
                _, err := bucket.GetReader(trial.path)
-               c.Assert(err, check.ErrorMatches, `404 Not Found`)
+               c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+               c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+               c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 
                buf := make([]byte, trial.size)
                rand.Read(buf)
 
                err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
-               c.Check(err, check.ErrorMatches, `400 Bad Request`)
+               c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
+               c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
+               c.Check(err, check.ErrorMatches, `(mkdir "by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
 
                _, err = bucket.GetReader(trial.path)
-               c.Assert(err, check.ErrorMatches, `404 Not Found`)
+               c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+               c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+               c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
        }
 }
 
@@ -395,13 +410,15 @@ func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket,
                        rand.Read(buf)
 
                        err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
-                       if !c.Check(err, check.ErrorMatches, `400 Bad.*`, check.Commentf("PUT %q should fail", objname)) {
+                       if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
                                return
                        }
 
                        if objname != "" && objname != "/" {
                                _, err = bucket.GetReader(objname)
-                               c.Check(err, check.ErrorMatches, `404 Not Found`, check.Commentf("GET %q should return 404", objname))
+                               c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+                               c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+                               c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
                        }
                }()
        }
@@ -477,6 +494,22 @@ func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
        }
 }
 
+// List response should include KeyCount field.
+func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
+       stage := s.s3setup(c)
+       defer stage.teardown(c)
+
+       req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
+       c.Assert(err, check.IsNil)
+       req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+       req.URL.RawQuery = "prefix=&delimiter=/"
+       resp, err := http.DefaultClient.Do(req)
+       c.Assert(err, check.IsNil)
+       buf, err := ioutil.ReadAll(resp.Body)
+       c.Assert(err, check.IsNil)
+       c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
+}
+
 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
        stage := s.s3setup(c)
        defer stage.teardown(c)
@@ -666,6 +699,10 @@ func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
        }
 }
 
+// TestS3cmd checks compatibility with the s3cmd command line tool, if
+// it's installed. As of Debian buster, s3cmd is only in backports, so
+// `arvados-server install` don't install it, and this test skips if
+// it's not installed.
 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
        if _, err := exec.LookPath("s3cmd"); err != nil {
                c.Skip("s3cmd not found")
@@ -680,3 +717,12 @@ func (s *IntegrationSuite) TestS3cmd(c *check.C) {
        c.Check(err, check.IsNil)
        c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
 }
+
+func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
+       stage := s.s3setup(c)
+       defer stage.teardown(c)
+
+       hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
+       c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
+       c.Check(body, check.Equals, "⛵\n")
+}