services/keep-web/s3_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/rand"
  11         "crypto/sha256"
  12         "fmt"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "os/exec"
  19         "strings"
  20         "sync"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/sdk/go/arvados"
  24         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  25         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         "github.com/AdRoll/goamz/aws"
  28         "github.com/AdRoll/goamz/s3"
  29         aws_aws "github.com/aws/aws-sdk-go/aws"
  30         aws_credentials "github.com/aws/aws-sdk-go/aws/credentials"
  31         aws_session "github.com/aws/aws-sdk-go/aws/session"
  32         aws_s3 "github.com/aws/aws-sdk-go/service/s3"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 type s3stage struct {
  37         arv        *arvados.Client
  38         ac         *arvadosclient.ArvadosClient
  39         kc         *keepclient.KeepClient
  40         proj       arvados.Group
  41         projbucket *s3.Bucket
  42         subproj    arvados.Group
  43         coll       arvados.Collection
  44         collbucket *s3.Bucket
  45 }
  46
  47 func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
  48         var proj, subproj arvados.Group
  49         var coll arvados.Collection
  50         arv := arvados.NewClientFromEnv()
  51         arv.AuthToken = arvadostest.ActiveToken
  52         err := arv.RequestAndDecode(&proj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
  53                 "group": map[string]interface{}{
  54                         "group_class": "project",
  55                         "name":        "keep-web s3 test",
  56                 },
  57                 "ensure_unique_name": true,
  58         })
  59         c.Assert(err, check.IsNil)
  60         err = arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
  61                 "group": map[string]interface{}{
  62                         "owner_uuid":  proj.UUID,
  63                         "group_class": "project",
  64                         "name":        "keep-web s3 test subproject",
  65                         "properties": map[string]interface{}{
  66                                 "subproject_properties_key": "subproject properties value",
  67                                 "invalid header key":        "this value will not be returned because key contains spaces",
  68                         },
  69                 },
  70         })
  71         c.Assert(err, check.IsNil)
  72         err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
  73                 "owner_uuid":    proj.UUID,
  74                 "name":          "keep-web s3 test collection",
  75                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
  76                 "properties": map[string]interface{}{
  77                         "string": "string value",
  78                         "array":  []string{"element1", "element2"},
  79                         "object": map[string]interface{}{"key": map[string]interface{}{"key2": "value"}},
  80                 },
  81         }})
  82         c.Assert(err, check.IsNil)
  83         ac, err := arvadosclient.New(arv)
  84         c.Assert(err, check.IsNil)
  85         kc, err := keepclient.MakeKeepClient(ac)
  86         c.Assert(err, check.IsNil)
  87         fs, err := coll.FileSystem(arv, kc)
  88         c.Assert(err, check.IsNil)
  89         f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
  90         c.Assert(err, check.IsNil)
  91         _, err = f.Write([]byte("⛵\n"))
  92         c.Assert(err, check.IsNil)
  93         err = f.Close()
  94         c.Assert(err, check.IsNil)
  95         err = fs.Sync()
  96         c.Assert(err, check.IsNil)
  97         err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
  98         c.Assert(err, check.IsNil)
  99
 100         auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
 101         region := aws.Region{
 102                 Name:       "zzzzz",
 103                 S3Endpoint: s.testServer.URL,
 104         }
 105         client := s3.New(*auth, region)
 106         client.Signature = aws.V4Signature
 107         return s3stage{
 108                 arv:  arv,
 109                 ac:   ac,
 110                 kc:   kc,
 111                 proj: proj,
 112                 projbucket: &s3.Bucket{
 113                         S3:   client,
 114                         Name: proj.UUID,
 115                 },
 116                 subproj: subproj,
 117                 coll:    coll,
 118                 collbucket: &s3.Bucket{
 119                         S3:   client,
 120                         Name: coll.UUID,
 121                 },
 122         }
 123 }
 124
 125 func (stage s3stage) teardown(c *check.C) {
 126         if stage.coll.UUID != "" {
 127                 err := stage.arv.RequestAndDecode(&stage.coll, "DELETE", "arvados/v1/collections/"+stage.coll.UUID, nil, nil)
 128                 c.Check(err, check.IsNil)
 129         }
 130         if stage.proj.UUID != "" {
 131                 err := stage.arv.RequestAndDecode(&stage.proj, "DELETE", "arvados/v1/groups/"+stage.proj.UUID, nil, nil)
 132                 c.Check(err, check.IsNil)
 133         }
 134 }
 135
 136 func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
 137         stage := s.s3setup(c)
 138         defer stage.teardown(c)
 139
 140         bucket := stage.collbucket
 141         for _, trial := range []struct {
 142                 success   bool
 143                 signature int
 144                 accesskey string
 145                 secretkey string
 146         }{
 147                 {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
 148                 {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
 149                 {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
 150                 {false, aws.V2Signature, "none", "none"},
 151                 {false, aws.V2Signature, "none", arvadostest.ActiveToken},
 152
 153                 {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
 154                 {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
 155                 {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
 156                 {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
 157                 {false, aws.V4Signature, arvadostest.ActiveToken, ""},
 158                 {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
 159                 {false, aws.V4Signature, "none", arvadostest.ActiveToken},
 160                 {false, aws.V4Signature, "none", "none"},
 161         } {
 162                 c.Logf("%#v", trial)
 163                 bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
 164                 bucket.S3.Signature = trial.signature
 165                 _, err := bucket.GetReader("emptyfile")
 166                 if trial.success {
 167                         c.Check(err, check.IsNil)
 168                 } else {
 169                         c.Check(err, check.NotNil)
 170                 }
 171         }
 172 }
 173
 174 func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
 175         stage := s.s3setup(c)
 176         defer stage.teardown(c)
 177
 178         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 179                 c.Logf("bucket %s", bucket.Name)
 180                 exists, err := bucket.Exists("")
 181                 c.Check(err, check.IsNil)
 182                 c.Check(exists, check.Equals, true)
 183         }
 184 }
 185
 186 func (s *IntegrationSuite) TestS3CollectionGetObject(c *check.C) {
 187         stage := s.s3setup(c)
 188         defer stage.teardown(c)
 189         s.testS3GetObject(c, stage.collbucket, "")
 190 }
 191 func (s *IntegrationSuite) TestS3ProjectGetObject(c *check.C) {
 192         stage := s.s3setup(c)
 193         defer stage.teardown(c)
 194         s.testS3GetObject(c, stage.projbucket, stage.coll.Name+"/")
 195 }
 196 func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix string) {
 197         rdr, err := bucket.GetReader(prefix + "emptyfile")
 198         c.Assert(err, check.IsNil)
 199         buf, err := ioutil.ReadAll(rdr)
 200         c.Check(err, check.IsNil)
 201         c.Check(len(buf), check.Equals, 0)
 202         err = rdr.Close()
 203         c.Check(err, check.IsNil)
 204
 205         // GetObject
 206         rdr, err = bucket.GetReader(prefix + "missingfile")
 207         c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 208         c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 209         c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
 210
 211         // HeadObject
 212         exists, err := bucket.Exists(prefix + "missingfile")
 213         c.Check(err, check.IsNil)
 214         c.Check(exists, check.Equals, false)
 215
 216         // GetObject
 217         rdr, err = bucket.GetReader(prefix + "sailboat.txt")
 218         c.Assert(err, check.IsNil)
 219         buf, err = ioutil.ReadAll(rdr)
 220         c.Check(err, check.IsNil)
 221         c.Check(buf, check.DeepEquals, []byte("⛵\n"))
 222         err = rdr.Close()
 223         c.Check(err, check.IsNil)
 224
 225         // HeadObject
 226         resp, err := bucket.Head(prefix+"sailboat.txt", nil)
 227         c.Check(err, check.IsNil)
 228         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 229         c.Check(resp.ContentLength, check.Equals, int64(4))
 230
 231         // HeadObject with superfluous leading slashes
 232         exists, err = bucket.Exists(prefix + "//sailboat.txt")
 233         c.Check(err, check.IsNil)
 234         c.Check(exists, check.Equals, true)
 235 }
 236
 237 func (s *IntegrationSuite) checkMetaEquals(c *check.C, resp *http.Response, expect map[string]string) {
 238         got := map[string]string{}
 239         for hk, hv := range resp.Header {
 240                 if k := strings.TrimPrefix(hk, "X-Amz-Meta-"); k != hk && len(hv) == 1 {
 241                         got[k] = hv[0]
 242                 }
 243         }
 244         c.Check(got, check.DeepEquals, expect)
 245 }
 246
 247 func (s *IntegrationSuite) TestS3PropertiesAsMetadata(c *check.C) {
 248         stage := s.s3setup(c)
 249         defer stage.teardown(c)
 250
 251         expectCollectionTags := map[string]string{
 252                 "String": "string value",
 253                 "Array":  `["element1","element2"]`,
 254                 "Object": `{"key":{"key2":"value"}}`,
 255         }
 256         expectSubprojectTags := map[string]string{
 257                 "Subproject_properties_key": "subproject properties value",
 258         }
 259
 260         resp, err := stage.collbucket.Head("sailboat.txt", nil)
 261         c.Assert(err, check.IsNil)
 262         s.checkMetaEquals(c, resp, expectCollectionTags)
 263
 264         resp, err = stage.projbucket.Head("keep-web s3 test collection/", nil)
 265         c.Assert(err, check.IsNil)
 266         s.checkMetaEquals(c, resp, expectCollectionTags)
 267
 268         resp, err = stage.projbucket.Head("keep-web s3 test collection/sailboat.txt", nil)
 269         c.Assert(err, check.IsNil)
 270         s.checkMetaEquals(c, resp, expectCollectionTags)
 271
 272         resp, err = stage.projbucket.Head("keep-web s3 test subproject/", nil)
 273         c.Assert(err, check.IsNil)
 274         s.checkMetaEquals(c, resp, expectSubprojectTags)
 275 }
 276
 277 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
 278         stage := s.s3setup(c)
 279         defer stage.teardown(c)
 280         s.testS3PutObjectSuccess(c, stage.collbucket, "")
 281 }
 282 func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) {
 283         stage := s.s3setup(c)
 284         defer stage.teardown(c)
 285         s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/")
 286 }
 287 func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string) {
 288         for _, trial := range []struct {
 289                 path        string
 290                 size        int
 291                 contentType string
 292         }{
 293                 {
 294                         path:        "newfile",
 295                         size:        128000000,
 296                         contentType: "application/octet-stream",
 297                 }, {
 298                         path:        "newdir/newfile",
 299                         size:        1 << 26,
 300                         contentType: "application/octet-stream",
 301                 }, {
 302                         path:        "/aaa",
 303                         size:        2,
 304                         contentType: "application/octet-stream",
 305                 }, {
 306                         path:        "//bbb",
 307                         size:        2,
 308                         contentType: "application/octet-stream",
 309                 }, {
 310                         path:        "ccc//",
 311                         size:        0,
 312                         contentType: "application/x-directory",
 313                 }, {
 314                         path:        "newdir1/newdir2/newfile",
 315                         size:        0,
 316                         contentType: "application/octet-stream",
 317                 }, {
 318                         path:        "newdir1/newdir2/newdir3/",
 319                         size:        0,
 320                         contentType: "application/x-directory",
 321                 },
 322         } {
 323                 c.Logf("=== %v", trial)
 324
 325                 objname := prefix + trial.path
 326
 327                 _, err := bucket.GetReader(objname)
 328                 if !c.Check(err, check.NotNil) {
 329                         continue
 330                 }
 331                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 332                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 333                 if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
 334                         continue
 335                 }
 336
 337                 buf := make([]byte, trial.size)
 338                 rand.Read(buf)
 339
 340                 err = bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 341                 c.Check(err, check.IsNil)
 342
 343                 rdr, err := bucket.GetReader(objname)
 344                 if strings.HasSuffix(trial.path, "/") && !s.handler.Cluster.Collections.S3FolderObjects {
 345                         c.Check(err, check.NotNil)
 346                         continue
 347                 } else if !c.Check(err, check.IsNil) {
 348                         continue
 349                 }
 350                 buf2, err := ioutil.ReadAll(rdr)
 351                 c.Check(err, check.IsNil)
 352                 c.Check(buf2, check.HasLen, len(buf))
 353                 c.Check(bytes.Equal(buf, buf2), check.Equals, true)
 354         }
 355 }
 356
 357 func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
 358         stage := s.s3setup(c)
 359         defer stage.teardown(c)
 360         bucket := stage.projbucket
 361
 362         for _, trial := range []struct {
 363                 path        string
 364                 size        int
 365                 contentType string
 366         }{
 367                 {
 368                         path:        "newfile",
 369                         size:        1234,
 370                         contentType: "application/octet-stream",
 371                 }, {
 372                         path:        "newdir/newfile",
 373                         size:        1234,
 374                         contentType: "application/octet-stream",
 375                 }, {
 376                         path:        "newdir2/",
 377                         size:        0,
 378                         contentType: "application/x-directory",
 379                 },
 380         } {
 381                 c.Logf("=== %v", trial)
 382
 383                 _, err := bucket.GetReader(trial.path)
 384                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 385                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 386                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 387
 388                 buf := make([]byte, trial.size)
 389                 rand.Read(buf)
 390
 391                 err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 392                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
 393                 c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
 394                 c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid (argument|operation)`)
 395
 396                 _, err = bucket.GetReader(trial.path)
 397                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 398                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 399                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 400         }
 401 }
 402
 403 func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) {
 404         stage := s.s3setup(c)
 405         defer stage.teardown(c)
 406         s.testS3DeleteObject(c, stage.collbucket, "")
 407 }
 408 func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) {
 409         stage := s.s3setup(c)
 410         defer stage.teardown(c)
 411         s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
 412 }
 413 func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
 414         s.handler.Cluster.Collections.S3FolderObjects = true
 415         for _, trial := range []struct {
 416                 path string
 417         }{
 418                 {"/"},
 419                 {"nonexistentfile"},
 420                 {"emptyfile"},
 421                 {"sailboat.txt"},
 422                 {"sailboat.txt/"},
 423                 {"emptydir"},
 424                 {"emptydir/"},
 425         } {
 426                 objname := prefix + trial.path
 427                 comment := check.Commentf("objname %q", objname)
 428
 429                 err := bucket.Del(objname)
 430                 if trial.path == "/" {
 431                         c.Check(err, check.NotNil)
 432                         continue
 433                 }
 434                 c.Check(err, check.IsNil, comment)
 435                 _, err = bucket.GetReader(objname)
 436                 c.Check(err, check.NotNil, comment)
 437         }
 438 }
 439
 440 func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) {
 441         stage := s.s3setup(c)
 442         defer stage.teardown(c)
 443         s.testS3PutObjectFailure(c, stage.collbucket, "")
 444 }
 445 func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
 446         stage := s.s3setup(c)
 447         defer stage.teardown(c)
 448         s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
 449 }
 450 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
 451         s.handler.Cluster.Collections.S3FolderObjects = false
 452
 453         var wg sync.WaitGroup
 454         for _, trial := range []struct {
 455                 path string
 456         }{
 457                 {
 458                         path: "emptyfile/newname", // emptyfile exists, see s3setup()
 459                 }, {
 460                         path: "emptyfile/", // emptyfile exists, see s3setup()
 461                 }, {
 462                         path: "emptydir", // dir already exists, see s3setup()
 463                 }, {
 464                         path: "emptydir/",
 465                 }, {
 466                         path: "emptydir//",
 467                 }, {
 468                         path: "newdir/",
 469                 }, {
 470                         path: "newdir//",
 471                 }, {
 472                         path: "/",
 473                 }, {
 474                         path: "//",
 475                 }, {
 476                         path: "",
 477                 },
 478         } {
 479                 trial := trial
 480                 wg.Add(1)
 481                 go func() {
 482                         defer wg.Done()
 483                         c.Logf("=== %v", trial)
 484
 485                         objname := prefix + trial.path
 486
 487                         buf := make([]byte, 1234)
 488                         rand.Read(buf)
 489
 490                         err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
 491                         if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
 492                                 return
 493                         }
 494
 495                         if objname != "" && objname != "/" {
 496                                 _, err = bucket.GetReader(objname)
 497                                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 498                                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 499                                 c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
 500                         }
 501                 }()
 502         }
 503         wg.Wait()
 504 }
 505
 506 func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
 507         fs, err := stage.coll.FileSystem(stage.arv, stage.kc)
 508         c.Assert(err, check.IsNil)
 509         for d := 0; d < dirs; d++ {
 510                 dir := fmt.Sprintf("dir%d", d)
 511                 c.Assert(fs.Mkdir(dir, 0755), check.IsNil)
 512                 for i := 0; i < filesPerDir; i++ {
 513                         f, err := fs.OpenFile(fmt.Sprintf("%s/file%d.txt", dir, i), os.O_CREATE|os.O_WRONLY, 0644)
 514                         c.Assert(err, check.IsNil)
 515                         c.Assert(f.Close(), check.IsNil)
 516                 }
 517         }
 518         c.Assert(fs.Sync(), check.IsNil)
 519 }
 520
 521 func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
 522         scope := "20200202/zzzzz/service/aws4_request"
 523         signedHeaders := "date"
 524         req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
 525         stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
 526         c.Assert(err, check.IsNil)
 527         sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
 528         c.Assert(err, check.IsNil)
 529         req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
 530 }
 531
 532 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
 533         stage := s.s3setup(c)
 534         defer stage.teardown(c)
 535         for _, trial := range []struct {
 536                 url            string
 537                 method         string
 538                 body           string
 539                 responseCode   int
 540                 responseRegexp []string
 541         }{
 542                 {
 543                         url:            "https://" + stage.collbucket.Name + ".example.com/",
 544                         method:         "GET",
 545                         responseCode:   http.StatusOK,
 546                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 547                 },
 548                 {
 549                         url:            "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
 550                         method:         "GET",
 551                         responseCode:   http.StatusOK,
 552                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 553                 },
 554                 {
 555                         url:            "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
 556                         method:         "GET",
 557                         responseCode:   http.StatusOK,
 558                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 559                 },
 560                 {
 561                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
 562                         method:         "GET",
 563                         responseCode:   http.StatusOK,
 564                         responseRegexp: []string{`⛵\n`},
 565                 },
 566                 {
 567                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 568                         method:       "PUT",
 569                         body:         "boop",
 570                         responseCode: http.StatusOK,
 571                 },
 572                 {
 573                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 574                         method:         "GET",
 575                         responseCode:   http.StatusOK,
 576                         responseRegexp: []string{`boop`},
 577                 },
 578                 {
 579                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 580                         method:       "GET",
 581                         responseCode: http.StatusNotFound,
 582                 },
 583                 {
 584                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 585                         method:       "PUT",
 586                         body:         "boop",
 587                         responseCode: http.StatusOK,
 588                 },
 589                 {
 590                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 591                         method:         "GET",
 592                         responseCode:   http.StatusOK,
 593                         responseRegexp: []string{`boop`},
 594                 },
 595         } {
 596                 url, err := url.Parse(trial.url)
 597                 c.Assert(err, check.IsNil)
 598                 req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
 599                 c.Assert(err, check.IsNil)
 600                 s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
 601                 rr := httptest.NewRecorder()
 602                 s.handler.ServeHTTP(rr, req)
 603                 resp := rr.Result()
 604                 c.Check(resp.StatusCode, check.Equals, trial.responseCode)
 605                 body, err := ioutil.ReadAll(resp.Body)
 606                 c.Assert(err, check.IsNil)
 607                 for _, re := range trial.responseRegexp {
 608                         c.Check(string(body), check.Matches, re)
 609                 }
 610         }
 611 }
 612
 613 func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
 614         stage := s.s3setup(c)
 615         defer stage.teardown(c)
 616         for _, trial := range []struct {
 617                 rawPath        string
 618                 normalizedPath string
 619         }{
 620                 {"/foo", "/foo"},                           // boring case
 621                 {"/foo%5fbar", "/foo_bar"},                 // _ must not be escaped
 622                 {"/foo%2fbar", "/foo/bar"},                 // / must not be escaped
 623                 {"/(foo)/[];,", "/%28foo%29/%5B%5D%3B%2C"}, // ()[];, must be escaped
 624                 {"/foo%5bbar", "/foo%5Bbar"},               // %XX must be uppercase
 625                 {"//foo///.bar", "/foo/.bar"},              // "//" and "///" must be squashed to "/"
 626         } {
 627                 c.Logf("trial %q", trial)
 628
 629                 date := time.Now().UTC().Format("20060102T150405Z")
 630                 scope := "20200202/zzzzz/S3/aws4_request"
 631                 canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
 632                 c.Logf("canonicalRequest %q", canonicalRequest)
 633                 expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
 634                 c.Logf("expected stringToSign %q", expect)
 635
 636                 req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
 637                 req.Header.Set("X-Amz-Date", date)
 638                 req.Host = "host.example.com"
 639                 c.Assert(err, check.IsNil)
 640
 641                 obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
 642                 if !c.Check(err, check.IsNil) {
 643                         continue
 644                 }
 645                 c.Check(obtained, check.Equals, expect)
 646         }
 647 }
 648
 649 func (s *IntegrationSuite) TestS3GetBucketLocation(c *check.C) {
 650         stage := s.s3setup(c)
 651         defer stage.teardown(c)
 652         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 653                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 654                 c.Check(err, check.IsNil)
 655                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 656                 req.URL.RawQuery = "location"
 657                 resp, err := http.DefaultClient.Do(req)
 658                 c.Assert(err, check.IsNil)
 659                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 660                 buf, err := ioutil.ReadAll(resp.Body)
 661                 c.Assert(err, check.IsNil)
 662                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<LocationConstraint><LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">zzzzz</LocationConstraint></LocationConstraint>\n")
 663         }
 664 }
 665
 666 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
 667         stage := s.s3setup(c)
 668         defer stage.teardown(c)
 669         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 670                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 671                 c.Check(err, check.IsNil)
 672                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 673                 req.URL.RawQuery = "versioning"
 674                 resp, err := http.DefaultClient.Do(req)
 675                 c.Assert(err, check.IsNil)
 676                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 677                 buf, err := ioutil.ReadAll(resp.Body)
 678                 c.Assert(err, check.IsNil)
 679                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>\n")
 680         }
 681 }
 682
 683 func (s *IntegrationSuite) TestS3UnsupportedAPIs(c *check.C) {
 684         stage := s.s3setup(c)
 685         defer stage.teardown(c)
 686         for _, trial := range []struct {
 687                 method   string
 688                 path     string
 689                 rawquery string
 690         }{
 691                 {"GET", "/", "acl&versionId=1234"},    // GetBucketAcl
 692                 {"GET", "/foo", "acl&versionId=1234"}, // GetObjectAcl
 693                 {"PUT", "/", "acl"},                   // PutBucketAcl
 694                 {"PUT", "/foo", "acl"},                // PutObjectAcl
 695                 {"DELETE", "/", "tagging"},            // DeleteBucketTagging
 696                 {"DELETE", "/foo", "tagging"},         // DeleteObjectTagging
 697         } {
 698                 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 699                         c.Logf("trial %v bucket %v", trial, bucket)
 700                         req, err := http.NewRequest(trial.method, bucket.URL(trial.path), nil)
 701                         c.Check(err, check.IsNil)
 702                         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 703                         req.URL.RawQuery = trial.rawquery
 704                         resp, err := http.DefaultClient.Do(req)
 705                         c.Assert(err, check.IsNil)
 706                         c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 707                         buf, err := ioutil.ReadAll(resp.Body)
 708                         c.Assert(err, check.IsNil)
 709                         c.Check(string(buf), check.Matches, "(?ms).*InvalidRequest.*API not supported.*")
 710                 }
 711         }
 712 }
 713
 714 // If there are no CommonPrefixes entries, the CommonPrefixes XML tag
 715 // should not appear at all.
 716 func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
 717         stage := s.s3setup(c)
 718         defer stage.teardown(c)
 719
 720         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 721         c.Assert(err, check.IsNil)
 722         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 723         req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
 724         resp, err := http.DefaultClient.Do(req)
 725         c.Assert(err, check.IsNil)
 726         buf, err := ioutil.ReadAll(resp.Body)
 727         c.Assert(err, check.IsNil)
 728         c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
 729 }
 730
 731 // If there is no delimiter in the request, or the results are not
 732 // truncated, the NextMarker XML tag should not appear in the response
 733 // body.
 734 func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
 735         stage := s.s3setup(c)
 736         defer stage.teardown(c)
 737
 738         for _, query := range []string{"prefix=e&delimiter=/", ""} {
 739                 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 740                 c.Assert(err, check.IsNil)
 741                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 742                 req.URL.RawQuery = query
 743                 resp, err := http.DefaultClient.Do(req)
 744                 c.Assert(err, check.IsNil)
 745                 buf, err := ioutil.ReadAll(resp.Body)
 746                 c.Assert(err, check.IsNil)
 747                 c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`)
 748         }
 749 }
 750
 751 // List response should include KeyCount field.
 752 func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
 753         stage := s.s3setup(c)
 754         defer stage.teardown(c)
 755
 756         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 757         c.Assert(err, check.IsNil)
 758         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 759         req.URL.RawQuery = "prefix=&delimiter=/"
 760         resp, err := http.DefaultClient.Do(req)
 761         c.Assert(err, check.IsNil)
 762         buf, err := ioutil.ReadAll(resp.Body)
 763         c.Assert(err, check.IsNil)
 764         c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
 765 }
 766
 767 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
 768         stage := s.s3setup(c)
 769         defer stage.teardown(c)
 770
 771         var markers int
 772         for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
 773                 dirs := 2
 774                 filesPerDir := 1001
 775                 stage.writeBigDirs(c, dirs, filesPerDir)
 776                 // Total # objects is:
 777                 //                 2 file entries from s3setup (emptyfile and sailboat.txt)
 778                 //                +1 fake "directory" marker from s3setup (emptydir) (if enabled)
 779                 //             +dirs fake "directory" marker from writeBigDirs (dir0/, dir1/) (if enabled)
 780                 // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
 781                 s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
 782                 s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
 783                 s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
 784         }
 785 }
 786 func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
 787         c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.handler.Cluster.Collections.S3FolderObjects)
 788         expectPageSize := pageSize
 789         if expectPageSize > 1000 {
 790                 expectPageSize = 1000
 791         }
 792         gotKeys := map[string]s3.Key{}
 793         nextMarker := ""
 794         pages := 0
 795         for {
 796                 resp, err := bucket.List(prefix, "", nextMarker, pageSize)
 797                 if !c.Check(err, check.IsNil) {
 798                         break
 799                 }
 800                 c.Check(len(resp.Contents) <= expectPageSize, check.Equals, true)
 801                 if pages++; !c.Check(pages <= (expectFiles/expectPageSize)+1, check.Equals, true) {
 802                         break
 803                 }
 804                 for _, key := range resp.Contents {
 805                         gotKeys[key.Key] = key
 806                         if strings.Contains(key.Key, "sailboat.txt") {
 807                                 c.Check(key.Size, check.Equals, int64(4))
 808                         }
 809                 }
 810                 if !resp.IsTruncated {
 811                         c.Check(resp.NextMarker, check.Equals, "")
 812                         break
 813                 }
 814                 if !c.Check(resp.NextMarker, check.Not(check.Equals), "") {
 815                         break
 816                 }
 817                 nextMarker = resp.NextMarker
 818         }
 819         c.Check(len(gotKeys), check.Equals, expectFiles)
 820 }
 821
 822 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
 823         for _, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
 824                 s.testS3CollectionListRollup(c)
 825         }
 826 }
 827
 828 func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
 829         stage := s.s3setup(c)
 830         defer stage.teardown(c)
 831
 832         dirs := 2
 833         filesPerDir := 500
 834         stage.writeBigDirs(c, dirs, filesPerDir)
 835         err := stage.collbucket.PutReader("dingbats", &bytes.Buffer{}, 0, "application/octet-stream", s3.Private, s3.Options{})
 836         c.Assert(err, check.IsNil)
 837         var allfiles []string
 838         for marker := ""; ; {
 839                 resp, err := stage.collbucket.List("", "", marker, 20000)
 840                 c.Check(err, check.IsNil)
 841                 for _, key := range resp.Contents {
 842                         if len(allfiles) == 0 || allfiles[len(allfiles)-1] != key.Key {
 843                                 allfiles = append(allfiles, key.Key)
 844                         }
 845                 }
 846                 marker = resp.NextMarker
 847                 if marker == "" {
 848                         break
 849                 }
 850         }
 851         markers := 0
 852         if s.handler.Cluster.Collections.S3FolderObjects {
 853                 markers = 1
 854         }
 855         c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
 856
 857         gotDirMarker := map[string]bool{}
 858         for _, name := range allfiles {
 859                 isDirMarker := strings.HasSuffix(name, "/")
 860                 if markers == 0 {
 861                         c.Check(isDirMarker, check.Equals, false, check.Commentf("name %q", name))
 862                 } else if isDirMarker {
 863                         gotDirMarker[name] = true
 864                 } else if i := strings.LastIndex(name, "/"); i >= 0 {
 865                         c.Check(gotDirMarker[name[:i+1]], check.Equals, true, check.Commentf("name %q", name))
 866                         gotDirMarker[name[:i+1]] = true // skip redundant complaints about this dir marker
 867                 }
 868         }
 869
 870         for _, trial := range []struct {
 871                 prefix    string
 872                 delimiter string
 873                 marker    string
 874         }{
 875                 {"", "", ""},
 876                 {"di", "/", ""},
 877                 {"di", "r", ""},
 878                 {"di", "n", ""},
 879                 {"dir0", "/", ""},
 880                 {"dir0/", "/", ""},
 881                 {"dir0/f", "/", ""},
 882                 {"dir0", "", ""},
 883                 {"dir0/", "", ""},
 884                 {"dir0/f", "", ""},
 885                 {"dir0", "/", "dir0/file14.txt"},       // no commonprefixes
 886                 {"", "", "dir0/file14.txt"},            // middle page, skip walking dir1
 887                 {"", "", "dir1/file14.txt"},            // middle page, skip walking dir0
 888                 {"", "", "dir1/file498.txt"},           // last page of results
 889                 {"dir1/file", "", "dir1/file498.txt"},  // last page of results, with prefix
 890                 {"dir1/file", "/", "dir1/file498.txt"}, // last page of results, with prefix + delimiter
 891                 {"dir1", "Z", "dir1/file498.txt"},      // delimiter "Z" never appears
 892                 {"dir2", "/", ""},                      // prefix "dir2" does not exist
 893                 {"", "/", ""},
 894         } {
 895                 c.Logf("\n\n=== trial %+v markers=%d", trial, markers)
 896
 897                 maxKeys := 20
 898                 resp, err := stage.collbucket.List(trial.prefix, trial.delimiter, trial.marker, maxKeys)
 899                 c.Check(err, check.IsNil)
 900                 if resp.IsTruncated && trial.delimiter == "" {
 901                         // goamz List method fills in the missing
 902                         // NextMarker field if resp.IsTruncated, so
 903                         // now we can't really tell whether it was
 904                         // sent by the server or by goamz. In cases
 905                         // where it should be empty but isn't, assume
 906                         // it's goamz's fault.
 907                         resp.NextMarker = ""
 908                 }
 909
 910                 var expectKeys []string
 911                 var expectPrefixes []string
 912                 var expectNextMarker string
 913                 var expectTruncated bool
 914                 for _, key := range allfiles {
 915                         full := len(expectKeys)+len(expectPrefixes) >= maxKeys
 916                         if !strings.HasPrefix(key, trial.prefix) || key < trial.marker {
 917                                 continue
 918                         } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
 919                                 prefix := key[:len(trial.prefix)+idx+1]
 920                                 if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
 921                                         // same prefix as previous key
 922                                 } else if full {
 923                                         expectNextMarker = key
 924                                         expectTruncated = true
 925                                 } else {
 926                                         expectPrefixes = append(expectPrefixes, prefix)
 927                                 }
 928                         } else if full {
 929                                 if trial.delimiter != "" {
 930                                         expectNextMarker = key
 931                                 }
 932                                 expectTruncated = true
 933                                 break
 934                         } else {
 935                                 expectKeys = append(expectKeys, key)
 936                         }
 937                 }
 938
 939                 var gotKeys []string
 940                 for _, key := range resp.Contents {
 941                         gotKeys = append(gotKeys, key.Key)
 942                 }
 943                 var gotPrefixes []string
 944                 for _, prefix := range resp.CommonPrefixes {
 945                         gotPrefixes = append(gotPrefixes, prefix)
 946                 }
 947                 commentf := check.Commentf("trial %+v markers=%d", trial, markers)
 948                 c.Check(gotKeys, check.DeepEquals, expectKeys, commentf)
 949                 c.Check(gotPrefixes, check.DeepEquals, expectPrefixes, commentf)
 950                 c.Check(resp.NextMarker, check.Equals, expectNextMarker, commentf)
 951                 c.Check(resp.IsTruncated, check.Equals, expectTruncated, commentf)
 952                 c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
 953         }
 954 }
 955
 956 func (s *IntegrationSuite) TestS3ListObjectsV2(c *check.C) {
 957         stage := s.s3setup(c)
 958         defer stage.teardown(c)
 959         dirs := 2
 960         filesPerDir := 40
 961         stage.writeBigDirs(c, dirs, filesPerDir)
 962
 963         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
 964                 Region:           aws_aws.String("auto"),
 965                 Endpoint:         aws_aws.String(s.testServer.URL),
 966                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
 967                 S3ForcePathStyle: aws_aws.Bool(true),
 968         }))
 969
 970         stringOrNil := func(s string) *string {
 971                 if s == "" {
 972                         return nil
 973                 } else {
 974                         return &s
 975                 }
 976         }
 977
 978         client := aws_s3.New(sess)
 979         ctx := context.Background()
 980
 981         for _, trial := range []struct {
 982                 prefix               string
 983                 delimiter            string
 984                 startAfter           string
 985                 maxKeys              int
 986                 expectKeys           int
 987                 expectCommonPrefixes map[string]bool
 988         }{
 989                 {
 990                         // Expect {filesPerDir plus the dir itself}
 991                         // for each dir, plus emptydir, emptyfile, and
 992                         // sailboat.txt.
 993                         expectKeys: (filesPerDir+1)*dirs + 3,
 994                 },
 995                 {
 996                         maxKeys:    15,
 997                         expectKeys: (filesPerDir+1)*dirs + 3,
 998                 },
 999                 {
1000                         startAfter: "dir0/z",
1001                         maxKeys:    15,
1002                         // Expect {filesPerDir plus the dir itself}
1003                         // for each dir except dir0, plus emptydir,
1004                         // emptyfile, and sailboat.txt.
1005                         expectKeys: (filesPerDir+1)*(dirs-1) + 3,
1006                 },
1007                 {
1008                         maxKeys:              1,
1009                         delimiter:            "/",
1010                         expectKeys:           2, // emptyfile, sailboat.txt
1011                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1012                 },
1013                 {
1014                         startAfter:           "dir0/z",
1015                         maxKeys:              15,
1016                         delimiter:            "/",
1017                         expectKeys:           2, // emptyfile, sailboat.txt
1018                         expectCommonPrefixes: map[string]bool{"dir1/": true, "emptydir/": true},
1019                 },
1020                 {
1021                         startAfter:           "dir0/file10.txt",
1022                         maxKeys:              15,
1023                         delimiter:            "/",
1024                         expectKeys:           2,
1025                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1026                 },
1027                 {
1028                         startAfter:           "dir0/file10.txt",
1029                         maxKeys:              15,
1030                         prefix:               "d",
1031                         delimiter:            "/",
1032                         expectKeys:           0,
1033                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true},
1034                 },
1035         } {
1036                 c.Logf("[trial %+v]", trial)
1037                 params := aws_s3.ListObjectsV2Input{
1038                         Bucket:     aws_aws.String(stage.collbucket.Name),
1039                         Prefix:     stringOrNil(trial.prefix),
1040                         Delimiter:  stringOrNil(trial.delimiter),
1041                         StartAfter: stringOrNil(trial.startAfter),
1042                         MaxKeys:    aws_aws.Int64(int64(trial.maxKeys)),
1043                 }
1044                 keySeen := map[string]bool{}
1045                 prefixSeen := map[string]bool{}
1046                 for {
1047                         result, err := client.ListObjectsV2WithContext(ctx, &params)
1048                         if !c.Check(err, check.IsNil) {
1049                                 break
1050                         }
1051                         c.Check(result.Name, check.DeepEquals, aws_aws.String(stage.collbucket.Name))
1052                         c.Check(result.Prefix, check.DeepEquals, aws_aws.String(trial.prefix))
1053                         c.Check(result.Delimiter, check.DeepEquals, aws_aws.String(trial.delimiter))
1054                         // The following two fields are expected to be
1055                         // nil (i.e., no tag in XML response) rather
1056                         // than "" when the corresponding request
1057                         // field was empty or nil.
1058                         c.Check(result.StartAfter, check.DeepEquals, stringOrNil(trial.startAfter))
1059                         c.Check(result.ContinuationToken, check.DeepEquals, params.ContinuationToken)
1060
1061                         if trial.maxKeys > 0 {
1062                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(trial.maxKeys)))
1063                                 c.Check(len(result.Contents)+len(result.CommonPrefixes) <= trial.maxKeys, check.Equals, true)
1064                         } else {
1065                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(s3MaxKeys)))
1066                         }
1067
1068                         for _, ent := range result.Contents {
1069                                 c.Assert(ent.Key, check.NotNil)
1070                                 c.Check(*ent.Key > trial.startAfter, check.Equals, true)
1071                                 c.Check(keySeen[*ent.Key], check.Equals, false, check.Commentf("dup key %q", *ent.Key))
1072                                 keySeen[*ent.Key] = true
1073                         }
1074                         for _, ent := range result.CommonPrefixes {
1075                                 c.Assert(ent.Prefix, check.NotNil)
1076                                 c.Check(strings.HasSuffix(*ent.Prefix, trial.delimiter), check.Equals, true, check.Commentf("bad CommonPrefix %q", *ent.Prefix))
1077                                 if strings.HasPrefix(trial.startAfter, *ent.Prefix) {
1078                                         // If we asked for
1079                                         // startAfter=dir0/file10.txt,
1080                                         // we expect dir0/ to be
1081                                         // returned as a common prefix
1082                                 } else {
1083                                         c.Check(*ent.Prefix > trial.startAfter, check.Equals, true)
1084                                 }
1085                                 c.Check(prefixSeen[*ent.Prefix], check.Equals, false, check.Commentf("dup common prefix %q", *ent.Prefix))
1086                                 prefixSeen[*ent.Prefix] = true
1087                         }
1088                         if *result.IsTruncated && c.Check(result.NextContinuationToken, check.Not(check.Equals), "") {
1089                                 params.ContinuationToken = aws_aws.String(*result.NextContinuationToken)
1090                         } else {
1091                                 break
1092                         }
1093                 }
1094                 c.Check(keySeen, check.HasLen, trial.expectKeys)
1095                 c.Check(prefixSeen, check.HasLen, len(trial.expectCommonPrefixes))
1096                 if len(trial.expectCommonPrefixes) > 0 {
1097                         c.Check(prefixSeen, check.DeepEquals, trial.expectCommonPrefixes)
1098                 }
1099         }
1100 }
1101
1102 func (s *IntegrationSuite) TestS3ListObjectsV2EncodingTypeURL(c *check.C) {
1103         stage := s.s3setup(c)
1104         defer stage.teardown(c)
1105         dirs := 2
1106         filesPerDir := 40
1107         stage.writeBigDirs(c, dirs, filesPerDir)
1108
1109         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1110                 Region:           aws_aws.String("auto"),
1111                 Endpoint:         aws_aws.String(s.testServer.URL),
1112                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1113                 S3ForcePathStyle: aws_aws.Bool(true),
1114         }))
1115
1116         client := aws_s3.New(sess)
1117         ctx := context.Background()
1118
1119         result, err := client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1120                 Bucket:       aws_aws.String(stage.collbucket.Name),
1121                 Prefix:       aws_aws.String("dir0/"),
1122                 Delimiter:    aws_aws.String("/"),
1123                 StartAfter:   aws_aws.String("dir0/"),
1124                 EncodingType: aws_aws.String("url"),
1125         })
1126         c.Assert(err, check.IsNil)
1127         c.Check(*result.Prefix, check.Equals, "dir0%2F")
1128         c.Check(*result.Delimiter, check.Equals, "%2F")
1129         c.Check(*result.StartAfter, check.Equals, "dir0%2F")
1130         for _, ent := range result.Contents {
1131                 c.Check(*ent.Key, check.Matches, "dir0%2F.*")
1132         }
1133         result, err = client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1134                 Bucket:       aws_aws.String(stage.collbucket.Name),
1135                 Delimiter:    aws_aws.String("/"),
1136                 EncodingType: aws_aws.String("url"),
1137         })
1138         c.Assert(err, check.IsNil)
1139         c.Check(*result.Delimiter, check.Equals, "%2F")
1140         c.Check(result.CommonPrefixes, check.HasLen, dirs+1)
1141         for _, ent := range result.CommonPrefixes {
1142                 c.Check(*ent.Prefix, check.Matches, ".*%2F")
1143         }
1144 }
1145
1146 // TestS3cmd checks compatibility with the s3cmd command line tool, if
1147 // it's installed. As of Debian buster, s3cmd is only in backports, so
1148 // `arvados-server install` don't install it, and this test skips if
1149 // it's not installed.
1150 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
1151         if _, err := exec.LookPath("s3cmd"); err != nil {
1152                 c.Skip("s3cmd not found")
1153                 return
1154         }
1155
1156         stage := s.s3setup(c)
1157         defer stage.teardown(c)
1158
1159         cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
1160         buf, err := cmd.CombinedOutput()
1161         c.Check(err, check.IsNil)
1162         c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
1163
1164         // This tests whether s3cmd's path normalization agrees with
1165         // keep-web's signature verification wrt chars like "|"
1166         // (neither reserved nor unreserved) and "," (not normally
1167         // percent-encoded in a path).
1168         tmpfile := c.MkDir() + "/dstfile"
1169         cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar", tmpfile)
1170         buf, err = cmd.CombinedOutput()
1171         c.Check(err, check.NotNil)
1172         c.Check(string(buf), check.Matches, `(?ms).*NoSuchKey.*\n`)
1173 }
1174
1175 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
1176         stage := s.s3setup(c)
1177         defer stage.teardown(c)
1178
1179         hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
1180         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
1181         c.Check(body, check.Equals, "⛵\n")
1182 }