services/keep-web/s3_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/rand"
  11         "crypto/sha256"
  12         "fmt"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "os/exec"
  19         "strings"
  20         "sync"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/sdk/go/arvados"
  24         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  25         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         "github.com/AdRoll/goamz/aws"
  28         "github.com/AdRoll/goamz/s3"
  29         aws_aws "github.com/aws/aws-sdk-go/aws"
  30         aws_credentials "github.com/aws/aws-sdk-go/aws/credentials"
  31         aws_session "github.com/aws/aws-sdk-go/aws/session"
  32         aws_s3 "github.com/aws/aws-sdk-go/service/s3"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 type s3stage struct {
  37         arv        *arvados.Client
  38         ac         *arvadosclient.ArvadosClient
  39         kc         *keepclient.KeepClient
  40         proj       arvados.Group
  41         projbucket *s3.Bucket
  42         subproj    arvados.Group
  43         coll       arvados.Collection
  44         collbucket *s3.Bucket
  45 }
  46
  47 func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
  48         var proj, subproj arvados.Group
  49         var coll arvados.Collection
  50         arv := arvados.NewClientFromEnv()
  51         arv.AuthToken = arvadostest.ActiveToken
  52         err := arv.RequestAndDecode(&proj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
  53                 "group": map[string]interface{}{
  54                         "group_class": "project",
  55                         "name":        "keep-web s3 test",
  56                         "properties": map[string]interface{}{
  57                                 "project-properties-key": "project properties value",
  58                         },
  59                 },
  60                 "ensure_unique_name": true,
  61         })
  62         c.Assert(err, check.IsNil)
  63         err = arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
  64                 "group": map[string]interface{}{
  65                         "owner_uuid":  proj.UUID,
  66                         "group_class": "project",
  67                         "name":        "keep-web s3 test subproject",
  68                         "properties": map[string]interface{}{
  69                                 "subproject_properties_key": "subproject properties value",
  70                                 "invalid header key":        "this value will not be returned because key contains spaces",
  71                         },
  72                 },
  73         })
  74         c.Assert(err, check.IsNil)
  75         err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
  76                 "owner_uuid":    proj.UUID,
  77                 "name":          "keep-web s3 test collection",
  78                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
  79                 "properties": map[string]interface{}{
  80                         "string": "string value",
  81                         "array":  []string{"element1", "element2"},
  82                         "object": map[string]interface{}{"key": map[string]interface{}{"key2": "value"}},
  83                 },
  84         }})
  85         c.Assert(err, check.IsNil)
  86         ac, err := arvadosclient.New(arv)
  87         c.Assert(err, check.IsNil)
  88         kc, err := keepclient.MakeKeepClient(ac)
  89         c.Assert(err, check.IsNil)
  90         fs, err := coll.FileSystem(arv, kc)
  91         c.Assert(err, check.IsNil)
  92         f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
  93         c.Assert(err, check.IsNil)
  94         _, err = f.Write([]byte("⛵\n"))
  95         c.Assert(err, check.IsNil)
  96         err = f.Close()
  97         c.Assert(err, check.IsNil)
  98         err = fs.Sync()
  99         c.Assert(err, check.IsNil)
 100         err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
 101         c.Assert(err, check.IsNil)
 102
 103         auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
 104         region := aws.Region{
 105                 Name:       "zzzzz",
 106                 S3Endpoint: s.testServer.URL,
 107         }
 108         client := s3.New(*auth, region)
 109         client.Signature = aws.V4Signature
 110         return s3stage{
 111                 arv:  arv,
 112                 ac:   ac,
 113                 kc:   kc,
 114                 proj: proj,
 115                 projbucket: &s3.Bucket{
 116                         S3:   client,
 117                         Name: proj.UUID,
 118                 },
 119                 subproj: subproj,
 120                 coll:    coll,
 121                 collbucket: &s3.Bucket{
 122                         S3:   client,
 123                         Name: coll.UUID,
 124                 },
 125         }
 126 }
 127
 128 func (stage s3stage) teardown(c *check.C) {
 129         if stage.coll.UUID != "" {
 130                 err := stage.arv.RequestAndDecode(&stage.coll, "DELETE", "arvados/v1/collections/"+stage.coll.UUID, nil, nil)
 131                 c.Check(err, check.IsNil)
 132         }
 133         if stage.proj.UUID != "" {
 134                 err := stage.arv.RequestAndDecode(&stage.proj, "DELETE", "arvados/v1/groups/"+stage.proj.UUID, nil, nil)
 135                 c.Check(err, check.IsNil)
 136         }
 137 }
 138
 139 func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
 140         stage := s.s3setup(c)
 141         defer stage.teardown(c)
 142
 143         bucket := stage.collbucket
 144         for _, trial := range []struct {
 145                 success   bool
 146                 signature int
 147                 accesskey string
 148                 secretkey string
 149         }{
 150                 {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
 151                 {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
 152                 {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
 153                 {false, aws.V2Signature, "none", "none"},
 154                 {false, aws.V2Signature, "none", arvadostest.ActiveToken},
 155
 156                 {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
 157                 {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
 158                 {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
 159                 {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
 160                 {false, aws.V4Signature, arvadostest.ActiveToken, ""},
 161                 {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
 162                 {false, aws.V4Signature, "none", arvadostest.ActiveToken},
 163                 {false, aws.V4Signature, "none", "none"},
 164         } {
 165                 c.Logf("%#v", trial)
 166                 bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
 167                 bucket.S3.Signature = trial.signature
 168                 _, err := bucket.GetReader("emptyfile")
 169                 if trial.success {
 170                         c.Check(err, check.IsNil)
 171                 } else {
 172                         c.Check(err, check.NotNil)
 173                 }
 174         }
 175 }
 176
 177 func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
 178         stage := s.s3setup(c)
 179         defer stage.teardown(c)
 180
 181         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 182                 c.Logf("bucket %s", bucket.Name)
 183                 exists, err := bucket.Exists("")
 184                 c.Check(err, check.IsNil)
 185                 c.Check(exists, check.Equals, true)
 186         }
 187 }
 188
 189 func (s *IntegrationSuite) TestS3CollectionGetObject(c *check.C) {
 190         stage := s.s3setup(c)
 191         defer stage.teardown(c)
 192         s.testS3GetObject(c, stage.collbucket, "")
 193 }
 194 func (s *IntegrationSuite) TestS3ProjectGetObject(c *check.C) {
 195         stage := s.s3setup(c)
 196         defer stage.teardown(c)
 197         s.testS3GetObject(c, stage.projbucket, stage.coll.Name+"/")
 198 }
 199 func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix string) {
 200         rdr, err := bucket.GetReader(prefix + "emptyfile")
 201         c.Assert(err, check.IsNil)
 202         buf, err := ioutil.ReadAll(rdr)
 203         c.Check(err, check.IsNil)
 204         c.Check(len(buf), check.Equals, 0)
 205         err = rdr.Close()
 206         c.Check(err, check.IsNil)
 207
 208         // GetObject
 209         rdr, err = bucket.GetReader(prefix + "missingfile")
 210         c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 211         c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 212         c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
 213
 214         // HeadObject
 215         exists, err := bucket.Exists(prefix + "missingfile")
 216         c.Check(err, check.IsNil)
 217         c.Check(exists, check.Equals, false)
 218
 219         // GetObject
 220         rdr, err = bucket.GetReader(prefix + "sailboat.txt")
 221         c.Assert(err, check.IsNil)
 222         buf, err = ioutil.ReadAll(rdr)
 223         c.Check(err, check.IsNil)
 224         c.Check(buf, check.DeepEquals, []byte("⛵\n"))
 225         err = rdr.Close()
 226         c.Check(err, check.IsNil)
 227
 228         // HeadObject
 229         resp, err := bucket.Head(prefix+"sailboat.txt", nil)
 230         c.Check(err, check.IsNil)
 231         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 232         c.Check(resp.ContentLength, check.Equals, int64(4))
 233
 234         // HeadObject with superfluous leading slashes
 235         exists, err = bucket.Exists(prefix + "//sailboat.txt")
 236         c.Check(err, check.IsNil)
 237         c.Check(exists, check.Equals, true)
 238 }
 239
 240 func (s *IntegrationSuite) checkMetaEquals(c *check.C, hdr http.Header, expect map[string]string) {
 241         got := map[string]string{}
 242         for hk, hv := range hdr {
 243                 if k := strings.TrimPrefix(hk, "X-Amz-Meta-"); k != hk && len(hv) == 1 {
 244                         got[k] = hv[0]
 245                 }
 246         }
 247         c.Check(got, check.DeepEquals, expect)
 248 }
 249
 250 func (s *IntegrationSuite) TestS3PropertiesAsMetadata(c *check.C) {
 251         stage := s.s3setup(c)
 252         defer stage.teardown(c)
 253
 254         expectCollectionTags := map[string]string{
 255                 "String": "string value",
 256                 "Array":  `["element1","element2"]`,
 257                 "Object": `{"key":{"key2":"value"}}`,
 258         }
 259         expectSubprojectTags := map[string]string{
 260                 "Subproject_properties_key": "subproject properties value",
 261         }
 262         expectProjectTags := map[string]string{
 263                 "Project-Properties-Key": "project properties value",
 264         }
 265
 266         c.Log("HEAD object with metadata from collection")
 267         resp, err := stage.collbucket.Head("sailboat.txt", nil)
 268         c.Assert(err, check.IsNil)
 269         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 270
 271         c.Log("GET object with metadata from collection")
 272         rdr, hdr, err := stage.collbucket.GetReaderWithHeaders("sailboat.txt")
 273         c.Assert(err, check.IsNil)
 274         content, err := ioutil.ReadAll(rdr)
 275         c.Check(err, check.IsNil)
 276         rdr.Close()
 277         c.Check(content, check.HasLen, 4)
 278         s.checkMetaEquals(c, hdr, expectCollectionTags)
 279
 280         c.Log("HEAD bucket with metadata from collection")
 281         resp, err = stage.collbucket.Head("/", nil)
 282         c.Assert(err, check.IsNil)
 283         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 284
 285         c.Log("HEAD directory placeholder with metadata from collection")
 286         resp, err = stage.projbucket.Head("keep-web s3 test collection/", nil)
 287         c.Assert(err, check.IsNil)
 288         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 289
 290         c.Log("HEAD file with metadata from collection")
 291         resp, err = stage.projbucket.Head("keep-web s3 test collection/sailboat.txt", nil)
 292         c.Assert(err, check.IsNil)
 293         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 294
 295         c.Log("HEAD directory placeholder with metadata from subproject")
 296         resp, err = stage.projbucket.Head("keep-web s3 test subproject/", nil)
 297         c.Assert(err, check.IsNil)
 298         s.checkMetaEquals(c, resp.Header, expectSubprojectTags)
 299
 300         c.Log("HEAD bucket with metadata from project")
 301         resp, err = stage.projbucket.Head("/", nil)
 302         c.Assert(err, check.IsNil)
 303         s.checkMetaEquals(c, resp.Header, expectProjectTags)
 304 }
 305
 306 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
 307         stage := s.s3setup(c)
 308         defer stage.teardown(c)
 309         s.testS3PutObjectSuccess(c, stage.collbucket, "")
 310 }
 311 func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) {
 312         stage := s.s3setup(c)
 313         defer stage.teardown(c)
 314         s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/")
 315 }
 316 func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string) {
 317         for _, trial := range []struct {
 318                 path        string
 319                 size        int
 320                 contentType string
 321         }{
 322                 {
 323                         path:        "newfile",
 324                         size:        128000000,
 325                         contentType: "application/octet-stream",
 326                 }, {
 327                         path:        "newdir/newfile",
 328                         size:        1 << 26,
 329                         contentType: "application/octet-stream",
 330                 }, {
 331                         path:        "/aaa",
 332                         size:        2,
 333                         contentType: "application/octet-stream",
 334                 }, {
 335                         path:        "//bbb",
 336                         size:        2,
 337                         contentType: "application/octet-stream",
 338                 }, {
 339                         path:        "ccc//",
 340                         size:        0,
 341                         contentType: "application/x-directory",
 342                 }, {
 343                         path:        "newdir1/newdir2/newfile",
 344                         size:        0,
 345                         contentType: "application/octet-stream",
 346                 }, {
 347                         path:        "newdir1/newdir2/newdir3/",
 348                         size:        0,
 349                         contentType: "application/x-directory",
 350                 },
 351         } {
 352                 c.Logf("=== %v", trial)
 353
 354                 objname := prefix + trial.path
 355
 356                 _, err := bucket.GetReader(objname)
 357                 if !c.Check(err, check.NotNil) {
 358                         continue
 359                 }
 360                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 361                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 362                 if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
 363                         continue
 364                 }
 365
 366                 buf := make([]byte, trial.size)
 367                 rand.Read(buf)
 368
 369                 err = bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 370                 c.Check(err, check.IsNil)
 371
 372                 rdr, err := bucket.GetReader(objname)
 373                 if strings.HasSuffix(trial.path, "/") && !s.handler.Cluster.Collections.S3FolderObjects {
 374                         c.Check(err, check.NotNil)
 375                         continue
 376                 } else if !c.Check(err, check.IsNil) {
 377                         continue
 378                 }
 379                 buf2, err := ioutil.ReadAll(rdr)
 380                 c.Check(err, check.IsNil)
 381                 c.Check(buf2, check.HasLen, len(buf))
 382                 c.Check(bytes.Equal(buf, buf2), check.Equals, true)
 383         }
 384 }
 385
 386 func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
 387         stage := s.s3setup(c)
 388         defer stage.teardown(c)
 389         bucket := stage.projbucket
 390
 391         for _, trial := range []struct {
 392                 path        string
 393                 size        int
 394                 contentType string
 395         }{
 396                 {
 397                         path:        "newfile",
 398                         size:        1234,
 399                         contentType: "application/octet-stream",
 400                 }, {
 401                         path:        "newdir/newfile",
 402                         size:        1234,
 403                         contentType: "application/octet-stream",
 404                 }, {
 405                         path:        "newdir2/",
 406                         size:        0,
 407                         contentType: "application/x-directory",
 408                 },
 409         } {
 410                 c.Logf("=== %v", trial)
 411
 412                 _, err := bucket.GetReader(trial.path)
 413                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 414                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 415                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 416
 417                 buf := make([]byte, trial.size)
 418                 rand.Read(buf)
 419
 420                 err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 421                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
 422                 c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
 423                 c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid (argument|operation)`)
 424
 425                 _, err = bucket.GetReader(trial.path)
 426                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 427                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 428                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 429         }
 430 }
 431
 432 func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) {
 433         stage := s.s3setup(c)
 434         defer stage.teardown(c)
 435         s.testS3DeleteObject(c, stage.collbucket, "")
 436 }
 437 func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) {
 438         stage := s.s3setup(c)
 439         defer stage.teardown(c)
 440         s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
 441 }
 442 func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
 443         s.handler.Cluster.Collections.S3FolderObjects = true
 444         for _, trial := range []struct {
 445                 path string
 446         }{
 447                 {"/"},
 448                 {"nonexistentfile"},
 449                 {"emptyfile"},
 450                 {"sailboat.txt"},
 451                 {"sailboat.txt/"},
 452                 {"emptydir"},
 453                 {"emptydir/"},
 454         } {
 455                 objname := prefix + trial.path
 456                 comment := check.Commentf("objname %q", objname)
 457
 458                 err := bucket.Del(objname)
 459                 if trial.path == "/" {
 460                         c.Check(err, check.NotNil)
 461                         continue
 462                 }
 463                 c.Check(err, check.IsNil, comment)
 464                 _, err = bucket.GetReader(objname)
 465                 c.Check(err, check.NotNil, comment)
 466         }
 467 }
 468
 469 func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) {
 470         stage := s.s3setup(c)
 471         defer stage.teardown(c)
 472         s.testS3PutObjectFailure(c, stage.collbucket, "")
 473 }
 474 func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
 475         stage := s.s3setup(c)
 476         defer stage.teardown(c)
 477         s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
 478 }
 479 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
 480         s.handler.Cluster.Collections.S3FolderObjects = false
 481
 482         var wg sync.WaitGroup
 483         for _, trial := range []struct {
 484                 path string
 485         }{
 486                 {
 487                         path: "emptyfile/newname", // emptyfile exists, see s3setup()
 488                 }, {
 489                         path: "emptyfile/", // emptyfile exists, see s3setup()
 490                 }, {
 491                         path: "emptydir", // dir already exists, see s3setup()
 492                 }, {
 493                         path: "emptydir/",
 494                 }, {
 495                         path: "emptydir//",
 496                 }, {
 497                         path: "newdir/",
 498                 }, {
 499                         path: "newdir//",
 500                 }, {
 501                         path: "/",
 502                 }, {
 503                         path: "//",
 504                 }, {
 505                         path: "",
 506                 },
 507         } {
 508                 trial := trial
 509                 wg.Add(1)
 510                 go func() {
 511                         defer wg.Done()
 512                         c.Logf("=== %v", trial)
 513
 514                         objname := prefix + trial.path
 515
 516                         buf := make([]byte, 1234)
 517                         rand.Read(buf)
 518
 519                         err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
 520                         if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
 521                                 return
 522                         }
 523
 524                         if objname != "" && objname != "/" {
 525                                 _, err = bucket.GetReader(objname)
 526                                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 527                                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 528                                 c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
 529                         }
 530                 }()
 531         }
 532         wg.Wait()
 533 }
 534
 535 func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
 536         fs, err := stage.coll.FileSystem(stage.arv, stage.kc)
 537         c.Assert(err, check.IsNil)
 538         for d := 0; d < dirs; d++ {
 539                 dir := fmt.Sprintf("dir%d", d)
 540                 c.Assert(fs.Mkdir(dir, 0755), check.IsNil)
 541                 for i := 0; i < filesPerDir; i++ {
 542                         f, err := fs.OpenFile(fmt.Sprintf("%s/file%d.txt", dir, i), os.O_CREATE|os.O_WRONLY, 0644)
 543                         c.Assert(err, check.IsNil)
 544                         c.Assert(f.Close(), check.IsNil)
 545                 }
 546         }
 547         c.Assert(fs.Sync(), check.IsNil)
 548 }
 549
 550 func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
 551         scope := "20200202/zzzzz/service/aws4_request"
 552         signedHeaders := "date"
 553         req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
 554         stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
 555         c.Assert(err, check.IsNil)
 556         sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
 557         c.Assert(err, check.IsNil)
 558         req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
 559 }
 560
 561 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
 562         stage := s.s3setup(c)
 563         defer stage.teardown(c)
 564         for _, trial := range []struct {
 565                 url            string
 566                 method         string
 567                 body           string
 568                 responseCode   int
 569                 responseRegexp []string
 570         }{
 571                 {
 572                         url:            "https://" + stage.collbucket.Name + ".example.com/",
 573                         method:         "GET",
 574                         responseCode:   http.StatusOK,
 575                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 576                 },
 577                 {
 578                         url:            "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
 579                         method:         "GET",
 580                         responseCode:   http.StatusOK,
 581                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 582                 },
 583                 {
 584                         url:            "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
 585                         method:         "GET",
 586                         responseCode:   http.StatusOK,
 587                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 588                 },
 589                 {
 590                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
 591                         method:         "GET",
 592                         responseCode:   http.StatusOK,
 593                         responseRegexp: []string{`⛵\n`},
 594                 },
 595                 {
 596                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 597                         method:       "PUT",
 598                         body:         "boop",
 599                         responseCode: http.StatusOK,
 600                 },
 601                 {
 602                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 603                         method:         "GET",
 604                         responseCode:   http.StatusOK,
 605                         responseRegexp: []string{`boop`},
 606                 },
 607                 {
 608                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 609                         method:       "GET",
 610                         responseCode: http.StatusNotFound,
 611                 },
 612                 {
 613                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 614                         method:       "PUT",
 615                         body:         "boop",
 616                         responseCode: http.StatusOK,
 617                 },
 618                 {
 619                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 620                         method:         "GET",
 621                         responseCode:   http.StatusOK,
 622                         responseRegexp: []string{`boop`},
 623                 },
 624         } {
 625                 url, err := url.Parse(trial.url)
 626                 c.Assert(err, check.IsNil)
 627                 req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
 628                 c.Assert(err, check.IsNil)
 629                 s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
 630                 rr := httptest.NewRecorder()
 631                 s.handler.ServeHTTP(rr, req)
 632                 resp := rr.Result()
 633                 c.Check(resp.StatusCode, check.Equals, trial.responseCode)
 634                 body, err := ioutil.ReadAll(resp.Body)
 635                 c.Assert(err, check.IsNil)
 636                 for _, re := range trial.responseRegexp {
 637                         c.Check(string(body), check.Matches, re)
 638                 }
 639         }
 640 }
 641
 642 func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
 643         stage := s.s3setup(c)
 644         defer stage.teardown(c)
 645         for _, trial := range []struct {
 646                 rawPath        string
 647                 normalizedPath string
 648         }{
 649                 {"/foo", "/foo"},                           // boring case
 650                 {"/foo%5fbar", "/foo_bar"},                 // _ must not be escaped
 651                 {"/foo%2fbar", "/foo/bar"},                 // / must not be escaped
 652                 {"/(foo)/[];,", "/%28foo%29/%5B%5D%3B%2C"}, // ()[];, must be escaped
 653                 {"/foo%5bbar", "/foo%5Bbar"},               // %XX must be uppercase
 654                 {"//foo///.bar", "/foo/.bar"},              // "//" and "///" must be squashed to "/"
 655         } {
 656                 c.Logf("trial %q", trial)
 657
 658                 date := time.Now().UTC().Format("20060102T150405Z")
 659                 scope := "20200202/zzzzz/S3/aws4_request"
 660                 canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
 661                 c.Logf("canonicalRequest %q", canonicalRequest)
 662                 expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
 663                 c.Logf("expected stringToSign %q", expect)
 664
 665                 req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
 666                 req.Header.Set("X-Amz-Date", date)
 667                 req.Host = "host.example.com"
 668                 c.Assert(err, check.IsNil)
 669
 670                 obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
 671                 if !c.Check(err, check.IsNil) {
 672                         continue
 673                 }
 674                 c.Check(obtained, check.Equals, expect)
 675         }
 676 }
 677
 678 func (s *IntegrationSuite) TestS3GetBucketLocation(c *check.C) {
 679         stage := s.s3setup(c)
 680         defer stage.teardown(c)
 681         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 682                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 683                 c.Check(err, check.IsNil)
 684                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 685                 req.URL.RawQuery = "location"
 686                 resp, err := http.DefaultClient.Do(req)
 687                 c.Assert(err, check.IsNil)
 688                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 689                 buf, err := ioutil.ReadAll(resp.Body)
 690                 c.Assert(err, check.IsNil)
 691                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<LocationConstraint><LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">zzzzz</LocationConstraint></LocationConstraint>\n")
 692         }
 693 }
 694
 695 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
 696         stage := s.s3setup(c)
 697         defer stage.teardown(c)
 698         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 699                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 700                 c.Check(err, check.IsNil)
 701                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 702                 req.URL.RawQuery = "versioning"
 703                 resp, err := http.DefaultClient.Do(req)
 704                 c.Assert(err, check.IsNil)
 705                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 706                 buf, err := ioutil.ReadAll(resp.Body)
 707                 c.Assert(err, check.IsNil)
 708                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>\n")
 709         }
 710 }
 711
 712 func (s *IntegrationSuite) TestS3UnsupportedAPIs(c *check.C) {
 713         stage := s.s3setup(c)
 714         defer stage.teardown(c)
 715         for _, trial := range []struct {
 716                 method   string
 717                 path     string
 718                 rawquery string
 719         }{
 720                 {"GET", "/", "acl&versionId=1234"},    // GetBucketAcl
 721                 {"GET", "/foo", "acl&versionId=1234"}, // GetObjectAcl
 722                 {"PUT", "/", "acl"},                   // PutBucketAcl
 723                 {"PUT", "/foo", "acl"},                // PutObjectAcl
 724                 {"DELETE", "/", "tagging"},            // DeleteBucketTagging
 725                 {"DELETE", "/foo", "tagging"},         // DeleteObjectTagging
 726         } {
 727                 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 728                         c.Logf("trial %v bucket %v", trial, bucket)
 729                         req, err := http.NewRequest(trial.method, bucket.URL(trial.path), nil)
 730                         c.Check(err, check.IsNil)
 731                         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 732                         req.URL.RawQuery = trial.rawquery
 733                         resp, err := http.DefaultClient.Do(req)
 734                         c.Assert(err, check.IsNil)
 735                         c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 736                         buf, err := ioutil.ReadAll(resp.Body)
 737                         c.Assert(err, check.IsNil)
 738                         c.Check(string(buf), check.Matches, "(?ms).*InvalidRequest.*API not supported.*")
 739                 }
 740         }
 741 }
 742
 743 // If there are no CommonPrefixes entries, the CommonPrefixes XML tag
 744 // should not appear at all.
 745 func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
 746         stage := s.s3setup(c)
 747         defer stage.teardown(c)
 748
 749         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 750         c.Assert(err, check.IsNil)
 751         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 752         req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
 753         resp, err := http.DefaultClient.Do(req)
 754         c.Assert(err, check.IsNil)
 755         buf, err := ioutil.ReadAll(resp.Body)
 756         c.Assert(err, check.IsNil)
 757         c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
 758 }
 759
 760 // If there is no delimiter in the request, or the results are not
 761 // truncated, the NextMarker XML tag should not appear in the response
 762 // body.
 763 func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
 764         stage := s.s3setup(c)
 765         defer stage.teardown(c)
 766
 767         for _, query := range []string{"prefix=e&delimiter=/", ""} {
 768                 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 769                 c.Assert(err, check.IsNil)
 770                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 771                 req.URL.RawQuery = query
 772                 resp, err := http.DefaultClient.Do(req)
 773                 c.Assert(err, check.IsNil)
 774                 buf, err := ioutil.ReadAll(resp.Body)
 775                 c.Assert(err, check.IsNil)
 776                 c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`)
 777         }
 778 }
 779
 780 // List response should include KeyCount field.
 781 func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
 782         stage := s.s3setup(c)
 783         defer stage.teardown(c)
 784
 785         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 786         c.Assert(err, check.IsNil)
 787         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 788         req.URL.RawQuery = "prefix=&delimiter=/"
 789         resp, err := http.DefaultClient.Do(req)
 790         c.Assert(err, check.IsNil)
 791         buf, err := ioutil.ReadAll(resp.Body)
 792         c.Assert(err, check.IsNil)
 793         c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
 794 }
 795
 796 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
 797         stage := s.s3setup(c)
 798         defer stage.teardown(c)
 799
 800         var markers int
 801         for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
 802                 dirs := 2
 803                 filesPerDir := 1001
 804                 stage.writeBigDirs(c, dirs, filesPerDir)
 805                 // Total # objects is:
 806                 //                 2 file entries from s3setup (emptyfile and sailboat.txt)
 807                 //                +1 fake "directory" marker from s3setup (emptydir) (if enabled)
 808                 //             +dirs fake "directory" marker from writeBigDirs (dir0/, dir1/) (if enabled)
 809                 // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
 810                 s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
 811                 s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
 812                 s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
 813         }
 814 }
 815 func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
 816         c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.handler.Cluster.Collections.S3FolderObjects)
 817         expectPageSize := pageSize
 818         if expectPageSize > 1000 {
 819                 expectPageSize = 1000
 820         }
 821         gotKeys := map[string]s3.Key{}
 822         nextMarker := ""
 823         pages := 0
 824         for {
 825                 resp, err := bucket.List(prefix, "", nextMarker, pageSize)
 826                 if !c.Check(err, check.IsNil) {
 827                         break
 828                 }
 829                 c.Check(len(resp.Contents) <= expectPageSize, check.Equals, true)
 830                 if pages++; !c.Check(pages <= (expectFiles/expectPageSize)+1, check.Equals, true) {
 831                         break
 832                 }
 833                 for _, key := range resp.Contents {
 834                         gotKeys[key.Key] = key
 835                         if strings.Contains(key.Key, "sailboat.txt") {
 836                                 c.Check(key.Size, check.Equals, int64(4))
 837                         }
 838                 }
 839                 if !resp.IsTruncated {
 840                         c.Check(resp.NextMarker, check.Equals, "")
 841                         break
 842                 }
 843                 if !c.Check(resp.NextMarker, check.Not(check.Equals), "") {
 844                         break
 845                 }
 846                 nextMarker = resp.NextMarker
 847         }
 848         c.Check(len(gotKeys), check.Equals, expectFiles)
 849 }
 850
 851 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
 852         for _, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
 853                 s.testS3CollectionListRollup(c)
 854         }
 855 }
 856
 857 func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
 858         stage := s.s3setup(c)
 859         defer stage.teardown(c)
 860
 861         dirs := 2
 862         filesPerDir := 500
 863         stage.writeBigDirs(c, dirs, filesPerDir)
 864         err := stage.collbucket.PutReader("dingbats", &bytes.Buffer{}, 0, "application/octet-stream", s3.Private, s3.Options{})
 865         c.Assert(err, check.IsNil)
 866         var allfiles []string
 867         for marker := ""; ; {
 868                 resp, err := stage.collbucket.List("", "", marker, 20000)
 869                 c.Check(err, check.IsNil)
 870                 for _, key := range resp.Contents {
 871                         if len(allfiles) == 0 || allfiles[len(allfiles)-1] != key.Key {
 872                                 allfiles = append(allfiles, key.Key)
 873                         }
 874                 }
 875                 marker = resp.NextMarker
 876                 if marker == "" {
 877                         break
 878                 }
 879         }
 880         markers := 0
 881         if s.handler.Cluster.Collections.S3FolderObjects {
 882                 markers = 1
 883         }
 884         c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
 885
 886         gotDirMarker := map[string]bool{}
 887         for _, name := range allfiles {
 888                 isDirMarker := strings.HasSuffix(name, "/")
 889                 if markers == 0 {
 890                         c.Check(isDirMarker, check.Equals, false, check.Commentf("name %q", name))
 891                 } else if isDirMarker {
 892                         gotDirMarker[name] = true
 893                 } else if i := strings.LastIndex(name, "/"); i >= 0 {
 894                         c.Check(gotDirMarker[name[:i+1]], check.Equals, true, check.Commentf("name %q", name))
 895                         gotDirMarker[name[:i+1]] = true // skip redundant complaints about this dir marker
 896                 }
 897         }
 898
 899         for _, trial := range []struct {
 900                 prefix    string
 901                 delimiter string
 902                 marker    string
 903         }{
 904                 {"", "", ""},
 905                 {"di", "/", ""},
 906                 {"di", "r", ""},
 907                 {"di", "n", ""},
 908                 {"dir0", "/", ""},
 909                 {"dir0/", "/", ""},
 910                 {"dir0/f", "/", ""},
 911                 {"dir0", "", ""},
 912                 {"dir0/", "", ""},
 913                 {"dir0/f", "", ""},
 914                 {"dir0", "/", "dir0/file14.txt"},       // no commonprefixes
 915                 {"", "", "dir0/file14.txt"},            // middle page, skip walking dir1
 916                 {"", "", "dir1/file14.txt"},            // middle page, skip walking dir0
 917                 {"", "", "dir1/file498.txt"},           // last page of results
 918                 {"dir1/file", "", "dir1/file498.txt"},  // last page of results, with prefix
 919                 {"dir1/file", "/", "dir1/file498.txt"}, // last page of results, with prefix + delimiter
 920                 {"dir1", "Z", "dir1/file498.txt"},      // delimiter "Z" never appears
 921                 {"dir2", "/", ""},                      // prefix "dir2" does not exist
 922                 {"", "/", ""},
 923         } {
 924                 c.Logf("\n\n=== trial %+v markers=%d", trial, markers)
 925
 926                 maxKeys := 20
 927                 resp, err := stage.collbucket.List(trial.prefix, trial.delimiter, trial.marker, maxKeys)
 928                 c.Check(err, check.IsNil)
 929                 if resp.IsTruncated && trial.delimiter == "" {
 930                         // goamz List method fills in the missing
 931                         // NextMarker field if resp.IsTruncated, so
 932                         // now we can't really tell whether it was
 933                         // sent by the server or by goamz. In cases
 934                         // where it should be empty but isn't, assume
 935                         // it's goamz's fault.
 936                         resp.NextMarker = ""
 937                 }
 938
 939                 var expectKeys []string
 940                 var expectPrefixes []string
 941                 var expectNextMarker string
 942                 var expectTruncated bool
 943                 for _, key := range allfiles {
 944                         full := len(expectKeys)+len(expectPrefixes) >= maxKeys
 945                         if !strings.HasPrefix(key, trial.prefix) || key < trial.marker {
 946                                 continue
 947                         } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
 948                                 prefix := key[:len(trial.prefix)+idx+1]
 949                                 if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
 950                                         // same prefix as previous key
 951                                 } else if full {
 952                                         expectNextMarker = key
 953                                         expectTruncated = true
 954                                 } else {
 955                                         expectPrefixes = append(expectPrefixes, prefix)
 956                                 }
 957                         } else if full {
 958                                 if trial.delimiter != "" {
 959                                         expectNextMarker = key
 960                                 }
 961                                 expectTruncated = true
 962                                 break
 963                         } else {
 964                                 expectKeys = append(expectKeys, key)
 965                         }
 966                 }
 967
 968                 var gotKeys []string
 969                 for _, key := range resp.Contents {
 970                         gotKeys = append(gotKeys, key.Key)
 971                 }
 972                 var gotPrefixes []string
 973                 for _, prefix := range resp.CommonPrefixes {
 974                         gotPrefixes = append(gotPrefixes, prefix)
 975                 }
 976                 commentf := check.Commentf("trial %+v markers=%d", trial, markers)
 977                 c.Check(gotKeys, check.DeepEquals, expectKeys, commentf)
 978                 c.Check(gotPrefixes, check.DeepEquals, expectPrefixes, commentf)
 979                 c.Check(resp.NextMarker, check.Equals, expectNextMarker, commentf)
 980                 c.Check(resp.IsTruncated, check.Equals, expectTruncated, commentf)
 981                 c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
 982         }
 983 }
 984
 985 func (s *IntegrationSuite) TestS3ListObjectsV2(c *check.C) {
 986         stage := s.s3setup(c)
 987         defer stage.teardown(c)
 988         dirs := 2
 989         filesPerDir := 40
 990         stage.writeBigDirs(c, dirs, filesPerDir)
 991
 992         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
 993                 Region:           aws_aws.String("auto"),
 994                 Endpoint:         aws_aws.String(s.testServer.URL),
 995                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
 996                 S3ForcePathStyle: aws_aws.Bool(true),
 997         }))
 998
 999         stringOrNil := func(s string) *string {
1000                 if s == "" {
1001                         return nil
1002                 } else {
1003                         return &s
1004                 }
1005         }
1006
1007         client := aws_s3.New(sess)
1008         ctx := context.Background()
1009
1010         for _, trial := range []struct {
1011                 prefix               string
1012                 delimiter            string
1013                 startAfter           string
1014                 maxKeys              int
1015                 expectKeys           int
1016                 expectCommonPrefixes map[string]bool
1017         }{
1018                 {
1019                         // Expect {filesPerDir plus the dir itself}
1020                         // for each dir, plus emptydir, emptyfile, and
1021                         // sailboat.txt.
1022                         expectKeys: (filesPerDir+1)*dirs + 3,
1023                 },
1024                 {
1025                         maxKeys:    15,
1026                         expectKeys: (filesPerDir+1)*dirs + 3,
1027                 },
1028                 {
1029                         startAfter: "dir0/z",
1030                         maxKeys:    15,
1031                         // Expect {filesPerDir plus the dir itself}
1032                         // for each dir except dir0, plus emptydir,
1033                         // emptyfile, and sailboat.txt.
1034                         expectKeys: (filesPerDir+1)*(dirs-1) + 3,
1035                 },
1036                 {
1037                         maxKeys:              1,
1038                         delimiter:            "/",
1039                         expectKeys:           2, // emptyfile, sailboat.txt
1040                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1041                 },
1042                 {
1043                         startAfter:           "dir0/z",
1044                         maxKeys:              15,
1045                         delimiter:            "/",
1046                         expectKeys:           2, // emptyfile, sailboat.txt
1047                         expectCommonPrefixes: map[string]bool{"dir1/": true, "emptydir/": true},
1048                 },
1049                 {
1050                         startAfter:           "dir0/file10.txt",
1051                         maxKeys:              15,
1052                         delimiter:            "/",
1053                         expectKeys:           2,
1054                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1055                 },
1056                 {
1057                         startAfter:           "dir0/file10.txt",
1058                         maxKeys:              15,
1059                         prefix:               "d",
1060                         delimiter:            "/",
1061                         expectKeys:           0,
1062                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true},
1063                 },
1064         } {
1065                 c.Logf("[trial %+v]", trial)
1066                 params := aws_s3.ListObjectsV2Input{
1067                         Bucket:     aws_aws.String(stage.collbucket.Name),
1068                         Prefix:     stringOrNil(trial.prefix),
1069                         Delimiter:  stringOrNil(trial.delimiter),
1070                         StartAfter: stringOrNil(trial.startAfter),
1071                         MaxKeys:    aws_aws.Int64(int64(trial.maxKeys)),
1072                 }
1073                 keySeen := map[string]bool{}
1074                 prefixSeen := map[string]bool{}
1075                 for {
1076                         result, err := client.ListObjectsV2WithContext(ctx, &params)
1077                         if !c.Check(err, check.IsNil) {
1078                                 break
1079                         }
1080                         c.Check(result.Name, check.DeepEquals, aws_aws.String(stage.collbucket.Name))
1081                         c.Check(result.Prefix, check.DeepEquals, aws_aws.String(trial.prefix))
1082                         c.Check(result.Delimiter, check.DeepEquals, aws_aws.String(trial.delimiter))
1083                         // The following two fields are expected to be
1084                         // nil (i.e., no tag in XML response) rather
1085                         // than "" when the corresponding request
1086                         // field was empty or nil.
1087                         c.Check(result.StartAfter, check.DeepEquals, stringOrNil(trial.startAfter))
1088                         c.Check(result.ContinuationToken, check.DeepEquals, params.ContinuationToken)
1089
1090                         if trial.maxKeys > 0 {
1091                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(trial.maxKeys)))
1092                                 c.Check(len(result.Contents)+len(result.CommonPrefixes) <= trial.maxKeys, check.Equals, true)
1093                         } else {
1094                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(s3MaxKeys)))
1095                         }
1096
1097                         for _, ent := range result.Contents {
1098                                 c.Assert(ent.Key, check.NotNil)
1099                                 c.Check(*ent.Key > trial.startAfter, check.Equals, true)
1100                                 c.Check(keySeen[*ent.Key], check.Equals, false, check.Commentf("dup key %q", *ent.Key))
1101                                 keySeen[*ent.Key] = true
1102                         }
1103                         for _, ent := range result.CommonPrefixes {
1104                                 c.Assert(ent.Prefix, check.NotNil)
1105                                 c.Check(strings.HasSuffix(*ent.Prefix, trial.delimiter), check.Equals, true, check.Commentf("bad CommonPrefix %q", *ent.Prefix))
1106                                 if strings.HasPrefix(trial.startAfter, *ent.Prefix) {
1107                                         // If we asked for
1108                                         // startAfter=dir0/file10.txt,
1109                                         // we expect dir0/ to be
1110                                         // returned as a common prefix
1111                                 } else {
1112                                         c.Check(*ent.Prefix > trial.startAfter, check.Equals, true)
1113                                 }
1114                                 c.Check(prefixSeen[*ent.Prefix], check.Equals, false, check.Commentf("dup common prefix %q", *ent.Prefix))
1115                                 prefixSeen[*ent.Prefix] = true
1116                         }
1117                         if *result.IsTruncated && c.Check(result.NextContinuationToken, check.Not(check.Equals), "") {
1118                                 params.ContinuationToken = aws_aws.String(*result.NextContinuationToken)
1119                         } else {
1120                                 break
1121                         }
1122                 }
1123                 c.Check(keySeen, check.HasLen, trial.expectKeys)
1124                 c.Check(prefixSeen, check.HasLen, len(trial.expectCommonPrefixes))
1125                 if len(trial.expectCommonPrefixes) > 0 {
1126                         c.Check(prefixSeen, check.DeepEquals, trial.expectCommonPrefixes)
1127                 }
1128         }
1129 }
1130
1131 func (s *IntegrationSuite) TestS3ListObjectsV2EncodingTypeURL(c *check.C) {
1132         stage := s.s3setup(c)
1133         defer stage.teardown(c)
1134         dirs := 2
1135         filesPerDir := 40
1136         stage.writeBigDirs(c, dirs, filesPerDir)
1137
1138         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1139                 Region:           aws_aws.String("auto"),
1140                 Endpoint:         aws_aws.String(s.testServer.URL),
1141                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1142                 S3ForcePathStyle: aws_aws.Bool(true),
1143         }))
1144
1145         client := aws_s3.New(sess)
1146         ctx := context.Background()
1147
1148         result, err := client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1149                 Bucket:       aws_aws.String(stage.collbucket.Name),
1150                 Prefix:       aws_aws.String("dir0/"),
1151                 Delimiter:    aws_aws.String("/"),
1152                 StartAfter:   aws_aws.String("dir0/"),
1153                 EncodingType: aws_aws.String("url"),
1154         })
1155         c.Assert(err, check.IsNil)
1156         c.Check(*result.Prefix, check.Equals, "dir0%2F")
1157         c.Check(*result.Delimiter, check.Equals, "%2F")
1158         c.Check(*result.StartAfter, check.Equals, "dir0%2F")
1159         for _, ent := range result.Contents {
1160                 c.Check(*ent.Key, check.Matches, "dir0%2F.*")
1161         }
1162         result, err = client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1163                 Bucket:       aws_aws.String(stage.collbucket.Name),
1164                 Delimiter:    aws_aws.String("/"),
1165                 EncodingType: aws_aws.String("url"),
1166         })
1167         c.Assert(err, check.IsNil)
1168         c.Check(*result.Delimiter, check.Equals, "%2F")
1169         c.Check(result.CommonPrefixes, check.HasLen, dirs+1)
1170         for _, ent := range result.CommonPrefixes {
1171                 c.Check(*ent.Prefix, check.Matches, ".*%2F")
1172         }
1173 }
1174
1175 // TestS3cmd checks compatibility with the s3cmd command line tool, if
1176 // it's installed. As of Debian buster, s3cmd is only in backports, so
1177 // `arvados-server install` don't install it, and this test skips if
1178 // it's not installed.
1179 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
1180         if _, err := exec.LookPath("s3cmd"); err != nil {
1181                 c.Skip("s3cmd not found")
1182                 return
1183         }
1184
1185         stage := s.s3setup(c)
1186         defer stage.teardown(c)
1187
1188         cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
1189         buf, err := cmd.CombinedOutput()
1190         c.Check(err, check.IsNil)
1191         c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
1192
1193         // This tests whether s3cmd's path normalization agrees with
1194         // keep-web's signature verification wrt chars like "|"
1195         // (neither reserved nor unreserved) and "," (not normally
1196         // percent-encoded in a path).
1197         tmpfile := c.MkDir() + "/dstfile"
1198         cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar", tmpfile)
1199         buf, err = cmd.CombinedOutput()
1200         c.Check(err, check.NotNil)
1201         c.Check(string(buf), check.Matches, `(?ms).*NoSuchKey.*\n`)
1202 }
1203
1204 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
1205         stage := s.s3setup(c)
1206         defer stage.teardown(c)
1207
1208         hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
1209         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
1210         c.Check(body, check.Equals, "⛵\n")
1211 }