services/keep-web/s3_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/rand"
  11         "crypto/sha256"
  12         "fmt"
  13         "io/ioutil"
  14         "mime"
  15         "net/http"
  16         "net/http/httptest"
  17         "net/url"
  18         "os"
  19         "os/exec"
  20         "strings"
  21         "sync"
  22         "time"
  23
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/keepclient"
  28         "github.com/AdRoll/goamz/aws"
  29         "github.com/AdRoll/goamz/s3"
  30         aws_aws "github.com/aws/aws-sdk-go/aws"
  31         aws_credentials "github.com/aws/aws-sdk-go/aws/credentials"
  32         aws_session "github.com/aws/aws-sdk-go/aws/session"
  33         aws_s3 "github.com/aws/aws-sdk-go/service/s3"
  34         check "gopkg.in/check.v1"
  35 )
  36
  37 type s3stage struct {
  38         arv        *arvados.Client
  39         ac         *arvadosclient.ArvadosClient
  40         kc         *keepclient.KeepClient
  41         proj       arvados.Group
  42         projbucket *s3.Bucket
  43         subproj    arvados.Group
  44         coll       arvados.Collection
  45         collbucket *s3.Bucket
  46 }
  47
  48 func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
  49         var proj, subproj arvados.Group
  50         var coll arvados.Collection
  51         arv := arvados.NewClientFromEnv()
  52         arv.AuthToken = arvadostest.ActiveToken
  53         err := arv.RequestAndDecode(&proj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
  54                 "group": map[string]interface{}{
  55                         "group_class": "project",
  56                         "name":        "keep-web s3 test",
  57                         "properties": map[string]interface{}{
  58                                 "project-properties-key": "project properties value",
  59                         },
  60                 },
  61                 "ensure_unique_name": true,
  62         })
  63         c.Assert(err, check.IsNil)
  64         err = arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
  65                 "group": map[string]interface{}{
  66                         "owner_uuid":  proj.UUID,
  67                         "group_class": "project",
  68                         "name":        "keep-web s3 test subproject",
  69                         "properties": map[string]interface{}{
  70                                 "subproject_properties_key": "subproject properties value",
  71                                 "invalid header key":        "this value will not be returned because key contains spaces",
  72                         },
  73                 },
  74         })
  75         c.Assert(err, check.IsNil)
  76         err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
  77                 "owner_uuid":    proj.UUID,
  78                 "name":          "keep-web s3 test collection",
  79                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
  80                 "properties": map[string]interface{}{
  81                         "string":   "string value",
  82                         "array":    []string{"element1", "element2"},
  83                         "object":   map[string]interface{}{"key": map[string]interface{}{"key2": "value⛵"}},
  84                         "nonascii": "⛵",
  85                 },
  86         }})
  87         c.Assert(err, check.IsNil)
  88         ac, err := arvadosclient.New(arv)
  89         c.Assert(err, check.IsNil)
  90         kc, err := keepclient.MakeKeepClient(ac)
  91         c.Assert(err, check.IsNil)
  92         fs, err := coll.FileSystem(arv, kc)
  93         c.Assert(err, check.IsNil)
  94         f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
  95         c.Assert(err, check.IsNil)
  96         _, err = f.Write([]byte("⛵\n"))
  97         c.Assert(err, check.IsNil)
  98         err = f.Close()
  99         c.Assert(err, check.IsNil)
 100         err = fs.Sync()
 101         c.Assert(err, check.IsNil)
 102         err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
 103         c.Assert(err, check.IsNil)
 104
 105         auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
 106         region := aws.Region{
 107                 Name:       "zzzzz",
 108                 S3Endpoint: s.testServer.URL,
 109         }
 110         client := s3.New(*auth, region)
 111         client.Signature = aws.V4Signature
 112         return s3stage{
 113                 arv:  arv,
 114                 ac:   ac,
 115                 kc:   kc,
 116                 proj: proj,
 117                 projbucket: &s3.Bucket{
 118                         S3:   client,
 119                         Name: proj.UUID,
 120                 },
 121                 subproj: subproj,
 122                 coll:    coll,
 123                 collbucket: &s3.Bucket{
 124                         S3:   client,
 125                         Name: coll.UUID,
 126                 },
 127         }
 128 }
 129
 130 func (stage s3stage) teardown(c *check.C) {
 131         if stage.coll.UUID != "" {
 132                 err := stage.arv.RequestAndDecode(&stage.coll, "DELETE", "arvados/v1/collections/"+stage.coll.UUID, nil, nil)
 133                 c.Check(err, check.IsNil)
 134         }
 135         if stage.proj.UUID != "" {
 136                 err := stage.arv.RequestAndDecode(&stage.proj, "DELETE", "arvados/v1/groups/"+stage.proj.UUID, nil, nil)
 137                 c.Check(err, check.IsNil)
 138         }
 139 }
 140
 141 func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
 142         stage := s.s3setup(c)
 143         defer stage.teardown(c)
 144
 145         bucket := stage.collbucket
 146         for _, trial := range []struct {
 147                 success   bool
 148                 signature int
 149                 accesskey string
 150                 secretkey string
 151         }{
 152                 {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
 153                 {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
 154                 {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
 155                 {false, aws.V2Signature, "none", "none"},
 156                 {false, aws.V2Signature, "none", arvadostest.ActiveToken},
 157
 158                 {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
 159                 {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
 160                 {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
 161                 {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
 162                 {false, aws.V4Signature, arvadostest.ActiveToken, ""},
 163                 {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
 164                 {false, aws.V4Signature, "none", arvadostest.ActiveToken},
 165                 {false, aws.V4Signature, "none", "none"},
 166         } {
 167                 c.Logf("%#v", trial)
 168                 bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
 169                 bucket.S3.Signature = trial.signature
 170                 _, err := bucket.GetReader("emptyfile")
 171                 if trial.success {
 172                         c.Check(err, check.IsNil)
 173                 } else {
 174                         c.Check(err, check.NotNil)
 175                 }
 176         }
 177 }
 178
 179 func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
 180         stage := s.s3setup(c)
 181         defer stage.teardown(c)
 182
 183         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 184                 c.Logf("bucket %s", bucket.Name)
 185                 exists, err := bucket.Exists("")
 186                 c.Check(err, check.IsNil)
 187                 c.Check(exists, check.Equals, true)
 188         }
 189 }
 190
 191 func (s *IntegrationSuite) TestS3CollectionGetObject(c *check.C) {
 192         stage := s.s3setup(c)
 193         defer stage.teardown(c)
 194         s.testS3GetObject(c, stage.collbucket, "")
 195 }
 196 func (s *IntegrationSuite) TestS3ProjectGetObject(c *check.C) {
 197         stage := s.s3setup(c)
 198         defer stage.teardown(c)
 199         s.testS3GetObject(c, stage.projbucket, stage.coll.Name+"/")
 200 }
 201 func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix string) {
 202         rdr, err := bucket.GetReader(prefix + "emptyfile")
 203         c.Assert(err, check.IsNil)
 204         buf, err := ioutil.ReadAll(rdr)
 205         c.Check(err, check.IsNil)
 206         c.Check(len(buf), check.Equals, 0)
 207         err = rdr.Close()
 208         c.Check(err, check.IsNil)
 209
 210         // GetObject
 211         rdr, err = bucket.GetReader(prefix + "missingfile")
 212         c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 213         c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 214         c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
 215
 216         // HeadObject
 217         exists, err := bucket.Exists(prefix + "missingfile")
 218         c.Check(err, check.IsNil)
 219         c.Check(exists, check.Equals, false)
 220
 221         // GetObject
 222         rdr, err = bucket.GetReader(prefix + "sailboat.txt")
 223         c.Assert(err, check.IsNil)
 224         buf, err = ioutil.ReadAll(rdr)
 225         c.Check(err, check.IsNil)
 226         c.Check(buf, check.DeepEquals, []byte("⛵\n"))
 227         err = rdr.Close()
 228         c.Check(err, check.IsNil)
 229
 230         // HeadObject
 231         resp, err := bucket.Head(prefix+"sailboat.txt", nil)
 232         c.Check(err, check.IsNil)
 233         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 234         c.Check(resp.ContentLength, check.Equals, int64(4))
 235
 236         // HeadObject with superfluous leading slashes
 237         exists, err = bucket.Exists(prefix + "//sailboat.txt")
 238         c.Check(err, check.IsNil)
 239         c.Check(exists, check.Equals, true)
 240 }
 241
 242 func (s *IntegrationSuite) checkMetaEquals(c *check.C, hdr http.Header, expect map[string]string) {
 243         got := map[string]string{}
 244         for hk, hv := range hdr {
 245                 if k := strings.TrimPrefix(hk, "X-Amz-Meta-"); k != hk && len(hv) == 1 {
 246                         got[k] = hv[0]
 247                 }
 248         }
 249         c.Check(got, check.DeepEquals, expect)
 250 }
 251
 252 func (s *IntegrationSuite) TestS3PropertiesAsMetadata(c *check.C) {
 253         stage := s.s3setup(c)
 254         defer stage.teardown(c)
 255
 256         expectCollectionTags := map[string]string{
 257                 "String":   "string value",
 258                 "Array":    `["element1","element2"]`,
 259                 "Object":   mime.BEncoding.Encode("UTF-8", `{"key":{"key2":"value⛵"}}`),
 260                 "Nonascii": "=?UTF-8?b?4pu1?=",
 261         }
 262         expectSubprojectTags := map[string]string{
 263                 "Subproject_properties_key": "subproject properties value",
 264         }
 265         expectProjectTags := map[string]string{
 266                 "Project-Properties-Key": "project properties value",
 267         }
 268
 269         c.Log("HEAD object with metadata from collection")
 270         resp, err := stage.collbucket.Head("sailboat.txt", nil)
 271         c.Assert(err, check.IsNil)
 272         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 273
 274         c.Log("GET object with metadata from collection")
 275         rdr, hdr, err := stage.collbucket.GetReaderWithHeaders("sailboat.txt")
 276         c.Assert(err, check.IsNil)
 277         content, err := ioutil.ReadAll(rdr)
 278         c.Check(err, check.IsNil)
 279         rdr.Close()
 280         c.Check(content, check.HasLen, 4)
 281         s.checkMetaEquals(c, hdr, expectCollectionTags)
 282
 283         c.Log("HEAD bucket with metadata from collection")
 284         resp, err = stage.collbucket.Head("/", nil)
 285         c.Assert(err, check.IsNil)
 286         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 287
 288         c.Log("HEAD directory placeholder with metadata from collection")
 289         resp, err = stage.projbucket.Head("keep-web s3 test collection/", nil)
 290         c.Assert(err, check.IsNil)
 291         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 292
 293         c.Log("HEAD file with metadata from collection")
 294         resp, err = stage.projbucket.Head("keep-web s3 test collection/sailboat.txt", nil)
 295         c.Assert(err, check.IsNil)
 296         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 297
 298         c.Log("HEAD directory placeholder with metadata from subproject")
 299         resp, err = stage.projbucket.Head("keep-web s3 test subproject/", nil)
 300         c.Assert(err, check.IsNil)
 301         s.checkMetaEquals(c, resp.Header, expectSubprojectTags)
 302
 303         c.Log("HEAD bucket with metadata from project")
 304         resp, err = stage.projbucket.Head("/", nil)
 305         c.Assert(err, check.IsNil)
 306         s.checkMetaEquals(c, resp.Header, expectProjectTags)
 307 }
 308
 309 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
 310         stage := s.s3setup(c)
 311         defer stage.teardown(c)
 312         s.testS3PutObjectSuccess(c, stage.collbucket, "")
 313 }
 314 func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) {
 315         stage := s.s3setup(c)
 316         defer stage.teardown(c)
 317         s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/")
 318 }
 319 func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string) {
 320         for _, trial := range []struct {
 321                 path        string
 322                 size        int
 323                 contentType string
 324         }{
 325                 {
 326                         path:        "newfile",
 327                         size:        128000000,
 328                         contentType: "application/octet-stream",
 329                 }, {
 330                         path:        "newdir/newfile",
 331                         size:        1 << 26,
 332                         contentType: "application/octet-stream",
 333                 }, {
 334                         path:        "/aaa",
 335                         size:        2,
 336                         contentType: "application/octet-stream",
 337                 }, {
 338                         path:        "//bbb",
 339                         size:        2,
 340                         contentType: "application/octet-stream",
 341                 }, {
 342                         path:        "ccc//",
 343                         size:        0,
 344                         contentType: "application/x-directory",
 345                 }, {
 346                         path:        "newdir1/newdir2/newfile",
 347                         size:        0,
 348                         contentType: "application/octet-stream",
 349                 }, {
 350                         path:        "newdir1/newdir2/newdir3/",
 351                         size:        0,
 352                         contentType: "application/x-directory",
 353                 },
 354         } {
 355                 c.Logf("=== %v", trial)
 356
 357                 objname := prefix + trial.path
 358
 359                 _, err := bucket.GetReader(objname)
 360                 if !c.Check(err, check.NotNil) {
 361                         continue
 362                 }
 363                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 364                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 365                 if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
 366                         continue
 367                 }
 368
 369                 buf := make([]byte, trial.size)
 370                 rand.Read(buf)
 371
 372                 err = bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 373                 c.Check(err, check.IsNil)
 374
 375                 rdr, err := bucket.GetReader(objname)
 376                 if strings.HasSuffix(trial.path, "/") && !s.handler.Cluster.Collections.S3FolderObjects {
 377                         c.Check(err, check.NotNil)
 378                         continue
 379                 } else if !c.Check(err, check.IsNil) {
 380                         continue
 381                 }
 382                 buf2, err := ioutil.ReadAll(rdr)
 383                 c.Check(err, check.IsNil)
 384                 c.Check(buf2, check.HasLen, len(buf))
 385                 c.Check(bytes.Equal(buf, buf2), check.Equals, true)
 386         }
 387 }
 388
 389 func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
 390         stage := s.s3setup(c)
 391         defer stage.teardown(c)
 392         bucket := stage.projbucket
 393
 394         for _, trial := range []struct {
 395                 path        string
 396                 size        int
 397                 contentType string
 398         }{
 399                 {
 400                         path:        "newfile",
 401                         size:        1234,
 402                         contentType: "application/octet-stream",
 403                 }, {
 404                         path:        "newdir/newfile",
 405                         size:        1234,
 406                         contentType: "application/octet-stream",
 407                 }, {
 408                         path:        "newdir2/",
 409                         size:        0,
 410                         contentType: "application/x-directory",
 411                 },
 412         } {
 413                 c.Logf("=== %v", trial)
 414
 415                 _, err := bucket.GetReader(trial.path)
 416                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 417                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 418                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 419
 420                 buf := make([]byte, trial.size)
 421                 rand.Read(buf)
 422
 423                 err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 424                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
 425                 c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
 426                 c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid (argument|operation)`)
 427
 428                 _, err = bucket.GetReader(trial.path)
 429                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 430                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 431                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 432         }
 433 }
 434
 435 func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) {
 436         stage := s.s3setup(c)
 437         defer stage.teardown(c)
 438         s.testS3DeleteObject(c, stage.collbucket, "")
 439 }
 440 func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) {
 441         stage := s.s3setup(c)
 442         defer stage.teardown(c)
 443         s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
 444 }
 445 func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
 446         s.handler.Cluster.Collections.S3FolderObjects = true
 447         for _, trial := range []struct {
 448                 path string
 449         }{
 450                 {"/"},
 451                 {"nonexistentfile"},
 452                 {"emptyfile"},
 453                 {"sailboat.txt"},
 454                 {"sailboat.txt/"},
 455                 {"emptydir"},
 456                 {"emptydir/"},
 457         } {
 458                 objname := prefix + trial.path
 459                 comment := check.Commentf("objname %q", objname)
 460
 461                 err := bucket.Del(objname)
 462                 if trial.path == "/" {
 463                         c.Check(err, check.NotNil)
 464                         continue
 465                 }
 466                 c.Check(err, check.IsNil, comment)
 467                 _, err = bucket.GetReader(objname)
 468                 c.Check(err, check.NotNil, comment)
 469         }
 470 }
 471
 472 func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) {
 473         stage := s.s3setup(c)
 474         defer stage.teardown(c)
 475         s.testS3PutObjectFailure(c, stage.collbucket, "")
 476 }
 477 func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
 478         stage := s.s3setup(c)
 479         defer stage.teardown(c)
 480         s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
 481 }
 482 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
 483         s.handler.Cluster.Collections.S3FolderObjects = false
 484
 485         var wg sync.WaitGroup
 486         for _, trial := range []struct {
 487                 path string
 488         }{
 489                 {
 490                         path: "emptyfile/newname", // emptyfile exists, see s3setup()
 491                 }, {
 492                         path: "emptyfile/", // emptyfile exists, see s3setup()
 493                 }, {
 494                         path: "emptydir", // dir already exists, see s3setup()
 495                 }, {
 496                         path: "emptydir/",
 497                 }, {
 498                         path: "emptydir//",
 499                 }, {
 500                         path: "newdir/",
 501                 }, {
 502                         path: "newdir//",
 503                 }, {
 504                         path: "/",
 505                 }, {
 506                         path: "//",
 507                 }, {
 508                         path: "",
 509                 },
 510         } {
 511                 trial := trial
 512                 wg.Add(1)
 513                 go func() {
 514                         defer wg.Done()
 515                         c.Logf("=== %v", trial)
 516
 517                         objname := prefix + trial.path
 518
 519                         buf := make([]byte, 1234)
 520                         rand.Read(buf)
 521
 522                         err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
 523                         if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
 524                                 return
 525                         }
 526
 527                         if objname != "" && objname != "/" {
 528                                 _, err = bucket.GetReader(objname)
 529                                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 530                                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 531                                 c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
 532                         }
 533                 }()
 534         }
 535         wg.Wait()
 536 }
 537
 538 func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
 539         fs, err := stage.coll.FileSystem(stage.arv, stage.kc)
 540         c.Assert(err, check.IsNil)
 541         for d := 0; d < dirs; d++ {
 542                 dir := fmt.Sprintf("dir%d", d)
 543                 c.Assert(fs.Mkdir(dir, 0755), check.IsNil)
 544                 for i := 0; i < filesPerDir; i++ {
 545                         f, err := fs.OpenFile(fmt.Sprintf("%s/file%d.txt", dir, i), os.O_CREATE|os.O_WRONLY, 0644)
 546                         c.Assert(err, check.IsNil)
 547                         c.Assert(f.Close(), check.IsNil)
 548                 }
 549         }
 550         c.Assert(fs.Sync(), check.IsNil)
 551 }
 552
 553 func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
 554         scope := "20200202/zzzzz/service/aws4_request"
 555         signedHeaders := "date"
 556         req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
 557         stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
 558         c.Assert(err, check.IsNil)
 559         sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
 560         c.Assert(err, check.IsNil)
 561         req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
 562 }
 563
 564 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
 565         stage := s.s3setup(c)
 566         defer stage.teardown(c)
 567         for _, trial := range []struct {
 568                 url            string
 569                 method         string
 570                 body           string
 571                 responseCode   int
 572                 responseRegexp []string
 573         }{
 574                 {
 575                         url:            "https://" + stage.collbucket.Name + ".example.com/",
 576                         method:         "GET",
 577                         responseCode:   http.StatusOK,
 578                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 579                 },
 580                 {
 581                         url:            "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
 582                         method:         "GET",
 583                         responseCode:   http.StatusOK,
 584                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 585                 },
 586                 {
 587                         url:            "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
 588                         method:         "GET",
 589                         responseCode:   http.StatusOK,
 590                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 591                 },
 592                 {
 593                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
 594                         method:         "GET",
 595                         responseCode:   http.StatusOK,
 596                         responseRegexp: []string{`⛵\n`},
 597                 },
 598                 {
 599                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 600                         method:       "PUT",
 601                         body:         "boop",
 602                         responseCode: http.StatusOK,
 603                 },
 604                 {
 605                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 606                         method:         "GET",
 607                         responseCode:   http.StatusOK,
 608                         responseRegexp: []string{`boop`},
 609                 },
 610                 {
 611                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 612                         method:       "GET",
 613                         responseCode: http.StatusNotFound,
 614                 },
 615                 {
 616                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 617                         method:       "PUT",
 618                         body:         "boop",
 619                         responseCode: http.StatusOK,
 620                 },
 621                 {
 622                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 623                         method:         "GET",
 624                         responseCode:   http.StatusOK,
 625                         responseRegexp: []string{`boop`},
 626                 },
 627         } {
 628                 url, err := url.Parse(trial.url)
 629                 c.Assert(err, check.IsNil)
 630                 req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
 631                 c.Assert(err, check.IsNil)
 632                 s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
 633                 rr := httptest.NewRecorder()
 634                 s.handler.ServeHTTP(rr, req)
 635                 resp := rr.Result()
 636                 c.Check(resp.StatusCode, check.Equals, trial.responseCode)
 637                 body, err := ioutil.ReadAll(resp.Body)
 638                 c.Assert(err, check.IsNil)
 639                 for _, re := range trial.responseRegexp {
 640                         c.Check(string(body), check.Matches, re)
 641                 }
 642         }
 643 }
 644
 645 func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
 646         stage := s.s3setup(c)
 647         defer stage.teardown(c)
 648         for _, trial := range []struct {
 649                 rawPath        string
 650                 normalizedPath string
 651         }{
 652                 {"/foo", "/foo"},                           // boring case
 653                 {"/foo%5fbar", "/foo_bar"},                 // _ must not be escaped
 654                 {"/foo%2fbar", "/foo/bar"},                 // / must not be escaped
 655                 {"/(foo)/[];,", "/%28foo%29/%5B%5D%3B%2C"}, // ()[];, must be escaped
 656                 {"/foo%5bbar", "/foo%5Bbar"},               // %XX must be uppercase
 657                 {"//foo///.bar", "/foo/.bar"},              // "//" and "///" must be squashed to "/"
 658         } {
 659                 c.Logf("trial %q", trial)
 660
 661                 date := time.Now().UTC().Format("20060102T150405Z")
 662                 scope := "20200202/zzzzz/S3/aws4_request"
 663                 canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
 664                 c.Logf("canonicalRequest %q", canonicalRequest)
 665                 expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
 666                 c.Logf("expected stringToSign %q", expect)
 667
 668                 req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
 669                 req.Header.Set("X-Amz-Date", date)
 670                 req.Host = "host.example.com"
 671                 c.Assert(err, check.IsNil)
 672
 673                 obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
 674                 if !c.Check(err, check.IsNil) {
 675                         continue
 676                 }
 677                 c.Check(obtained, check.Equals, expect)
 678         }
 679 }
 680
 681 func (s *IntegrationSuite) TestS3GetBucketLocation(c *check.C) {
 682         stage := s.s3setup(c)
 683         defer stage.teardown(c)
 684         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 685                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 686                 c.Check(err, check.IsNil)
 687                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 688                 req.URL.RawQuery = "location"
 689                 resp, err := http.DefaultClient.Do(req)
 690                 c.Assert(err, check.IsNil)
 691                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 692                 buf, err := ioutil.ReadAll(resp.Body)
 693                 c.Assert(err, check.IsNil)
 694                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<LocationConstraint><LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">zzzzz</LocationConstraint></LocationConstraint>\n")
 695         }
 696 }
 697
 698 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
 699         stage := s.s3setup(c)
 700         defer stage.teardown(c)
 701         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 702                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 703                 c.Check(err, check.IsNil)
 704                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 705                 req.URL.RawQuery = "versioning"
 706                 resp, err := http.DefaultClient.Do(req)
 707                 c.Assert(err, check.IsNil)
 708                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 709                 buf, err := ioutil.ReadAll(resp.Body)
 710                 c.Assert(err, check.IsNil)
 711                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>\n")
 712         }
 713 }
 714
 715 func (s *IntegrationSuite) TestS3UnsupportedAPIs(c *check.C) {
 716         stage := s.s3setup(c)
 717         defer stage.teardown(c)
 718         for _, trial := range []struct {
 719                 method   string
 720                 path     string
 721                 rawquery string
 722         }{
 723                 {"GET", "/", "acl&versionId=1234"},    // GetBucketAcl
 724                 {"GET", "/foo", "acl&versionId=1234"}, // GetObjectAcl
 725                 {"PUT", "/", "acl"},                   // PutBucketAcl
 726                 {"PUT", "/foo", "acl"},                // PutObjectAcl
 727                 {"DELETE", "/", "tagging"},            // DeleteBucketTagging
 728                 {"DELETE", "/foo", "tagging"},         // DeleteObjectTagging
 729         } {
 730                 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 731                         c.Logf("trial %v bucket %v", trial, bucket)
 732                         req, err := http.NewRequest(trial.method, bucket.URL(trial.path), nil)
 733                         c.Check(err, check.IsNil)
 734                         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 735                         req.URL.RawQuery = trial.rawquery
 736                         resp, err := http.DefaultClient.Do(req)
 737                         c.Assert(err, check.IsNil)
 738                         c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 739                         buf, err := ioutil.ReadAll(resp.Body)
 740                         c.Assert(err, check.IsNil)
 741                         c.Check(string(buf), check.Matches, "(?ms).*InvalidRequest.*API not supported.*")
 742                 }
 743         }
 744 }
 745
 746 // If there are no CommonPrefixes entries, the CommonPrefixes XML tag
 747 // should not appear at all.
 748 func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
 749         stage := s.s3setup(c)
 750         defer stage.teardown(c)
 751
 752         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 753         c.Assert(err, check.IsNil)
 754         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 755         req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
 756         resp, err := http.DefaultClient.Do(req)
 757         c.Assert(err, check.IsNil)
 758         buf, err := ioutil.ReadAll(resp.Body)
 759         c.Assert(err, check.IsNil)
 760         c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
 761 }
 762
 763 // If there is no delimiter in the request, or the results are not
 764 // truncated, the NextMarker XML tag should not appear in the response
 765 // body.
 766 func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
 767         stage := s.s3setup(c)
 768         defer stage.teardown(c)
 769
 770         for _, query := range []string{"prefix=e&delimiter=/", ""} {
 771                 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 772                 c.Assert(err, check.IsNil)
 773                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 774                 req.URL.RawQuery = query
 775                 resp, err := http.DefaultClient.Do(req)
 776                 c.Assert(err, check.IsNil)
 777                 buf, err := ioutil.ReadAll(resp.Body)
 778                 c.Assert(err, check.IsNil)
 779                 c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`)
 780         }
 781 }
 782
 783 // List response should include KeyCount field.
 784 func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
 785         stage := s.s3setup(c)
 786         defer stage.teardown(c)
 787
 788         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 789         c.Assert(err, check.IsNil)
 790         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 791         req.URL.RawQuery = "prefix=&delimiter=/"
 792         resp, err := http.DefaultClient.Do(req)
 793         c.Assert(err, check.IsNil)
 794         buf, err := ioutil.ReadAll(resp.Body)
 795         c.Assert(err, check.IsNil)
 796         c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
 797 }
 798
 799 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
 800         stage := s.s3setup(c)
 801         defer stage.teardown(c)
 802
 803         var markers int
 804         for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
 805                 dirs := 2
 806                 filesPerDir := 1001
 807                 stage.writeBigDirs(c, dirs, filesPerDir)
 808                 // Total # objects is:
 809                 //                 2 file entries from s3setup (emptyfile and sailboat.txt)
 810                 //                +1 fake "directory" marker from s3setup (emptydir) (if enabled)
 811                 //             +dirs fake "directory" marker from writeBigDirs (dir0/, dir1/) (if enabled)
 812                 // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
 813                 s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
 814                 s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
 815                 s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
 816         }
 817 }
 818 func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
 819         c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.handler.Cluster.Collections.S3FolderObjects)
 820         expectPageSize := pageSize
 821         if expectPageSize > 1000 {
 822                 expectPageSize = 1000
 823         }
 824         gotKeys := map[string]s3.Key{}
 825         nextMarker := ""
 826         pages := 0
 827         for {
 828                 resp, err := bucket.List(prefix, "", nextMarker, pageSize)
 829                 if !c.Check(err, check.IsNil) {
 830                         break
 831                 }
 832                 c.Check(len(resp.Contents) <= expectPageSize, check.Equals, true)
 833                 if pages++; !c.Check(pages <= (expectFiles/expectPageSize)+1, check.Equals, true) {
 834                         break
 835                 }
 836                 for _, key := range resp.Contents {
 837                         gotKeys[key.Key] = key
 838                         if strings.Contains(key.Key, "sailboat.txt") {
 839                                 c.Check(key.Size, check.Equals, int64(4))
 840                         }
 841                 }
 842                 if !resp.IsTruncated {
 843                         c.Check(resp.NextMarker, check.Equals, "")
 844                         break
 845                 }
 846                 if !c.Check(resp.NextMarker, check.Not(check.Equals), "") {
 847                         break
 848                 }
 849                 nextMarker = resp.NextMarker
 850         }
 851         c.Check(len(gotKeys), check.Equals, expectFiles)
 852 }
 853
 854 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
 855         for _, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
 856                 s.testS3CollectionListRollup(c)
 857         }
 858 }
 859
 860 func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
 861         stage := s.s3setup(c)
 862         defer stage.teardown(c)
 863
 864         dirs := 2
 865         filesPerDir := 500
 866         stage.writeBigDirs(c, dirs, filesPerDir)
 867         err := stage.collbucket.PutReader("dingbats", &bytes.Buffer{}, 0, "application/octet-stream", s3.Private, s3.Options{})
 868         c.Assert(err, check.IsNil)
 869         var allfiles []string
 870         for marker := ""; ; {
 871                 resp, err := stage.collbucket.List("", "", marker, 20000)
 872                 c.Check(err, check.IsNil)
 873                 for _, key := range resp.Contents {
 874                         if len(allfiles) == 0 || allfiles[len(allfiles)-1] != key.Key {
 875                                 allfiles = append(allfiles, key.Key)
 876                         }
 877                 }
 878                 marker = resp.NextMarker
 879                 if marker == "" {
 880                         break
 881                 }
 882         }
 883         markers := 0
 884         if s.handler.Cluster.Collections.S3FolderObjects {
 885                 markers = 1
 886         }
 887         c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
 888
 889         gotDirMarker := map[string]bool{}
 890         for _, name := range allfiles {
 891                 isDirMarker := strings.HasSuffix(name, "/")
 892                 if markers == 0 {
 893                         c.Check(isDirMarker, check.Equals, false, check.Commentf("name %q", name))
 894                 } else if isDirMarker {
 895                         gotDirMarker[name] = true
 896                 } else if i := strings.LastIndex(name, "/"); i >= 0 {
 897                         c.Check(gotDirMarker[name[:i+1]], check.Equals, true, check.Commentf("name %q", name))
 898                         gotDirMarker[name[:i+1]] = true // skip redundant complaints about this dir marker
 899                 }
 900         }
 901
 902         for _, trial := range []struct {
 903                 prefix    string
 904                 delimiter string
 905                 marker    string
 906         }{
 907                 {"", "", ""},
 908                 {"di", "/", ""},
 909                 {"di", "r", ""},
 910                 {"di", "n", ""},
 911                 {"dir0", "/", ""},
 912                 {"dir0/", "/", ""},
 913                 {"dir0/f", "/", ""},
 914                 {"dir0", "", ""},
 915                 {"dir0/", "", ""},
 916                 {"dir0/f", "", ""},
 917                 {"dir0", "/", "dir0/file14.txt"},       // no commonprefixes
 918                 {"", "", "dir0/file14.txt"},            // middle page, skip walking dir1
 919                 {"", "", "dir1/file14.txt"},            // middle page, skip walking dir0
 920                 {"", "", "dir1/file498.txt"},           // last page of results
 921                 {"dir1/file", "", "dir1/file498.txt"},  // last page of results, with prefix
 922                 {"dir1/file", "/", "dir1/file498.txt"}, // last page of results, with prefix + delimiter
 923                 {"dir1", "Z", "dir1/file498.txt"},      // delimiter "Z" never appears
 924                 {"dir2", "/", ""},                      // prefix "dir2" does not exist
 925                 {"", "/", ""},
 926         } {
 927                 c.Logf("\n\n=== trial %+v markers=%d", trial, markers)
 928
 929                 maxKeys := 20
 930                 resp, err := stage.collbucket.List(trial.prefix, trial.delimiter, trial.marker, maxKeys)
 931                 c.Check(err, check.IsNil)
 932                 if resp.IsTruncated && trial.delimiter == "" {
 933                         // goamz List method fills in the missing
 934                         // NextMarker field if resp.IsTruncated, so
 935                         // now we can't really tell whether it was
 936                         // sent by the server or by goamz. In cases
 937                         // where it should be empty but isn't, assume
 938                         // it's goamz's fault.
 939                         resp.NextMarker = ""
 940                 }
 941
 942                 var expectKeys []string
 943                 var expectPrefixes []string
 944                 var expectNextMarker string
 945                 var expectTruncated bool
 946                 for _, key := range allfiles {
 947                         full := len(expectKeys)+len(expectPrefixes) >= maxKeys
 948                         if !strings.HasPrefix(key, trial.prefix) || key < trial.marker {
 949                                 continue
 950                         } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
 951                                 prefix := key[:len(trial.prefix)+idx+1]
 952                                 if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
 953                                         // same prefix as previous key
 954                                 } else if full {
 955                                         expectNextMarker = key
 956                                         expectTruncated = true
 957                                 } else {
 958                                         expectPrefixes = append(expectPrefixes, prefix)
 959                                 }
 960                         } else if full {
 961                                 if trial.delimiter != "" {
 962                                         expectNextMarker = key
 963                                 }
 964                                 expectTruncated = true
 965                                 break
 966                         } else {
 967                                 expectKeys = append(expectKeys, key)
 968                         }
 969                 }
 970
 971                 var gotKeys []string
 972                 for _, key := range resp.Contents {
 973                         gotKeys = append(gotKeys, key.Key)
 974                 }
 975                 var gotPrefixes []string
 976                 for _, prefix := range resp.CommonPrefixes {
 977                         gotPrefixes = append(gotPrefixes, prefix)
 978                 }
 979                 commentf := check.Commentf("trial %+v markers=%d", trial, markers)
 980                 c.Check(gotKeys, check.DeepEquals, expectKeys, commentf)
 981                 c.Check(gotPrefixes, check.DeepEquals, expectPrefixes, commentf)
 982                 c.Check(resp.NextMarker, check.Equals, expectNextMarker, commentf)
 983                 c.Check(resp.IsTruncated, check.Equals, expectTruncated, commentf)
 984                 c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
 985         }
 986 }
 987
 988 func (s *IntegrationSuite) TestS3ListObjectsV2(c *check.C) {
 989         stage := s.s3setup(c)
 990         defer stage.teardown(c)
 991         dirs := 2
 992         filesPerDir := 40
 993         stage.writeBigDirs(c, dirs, filesPerDir)
 994
 995         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
 996                 Region:           aws_aws.String("auto"),
 997                 Endpoint:         aws_aws.String(s.testServer.URL),
 998                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
 999                 S3ForcePathStyle: aws_aws.Bool(true),
1000         }))
1001
1002         stringOrNil := func(s string) *string {
1003                 if s == "" {
1004                         return nil
1005                 } else {
1006                         return &s
1007                 }
1008         }
1009
1010         client := aws_s3.New(sess)
1011         ctx := context.Background()
1012
1013         for _, trial := range []struct {
1014                 prefix               string
1015                 delimiter            string
1016                 startAfter           string
1017                 maxKeys              int
1018                 expectKeys           int
1019                 expectCommonPrefixes map[string]bool
1020         }{
1021                 {
1022                         // Expect {filesPerDir plus the dir itself}
1023                         // for each dir, plus emptydir, emptyfile, and
1024                         // sailboat.txt.
1025                         expectKeys: (filesPerDir+1)*dirs + 3,
1026                 },
1027                 {
1028                         maxKeys:    15,
1029                         expectKeys: (filesPerDir+1)*dirs + 3,
1030                 },
1031                 {
1032                         startAfter: "dir0/z",
1033                         maxKeys:    15,
1034                         // Expect {filesPerDir plus the dir itself}
1035                         // for each dir except dir0, plus emptydir,
1036                         // emptyfile, and sailboat.txt.
1037                         expectKeys: (filesPerDir+1)*(dirs-1) + 3,
1038                 },
1039                 {
1040                         maxKeys:              1,
1041                         delimiter:            "/",
1042                         expectKeys:           2, // emptyfile, sailboat.txt
1043                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1044                 },
1045                 {
1046                         startAfter:           "dir0/z",
1047                         maxKeys:              15,
1048                         delimiter:            "/",
1049                         expectKeys:           2, // emptyfile, sailboat.txt
1050                         expectCommonPrefixes: map[string]bool{"dir1/": true, "emptydir/": true},
1051                 },
1052                 {
1053                         startAfter:           "dir0/file10.txt",
1054                         maxKeys:              15,
1055                         delimiter:            "/",
1056                         expectKeys:           2,
1057                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1058                 },
1059                 {
1060                         startAfter:           "dir0/file10.txt",
1061                         maxKeys:              15,
1062                         prefix:               "d",
1063                         delimiter:            "/",
1064                         expectKeys:           0,
1065                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true},
1066                 },
1067         } {
1068                 c.Logf("[trial %+v]", trial)
1069                 params := aws_s3.ListObjectsV2Input{
1070                         Bucket:     aws_aws.String(stage.collbucket.Name),
1071                         Prefix:     stringOrNil(trial.prefix),
1072                         Delimiter:  stringOrNil(trial.delimiter),
1073                         StartAfter: stringOrNil(trial.startAfter),
1074                         MaxKeys:    aws_aws.Int64(int64(trial.maxKeys)),
1075                 }
1076                 keySeen := map[string]bool{}
1077                 prefixSeen := map[string]bool{}
1078                 for {
1079                         result, err := client.ListObjectsV2WithContext(ctx, &params)
1080                         if !c.Check(err, check.IsNil) {
1081                                 break
1082                         }
1083                         c.Check(result.Name, check.DeepEquals, aws_aws.String(stage.collbucket.Name))
1084                         c.Check(result.Prefix, check.DeepEquals, aws_aws.String(trial.prefix))
1085                         c.Check(result.Delimiter, check.DeepEquals, aws_aws.String(trial.delimiter))
1086                         // The following two fields are expected to be
1087                         // nil (i.e., no tag in XML response) rather
1088                         // than "" when the corresponding request
1089                         // field was empty or nil.
1090                         c.Check(result.StartAfter, check.DeepEquals, stringOrNil(trial.startAfter))
1091                         c.Check(result.ContinuationToken, check.DeepEquals, params.ContinuationToken)
1092
1093                         if trial.maxKeys > 0 {
1094                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(trial.maxKeys)))
1095                                 c.Check(len(result.Contents)+len(result.CommonPrefixes) <= trial.maxKeys, check.Equals, true)
1096                         } else {
1097                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(s3MaxKeys)))
1098                         }
1099
1100                         for _, ent := range result.Contents {
1101                                 c.Assert(ent.Key, check.NotNil)
1102                                 c.Check(*ent.Key > trial.startAfter, check.Equals, true)
1103                                 c.Check(keySeen[*ent.Key], check.Equals, false, check.Commentf("dup key %q", *ent.Key))
1104                                 keySeen[*ent.Key] = true
1105                         }
1106                         for _, ent := range result.CommonPrefixes {
1107                                 c.Assert(ent.Prefix, check.NotNil)
1108                                 c.Check(strings.HasSuffix(*ent.Prefix, trial.delimiter), check.Equals, true, check.Commentf("bad CommonPrefix %q", *ent.Prefix))
1109                                 if strings.HasPrefix(trial.startAfter, *ent.Prefix) {
1110                                         // If we asked for
1111                                         // startAfter=dir0/file10.txt,
1112                                         // we expect dir0/ to be
1113                                         // returned as a common prefix
1114                                 } else {
1115                                         c.Check(*ent.Prefix > trial.startAfter, check.Equals, true)
1116                                 }
1117                                 c.Check(prefixSeen[*ent.Prefix], check.Equals, false, check.Commentf("dup common prefix %q", *ent.Prefix))
1118                                 prefixSeen[*ent.Prefix] = true
1119                         }
1120                         if *result.IsTruncated && c.Check(result.NextContinuationToken, check.Not(check.Equals), "") {
1121                                 params.ContinuationToken = aws_aws.String(*result.NextContinuationToken)
1122                         } else {
1123                                 break
1124                         }
1125                 }
1126                 c.Check(keySeen, check.HasLen, trial.expectKeys)
1127                 c.Check(prefixSeen, check.HasLen, len(trial.expectCommonPrefixes))
1128                 if len(trial.expectCommonPrefixes) > 0 {
1129                         c.Check(prefixSeen, check.DeepEquals, trial.expectCommonPrefixes)
1130                 }
1131         }
1132 }
1133
1134 func (s *IntegrationSuite) TestS3ListObjectsV2EncodingTypeURL(c *check.C) {
1135         stage := s.s3setup(c)
1136         defer stage.teardown(c)
1137         dirs := 2
1138         filesPerDir := 40
1139         stage.writeBigDirs(c, dirs, filesPerDir)
1140
1141         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1142                 Region:           aws_aws.String("auto"),
1143                 Endpoint:         aws_aws.String(s.testServer.URL),
1144                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1145                 S3ForcePathStyle: aws_aws.Bool(true),
1146         }))
1147
1148         client := aws_s3.New(sess)
1149         ctx := context.Background()
1150
1151         result, err := client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1152                 Bucket:       aws_aws.String(stage.collbucket.Name),
1153                 Prefix:       aws_aws.String("dir0/"),
1154                 Delimiter:    aws_aws.String("/"),
1155                 StartAfter:   aws_aws.String("dir0/"),
1156                 EncodingType: aws_aws.String("url"),
1157         })
1158         c.Assert(err, check.IsNil)
1159         c.Check(*result.Prefix, check.Equals, "dir0%2F")
1160         c.Check(*result.Delimiter, check.Equals, "%2F")
1161         c.Check(*result.StartAfter, check.Equals, "dir0%2F")
1162         for _, ent := range result.Contents {
1163                 c.Check(*ent.Key, check.Matches, "dir0%2F.*")
1164         }
1165         result, err = client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1166                 Bucket:       aws_aws.String(stage.collbucket.Name),
1167                 Delimiter:    aws_aws.String("/"),
1168                 EncodingType: aws_aws.String("url"),
1169         })
1170         c.Assert(err, check.IsNil)
1171         c.Check(*result.Delimiter, check.Equals, "%2F")
1172         c.Check(result.CommonPrefixes, check.HasLen, dirs+1)
1173         for _, ent := range result.CommonPrefixes {
1174                 c.Check(*ent.Prefix, check.Matches, ".*%2F")
1175         }
1176 }
1177
1178 // TestS3cmd checks compatibility with the s3cmd command line tool, if
1179 // it's installed. As of Debian buster, s3cmd is only in backports, so
1180 // `arvados-server install` don't install it, and this test skips if
1181 // it's not installed.
1182 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
1183         if _, err := exec.LookPath("s3cmd"); err != nil {
1184                 c.Skip("s3cmd not found")
1185                 return
1186         }
1187
1188         stage := s.s3setup(c)
1189         defer stage.teardown(c)
1190
1191         cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
1192         buf, err := cmd.CombinedOutput()
1193         c.Check(err, check.IsNil)
1194         c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
1195
1196         // This tests whether s3cmd's path normalization agrees with
1197         // keep-web's signature verification wrt chars like "|"
1198         // (neither reserved nor unreserved) and "," (not normally
1199         // percent-encoded in a path).
1200         tmpfile := c.MkDir() + "/dstfile"
1201         cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar", tmpfile)
1202         buf, err = cmd.CombinedOutput()
1203         c.Check(err, check.NotNil)
1204         c.Check(string(buf), check.Matches, `(?ms).*NoSuchKey.*\n`)
1205 }
1206
1207 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
1208         stage := s.s3setup(c)
1209         defer stage.teardown(c)
1210
1211         hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
1212         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
1213         c.Check(body, check.Equals, "⛵\n")
1214 }