services/keep-web/s3_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/rand"
  11         "crypto/sha256"
  12         "fmt"
  13         "io/ioutil"
  14         "mime"
  15         "net/http"
  16         "net/http/httptest"
  17         "net/url"
  18         "os"
  19         "os/exec"
  20         "sort"
  21         "strings"
  22         "sync"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/keepclient"
  29         "github.com/AdRoll/goamz/aws"
  30         "github.com/AdRoll/goamz/s3"
  31         aws_aws "github.com/aws/aws-sdk-go/aws"
  32         aws_credentials "github.com/aws/aws-sdk-go/aws/credentials"
  33         aws_session "github.com/aws/aws-sdk-go/aws/session"
  34         aws_s3 "github.com/aws/aws-sdk-go/service/s3"
  35         check "gopkg.in/check.v1"
  36 )
  37
  38 type CachedS3SecretSuite struct{}
  39
  40 var _ = check.Suite(&CachedS3SecretSuite{})
  41
  42 func (s *CachedS3SecretSuite) activeACA(expiresAt time.Time) *arvados.APIClientAuthorization {
  43         return &arvados.APIClientAuthorization{
  44                 UUID:      arvadostest.ActiveTokenUUID,
  45                 APIToken:  arvadostest.ActiveToken,
  46                 ExpiresAt: expiresAt,
  47         }
  48 }
  49
  50 func (s *CachedS3SecretSuite) TestNewCachedS3SecretExpiresBeforeTTL(c *check.C) {
  51         expected := time.Unix(1<<29, 0)
  52         aca := s.activeACA(expected)
  53         actual := newCachedS3Secret(aca, time.Unix(1<<30, 0))
  54         c.Check(actual.expiry, check.Equals, expected)
  55 }
  56
  57 func (s *CachedS3SecretSuite) TestNewCachedS3SecretExpiresAfterTTL(c *check.C) {
  58         expected := time.Unix(1<<29, 0)
  59         aca := s.activeACA(time.Unix(1<<30, 0))
  60         actual := newCachedS3Secret(aca, expected)
  61         c.Check(actual.expiry, check.Equals, expected)
  62 }
  63
  64 func (s *CachedS3SecretSuite) TestNewCachedS3SecretWithoutExpiry(c *check.C) {
  65         expected := time.Unix(1<<29, 0)
  66         aca := s.activeACA(time.Time{})
  67         actual := newCachedS3Secret(aca, expected)
  68         c.Check(actual.expiry, check.Equals, expected)
  69 }
  70
  71 func (s *CachedS3SecretSuite) cachedSecretWithExpiry(expiry time.Time) *cachedS3Secret {
  72         return &cachedS3Secret{
  73                 auth:   s.activeACA(expiry),
  74                 expiry: expiry,
  75         }
  76 }
  77
  78 func (s *CachedS3SecretSuite) TestIsValidAtEmpty(c *check.C) {
  79         cache := &cachedS3Secret{}
  80         c.Check(cache.isValidAt(time.Unix(0, 0)), check.Equals, false)
  81         c.Check(cache.isValidAt(time.Unix(1<<31, 0)), check.Equals, false)
  82 }
  83
  84 func (s *CachedS3SecretSuite) TestIsValidAtNoAuth(c *check.C) {
  85         cache := &cachedS3Secret{expiry: time.Unix(3, 0)}
  86         c.Check(cache.isValidAt(time.Unix(2, 0)), check.Equals, false)
  87         c.Check(cache.isValidAt(time.Unix(4, 0)), check.Equals, false)
  88 }
  89
  90 func (s *CachedS3SecretSuite) TestIsValidAtNoExpiry(c *check.C) {
  91         cache := &cachedS3Secret{auth: s.activeACA(time.Unix(3, 0))}
  92         c.Check(cache.isValidAt(time.Unix(2, 0)), check.Equals, false)
  93         c.Check(cache.isValidAt(time.Unix(4, 0)), check.Equals, false)
  94 }
  95
  96 func (s *CachedS3SecretSuite) TestIsValidAtTimeAfterExpiry(c *check.C) {
  97         expiry := time.Unix(10, 0)
  98         cache := s.cachedSecretWithExpiry(expiry)
  99         c.Check(cache.isValidAt(expiry), check.Equals, false)
 100         c.Check(cache.isValidAt(time.Unix(1<<25, 0)), check.Equals, false)
 101         c.Check(cache.isValidAt(time.Unix(1<<30, 0)), check.Equals, false)
 102 }
 103
 104 func (s *CachedS3SecretSuite) TestIsValidAtTimeBeforeExpiry(c *check.C) {
 105         cache := s.cachedSecretWithExpiry(time.Unix(1<<30, 0))
 106         c.Check(cache.isValidAt(time.Unix(1<<25, 0)), check.Equals, true)
 107         c.Check(cache.isValidAt(time.Unix(1<<27, 0)), check.Equals, true)
 108         c.Check(cache.isValidAt(time.Unix(1<<29, 0)), check.Equals, true)
 109 }
 110
 111 func (s *CachedS3SecretSuite) TestIsValidAtZeroTime(c *check.C) {
 112         cache := s.cachedSecretWithExpiry(time.Unix(10, 0))
 113         c.Check(cache.isValidAt(time.Time{}), check.Equals, false)
 114 }
 115
 116 type s3stage struct {
 117         arv        *arvados.Client
 118         ac         *arvadosclient.ArvadosClient
 119         kc         *keepclient.KeepClient
 120         proj       arvados.Group
 121         projbucket *s3.Bucket
 122         subproj    arvados.Group
 123         coll       arvados.Collection
 124         collbucket *s3.Bucket
 125 }
 126
 127 func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
 128         var proj, subproj arvados.Group
 129         var coll arvados.Collection
 130         arv := arvados.NewClientFromEnv()
 131         arv.AuthToken = arvadostest.ActiveToken
 132         err := arv.RequestAndDecode(&proj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
 133                 "group": map[string]interface{}{
 134                         "group_class": "project",
 135                         "name":        "keep-web s3 test",
 136                         "properties": map[string]interface{}{
 137                                 "project-properties-key": "project properties value",
 138                         },
 139                 },
 140                 "ensure_unique_name": true,
 141         })
 142         c.Assert(err, check.IsNil)
 143         err = arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
 144                 "group": map[string]interface{}{
 145                         "owner_uuid":  proj.UUID,
 146                         "group_class": "project",
 147                         "name":        "keep-web s3 test subproject",
 148                         "properties": map[string]interface{}{
 149                                 "subproject_properties_key": "subproject properties value",
 150                                 "invalid header key":        "this value will not be returned because key contains spaces",
 151                         },
 152                 },
 153         })
 154         c.Assert(err, check.IsNil)
 155         err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
 156                 "owner_uuid":    proj.UUID,
 157                 "name":          "keep-web s3 test collection",
 158                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
 159                 "properties": map[string]interface{}{
 160                         "string":   "string value",
 161                         "array":    []string{"element1", "element2"},
 162                         "object":   map[string]interface{}{"key": map[string]interface{}{"key2": "value⛵"}},
 163                         "nonascii": "⛵",
 164                         "newline":  "foo\r\nX-Bad: header",
 165                         // This key cannot be expressed as a MIME
 166                         // header key, so it will be silently skipped
 167                         // (see "Inject" in PropertiesAsMetadata test)
 168                         "a: a\r\nInject": "bogus",
 169                 },
 170         }})
 171         c.Assert(err, check.IsNil)
 172         ac, err := arvadosclient.New(arv)
 173         c.Assert(err, check.IsNil)
 174         kc, err := keepclient.MakeKeepClient(ac)
 175         c.Assert(err, check.IsNil)
 176         fs, err := coll.FileSystem(arv, kc)
 177         c.Assert(err, check.IsNil)
 178         f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
 179         c.Assert(err, check.IsNil)
 180         _, err = f.Write([]byte("⛵\n"))
 181         c.Assert(err, check.IsNil)
 182         err = f.Close()
 183         c.Assert(err, check.IsNil)
 184         err = fs.Sync()
 185         c.Assert(err, check.IsNil)
 186         err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
 187         c.Assert(err, check.IsNil)
 188
 189         auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
 190         region := aws.Region{
 191                 Name:       "zzzzz",
 192                 S3Endpoint: s.testServer.URL,
 193         }
 194         client := s3.New(*auth, region)
 195         client.Signature = aws.V4Signature
 196         return s3stage{
 197                 arv:  arv,
 198                 ac:   ac,
 199                 kc:   kc,
 200                 proj: proj,
 201                 projbucket: &s3.Bucket{
 202                         S3:   client,
 203                         Name: proj.UUID,
 204                 },
 205                 subproj: subproj,
 206                 coll:    coll,
 207                 collbucket: &s3.Bucket{
 208                         S3:   client,
 209                         Name: coll.UUID,
 210                 },
 211         }
 212 }
 213
 214 func (stage s3stage) teardown(c *check.C) {
 215         if stage.coll.UUID != "" {
 216                 err := stage.arv.RequestAndDecode(&stage.coll, "DELETE", "arvados/v1/collections/"+stage.coll.UUID, nil, nil)
 217                 c.Check(err, check.IsNil)
 218         }
 219         if stage.proj.UUID != "" {
 220                 err := stage.arv.RequestAndDecode(&stage.proj, "DELETE", "arvados/v1/groups/"+stage.proj.UUID, nil, nil)
 221                 c.Check(err, check.IsNil)
 222         }
 223 }
 224
 225 func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
 226         stage := s.s3setup(c)
 227         defer stage.teardown(c)
 228
 229         bucket := stage.collbucket
 230         for _, trial := range []struct {
 231                 success   bool
 232                 signature int
 233                 accesskey string
 234                 secretkey string
 235         }{
 236                 {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
 237                 {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
 238                 {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
 239                 {false, aws.V2Signature, "none", "none"},
 240                 {false, aws.V2Signature, "none", arvadostest.ActiveToken},
 241
 242                 {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
 243                 {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
 244                 {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
 245                 {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
 246                 {false, aws.V4Signature, arvadostest.ActiveToken, ""},
 247                 {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
 248                 {false, aws.V4Signature, "none", arvadostest.ActiveToken},
 249                 {false, aws.V4Signature, "none", "none"},
 250         } {
 251                 c.Logf("%#v", trial)
 252                 bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
 253                 bucket.S3.Signature = trial.signature
 254                 _, err := bucket.GetReader("emptyfile")
 255                 if trial.success {
 256                         c.Check(err, check.IsNil)
 257                 } else {
 258                         c.Check(err, check.NotNil)
 259                 }
 260         }
 261 }
 262
 263 func (s *IntegrationSuite) TestS3SecretCacheUpdates(c *check.C) {
 264         stage := s.s3setup(c)
 265         defer stage.teardown(c)
 266         reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
 267         c.Assert(err, check.IsNil)
 268
 269         for trialName, trialAuth := range map[string]string{
 270                 "v1 token":                    arvadostest.ActiveToken,
 271                 "token UUID":                  arvadostest.ActiveTokenUUID,
 272                 "v2 token query escaped":      url.QueryEscape(arvadostest.ActiveTokenV2),
 273                 "v2 token underscore escaped": strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1),
 274         } {
 275                 s.handler.s3SecretCache = nil
 276                 req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
 277                 if !c.Check(err, check.IsNil) {
 278                         continue
 279                 }
 280                 secret := trialAuth
 281                 if secret[5:12] == "-gj3su-" {
 282                         secret = arvadostest.ActiveToken
 283                 }
 284                 s.sign(c, req, trialAuth, secret)
 285                 rec := httptest.NewRecorder()
 286                 s.handler.ServeHTTP(rec, req)
 287                 if !c.Check(rec.Result().StatusCode, check.Equals, http.StatusOK,
 288                         check.Commentf("%s auth did not get 200 OK response: %v", trialName, req)) {
 289                         continue
 290                 }
 291
 292                 for name, key := range map[string]string{
 293                         "v1 token":   arvadostest.ActiveToken,
 294                         "token UUID": arvadostest.ActiveTokenUUID,
 295                         "v2 token":   arvadostest.ActiveTokenV2,
 296                 } {
 297                         actual, ok := s.handler.s3SecretCache[key]
 298                         if c.Check(ok, check.Equals, true, check.Commentf("%s not cached from %s", name, trialName)) {
 299                                 c.Check(actual.auth.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 300                         }
 301                 }
 302         }
 303 }
 304
 305 func (s *IntegrationSuite) TestS3SecretCacheUsed(c *check.C) {
 306         stage := s.s3setup(c)
 307         defer stage.teardown(c)
 308
 309         token := arvadostest.ActiveToken
 310         // Step 1: Make a request to get the active token in the cache.
 311         reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
 312         c.Assert(err, check.IsNil)
 313         req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
 314         s.sign(c, req, token, token)
 315         rec := httptest.NewRecorder()
 316         s.handler.ServeHTTP(rec, req)
 317         resp := rec.Result()
 318         c.Assert(resp.StatusCode, check.Equals, http.StatusOK,
 319                 check.Commentf("first request did not get 200 OK response"))
 320
 321         // Step 2: Remove some cache keys our request doesn't rely upon.
 322         c.Assert(s.handler.s3SecretCache[arvadostest.ActiveTokenUUID], check.NotNil)
 323         delete(s.handler.s3SecretCache, arvadostest.ActiveTokenUUID)
 324         c.Assert(s.handler.s3SecretCache[arvadostest.ActiveTokenV2], check.NotNil)
 325         delete(s.handler.s3SecretCache, arvadostest.ActiveTokenV2)
 326
 327         // Step 3: Repeat the original request.
 328         rec = httptest.NewRecorder()
 329         s.handler.ServeHTTP(rec, req)
 330         resp = rec.Result()
 331         c.Assert(resp.StatusCode, check.Equals, http.StatusOK,
 332                 check.Commentf("cached auth request did not get 200 OK response"))
 333
 334         // Step 4: Confirm the deleted cache keys were not re-added
 335         // (which would imply the authorization was re-requested and cached).
 336         c.Check(s.handler.s3SecretCache[arvadostest.ActiveTokenUUID], check.IsNil,
 337                 check.Commentf("token UUID re-added to cache after removal"))
 338         c.Check(s.handler.s3SecretCache[arvadostest.ActiveTokenV2], check.IsNil,
 339                 check.Commentf("v2 token re-added to cache after removal"))
 340 }
 341
 342 func (s *IntegrationSuite) TestS3SecretCacheCleanup(c *check.C) {
 343         stage := s.s3setup(c)
 344         defer stage.teardown(c)
 345         td := -2 * s3SecretCacheTidyInterval
 346         startTidied := time.Now().Add(td)
 347         s.handler.s3SecretCacheNextTidy = startTidied
 348         s.handler.s3SecretCache = make(map[string]*cachedS3Secret)
 349         s.handler.s3SecretCache["old"] = &cachedS3Secret{expiry: startTidied.Add(td)}
 350
 351         reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
 352         c.Assert(err, check.IsNil)
 353         req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
 354         token := arvadostest.ActiveToken
 355         s.sign(c, req, token, token)
 356         rec := httptest.NewRecorder()
 357         s.handler.ServeHTTP(rec, req)
 358
 359         c.Check(s.handler.s3SecretCache["old"], check.IsNil,
 360                 check.Commentf("expired token not removed from cache"))
 361         c.Check(s.handler.s3SecretCacheNextTidy.After(startTidied), check.Equals, true,
 362                 check.Commentf("s3SecretCacheNextTidy not updated"))
 363         c.Check(s.handler.s3SecretCache[token], check.NotNil,
 364                 check.Commentf("just-used token not found in cache"))
 365 }
 366
 367 func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
 368         stage := s.s3setup(c)
 369         defer stage.teardown(c)
 370
 371         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 372                 c.Logf("bucket %s", bucket.Name)
 373                 exists, err := bucket.Exists("")
 374                 c.Check(err, check.IsNil)
 375                 c.Check(exists, check.Equals, true)
 376         }
 377 }
 378
 379 func (s *IntegrationSuite) TestS3CollectionGetObject(c *check.C) {
 380         stage := s.s3setup(c)
 381         defer stage.teardown(c)
 382         s.testS3GetObject(c, stage.collbucket, "")
 383 }
 384 func (s *IntegrationSuite) TestS3ProjectGetObject(c *check.C) {
 385         stage := s.s3setup(c)
 386         defer stage.teardown(c)
 387         s.testS3GetObject(c, stage.projbucket, stage.coll.Name+"/")
 388 }
 389 func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix string) {
 390         rdr, err := bucket.GetReader(prefix + "emptyfile")
 391         c.Assert(err, check.IsNil)
 392         buf, err := ioutil.ReadAll(rdr)
 393         c.Check(err, check.IsNil)
 394         c.Check(len(buf), check.Equals, 0)
 395         err = rdr.Close()
 396         c.Check(err, check.IsNil)
 397
 398         // GetObject
 399         rdr, err = bucket.GetReader(prefix + "missingfile")
 400         c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 401         c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 402         c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
 403
 404         // HeadObject
 405         exists, err := bucket.Exists(prefix + "missingfile")
 406         c.Check(err, check.IsNil)
 407         c.Check(exists, check.Equals, false)
 408
 409         // GetObject
 410         rdr, err = bucket.GetReader(prefix + "sailboat.txt")
 411         c.Assert(err, check.IsNil)
 412         buf, err = ioutil.ReadAll(rdr)
 413         c.Check(err, check.IsNil)
 414         c.Check(buf, check.DeepEquals, []byte("⛵\n"))
 415         err = rdr.Close()
 416         c.Check(err, check.IsNil)
 417
 418         // HeadObject
 419         resp, err := bucket.Head(prefix+"sailboat.txt", nil)
 420         c.Check(err, check.IsNil)
 421         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 422         c.Check(resp.ContentLength, check.Equals, int64(4))
 423
 424         // HeadObject with superfluous leading slashes
 425         exists, err = bucket.Exists(prefix + "//sailboat.txt")
 426         c.Check(err, check.IsNil)
 427         c.Check(exists, check.Equals, false)
 428 }
 429
 430 func (s *IntegrationSuite) checkMetaEquals(c *check.C, hdr http.Header, expect map[string]string) {
 431         got := map[string]string{}
 432         for hk, hv := range hdr {
 433                 if k := strings.TrimPrefix(hk, "X-Amz-Meta-"); k != hk && len(hv) == 1 {
 434                         got[k] = hv[0]
 435                 }
 436         }
 437         c.Check(got, check.DeepEquals, expect)
 438 }
 439
 440 func (s *IntegrationSuite) TestS3PropertiesAsMetadata(c *check.C) {
 441         stage := s.s3setup(c)
 442         defer stage.teardown(c)
 443
 444         expectCollectionTags := map[string]string{
 445                 "String":   "string value",
 446                 "Array":    `["element1","element2"]`,
 447                 "Object":   mime.BEncoding.Encode("UTF-8", `{"key":{"key2":"value⛵"}}`),
 448                 "Nonascii": "=?UTF-8?b?4pu1?=",
 449                 "Newline":  mime.BEncoding.Encode("UTF-8", "foo\r\nX-Bad: header"),
 450         }
 451         expectSubprojectTags := map[string]string{
 452                 "Subproject_properties_key": "subproject properties value",
 453         }
 454         expectProjectTags := map[string]string{
 455                 "Project-Properties-Key": "project properties value",
 456         }
 457
 458         c.Log("HEAD object with metadata from collection")
 459         resp, err := stage.collbucket.Head("sailboat.txt", nil)
 460         c.Assert(err, check.IsNil)
 461         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 462
 463         c.Log("GET object with metadata from collection")
 464         rdr, hdr, err := stage.collbucket.GetReaderWithHeaders("sailboat.txt")
 465         c.Assert(err, check.IsNil)
 466         content, err := ioutil.ReadAll(rdr)
 467         c.Check(err, check.IsNil)
 468         rdr.Close()
 469         c.Check(content, check.HasLen, 4)
 470         s.checkMetaEquals(c, hdr, expectCollectionTags)
 471         c.Check(hdr["Inject"], check.IsNil)
 472
 473         c.Log("HEAD bucket with metadata from collection")
 474         resp, err = stage.collbucket.Head("/", nil)
 475         c.Assert(err, check.IsNil)
 476         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 477
 478         c.Log("HEAD directory placeholder with metadata from collection")
 479         resp, err = stage.projbucket.Head("keep-web s3 test collection/", nil)
 480         c.Assert(err, check.IsNil)
 481         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 482
 483         c.Log("HEAD file with metadata from collection")
 484         resp, err = stage.projbucket.Head("keep-web s3 test collection/sailboat.txt", nil)
 485         c.Assert(err, check.IsNil)
 486         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 487
 488         c.Log("HEAD directory placeholder with metadata from subproject")
 489         resp, err = stage.projbucket.Head("keep-web s3 test subproject/", nil)
 490         c.Assert(err, check.IsNil)
 491         s.checkMetaEquals(c, resp.Header, expectSubprojectTags)
 492
 493         c.Log("HEAD bucket with metadata from project")
 494         resp, err = stage.projbucket.Head("/", nil)
 495         c.Assert(err, check.IsNil)
 496         s.checkMetaEquals(c, resp.Header, expectProjectTags)
 497 }
 498
 499 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
 500         stage := s.s3setup(c)
 501         defer stage.teardown(c)
 502         s.testS3PutObjectSuccess(c, stage.collbucket, "", stage.coll.UUID)
 503 }
 504 func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) {
 505         stage := s.s3setup(c)
 506         defer stage.teardown(c)
 507         s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/", stage.coll.UUID)
 508 }
 509 func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string, collUUID string) {
 510         // We insert a delay between test cases to ensure we exercise
 511         // rollover of expired sessions.
 512         sleep := time.Second / 100
 513         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(sleep * 3)
 514
 515         for _, trial := range []struct {
 516                 path        string
 517                 size        int
 518                 contentType string
 519         }{
 520                 {
 521                         path:        "newfile",
 522                         size:        128000000,
 523                         contentType: "application/octet-stream",
 524                 }, {
 525                         path:        "newdir/newfile",
 526                         size:        1 << 26,
 527                         contentType: "application/octet-stream",
 528                 }, {
 529                         path:        "ccc/",
 530                         size:        0,
 531                         contentType: "application/x-directory",
 532                 }, {
 533                         path:        "newdir1/newdir2/newfile",
 534                         size:        0,
 535                         contentType: "application/octet-stream",
 536                 }, {
 537                         path:        "newdir1/newdir2/newdir3/",
 538                         size:        0,
 539                         contentType: "application/x-directory",
 540                 },
 541         } {
 542                 time.Sleep(sleep)
 543                 c.Logf("=== %v", trial)
 544
 545                 objname := prefix + trial.path
 546
 547                 _, err := bucket.GetReader(objname)
 548                 if !c.Check(err, check.NotNil) {
 549                         continue
 550                 }
 551                 c.Check(err.(*s3.Error).StatusCode, check.Equals, http.StatusNotFound)
 552                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 553                 if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
 554                         continue
 555                 }
 556
 557                 buf := make([]byte, trial.size)
 558                 rand.Read(buf)
 559
 560                 err = bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 561                 c.Check(err, check.IsNil)
 562
 563                 rdr, err := bucket.GetReader(objname)
 564                 if strings.HasSuffix(trial.path, "/") && !s.handler.Cluster.Collections.S3FolderObjects {
 565                         c.Check(err, check.NotNil)
 566                         continue
 567                 } else if !c.Check(err, check.IsNil) {
 568                         continue
 569                 }
 570                 buf2, err := ioutil.ReadAll(rdr)
 571                 c.Check(err, check.IsNil)
 572                 c.Check(buf2, check.HasLen, len(buf))
 573                 c.Check(bytes.Equal(buf, buf2), check.Equals, true)
 574
 575                 // Check that the change is immediately visible via
 576                 // (non-S3) webdav request.
 577                 _, resp := s.do("GET", "http://"+collUUID+".keep-web.example/"+trial.path, arvadostest.ActiveTokenV2, nil)
 578                 c.Check(resp.Code, check.Equals, http.StatusOK)
 579                 if !strings.HasSuffix(trial.path, "/") {
 580                         c.Check(resp.Body.Len(), check.Equals, trial.size)
 581                 }
 582         }
 583 }
 584
 585 func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
 586         stage := s.s3setup(c)
 587         defer stage.teardown(c)
 588         bucket := stage.projbucket
 589
 590         for _, trial := range []struct {
 591                 path        string
 592                 size        int
 593                 contentType string
 594         }{
 595                 {
 596                         path:        "newfile",
 597                         size:        1234,
 598                         contentType: "application/octet-stream",
 599                 }, {
 600                         path:        "newdir/newfile",
 601                         size:        1234,
 602                         contentType: "application/octet-stream",
 603                 }, {
 604                         path:        "newdir2/",
 605                         size:        0,
 606                         contentType: "application/x-directory",
 607                 },
 608         } {
 609                 c.Logf("=== %v", trial)
 610
 611                 _, err := bucket.GetReader(trial.path)
 612                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 613                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 614                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 615
 616                 buf := make([]byte, trial.size)
 617                 rand.Read(buf)
 618
 619                 err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 620                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
 621                 c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
 622                 c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid (argument|operation)`)
 623
 624                 _, err = bucket.GetReader(trial.path)
 625                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 626                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 627                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 628         }
 629 }
 630
 631 func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) {
 632         stage := s.s3setup(c)
 633         defer stage.teardown(c)
 634         s.testS3DeleteObject(c, stage.collbucket, "")
 635 }
 636 func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) {
 637         stage := s.s3setup(c)
 638         defer stage.teardown(c)
 639         s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
 640 }
 641 func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
 642         s.handler.Cluster.Collections.S3FolderObjects = true
 643         for _, trial := range []struct {
 644                 path string
 645         }{
 646                 {"/"},
 647                 {"nonexistentfile"},
 648                 {"emptyfile"},
 649                 {"sailboat.txt"},
 650                 {"sailboat.txt/"},
 651                 {"emptydir"},
 652                 {"emptydir/"},
 653         } {
 654                 objname := prefix + trial.path
 655                 comment := check.Commentf("objname %q", objname)
 656
 657                 err := bucket.Del(objname)
 658                 if trial.path == "/" {
 659                         c.Check(err, check.NotNil)
 660                         continue
 661                 }
 662                 c.Check(err, check.IsNil, comment)
 663                 _, err = bucket.GetReader(objname)
 664                 c.Check(err, check.NotNil, comment)
 665         }
 666 }
 667
 668 func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) {
 669         stage := s.s3setup(c)
 670         defer stage.teardown(c)
 671         s.testS3PutObjectFailure(c, stage.collbucket, "")
 672 }
 673 func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
 674         stage := s.s3setup(c)
 675         defer stage.teardown(c)
 676         s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
 677 }
 678 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
 679         s.handler.Cluster.Collections.S3FolderObjects = false
 680
 681         var wg sync.WaitGroup
 682         for _, trial := range []struct {
 683                 path string
 684         }{
 685                 {
 686                         path: "emptyfile/newname", // emptyfile exists, see s3setup()
 687                 }, {
 688                         path: "emptyfile/", // emptyfile exists, see s3setup()
 689                 }, {
 690                         path: "emptydir", // dir already exists, see s3setup()
 691                 }, {
 692                         path: "emptydir/",
 693                 }, {
 694                         path: "newdir/",
 695                 }, {
 696                         path: "",
 697                 },
 698         } {
 699                 trial := trial
 700                 wg.Add(1)
 701                 go func() {
 702                         defer wg.Done()
 703                         c.Logf("=== %v", trial)
 704
 705                         objname := prefix + trial.path
 706
 707                         buf := make([]byte, 1234)
 708                         rand.Read(buf)
 709
 710                         err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
 711                         if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
 712                                 return
 713                         }
 714
 715                         if objname != "" && objname != "/" {
 716                                 _, err = bucket.GetReader(objname)
 717                                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 718                                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 719                                 c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
 720                         }
 721                 }()
 722         }
 723         wg.Wait()
 724 }
 725
 726 func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
 727         fs, err := stage.coll.FileSystem(stage.arv, stage.kc)
 728         c.Assert(err, check.IsNil)
 729         for d := 0; d < dirs; d++ {
 730                 dir := fmt.Sprintf("dir%d", d)
 731                 c.Assert(fs.Mkdir(dir, 0755), check.IsNil)
 732                 for i := 0; i < filesPerDir; i++ {
 733                         f, err := fs.OpenFile(fmt.Sprintf("%s/file%d.txt", dir, i), os.O_CREATE|os.O_WRONLY, 0644)
 734                         c.Assert(err, check.IsNil)
 735                         c.Assert(f.Close(), check.IsNil)
 736                 }
 737         }
 738         c.Assert(fs.Sync(), check.IsNil)
 739 }
 740
 741 func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
 742         scope := "20200202/zzzzz/service/aws4_request"
 743         signedHeaders := "date"
 744         req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
 745         stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
 746         c.Assert(err, check.IsNil)
 747         sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
 748         c.Assert(err, check.IsNil)
 749         req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
 750 }
 751
 752 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
 753         stage := s.s3setup(c)
 754         defer stage.teardown(c)
 755         for _, trial := range []struct {
 756                 url            string
 757                 method         string
 758                 body           string
 759                 responseCode   int
 760                 responseRegexp []string
 761                 checkEtag      bool
 762         }{
 763                 {
 764                         url:            "https://" + stage.collbucket.Name + ".example.com/",
 765                         method:         "GET",
 766                         responseCode:   http.StatusOK,
 767                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 768                 },
 769                 {
 770                         url:            "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
 771                         method:         "GET",
 772                         responseCode:   http.StatusOK,
 773                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 774                 },
 775                 {
 776                         url:            "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
 777                         method:         "GET",
 778                         responseCode:   http.StatusOK,
 779                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 780                 },
 781                 {
 782                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
 783                         method:         "GET",
 784                         responseCode:   http.StatusOK,
 785                         responseRegexp: []string{`⛵\n`},
 786                         checkEtag:      true,
 787                 },
 788                 {
 789                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 790                         method:       "PUT",
 791                         body:         "boop",
 792                         responseCode: http.StatusOK,
 793                 },
 794                 {
 795                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 796                         method:         "GET",
 797                         responseCode:   http.StatusOK,
 798                         responseRegexp: []string{`boop`},
 799                         checkEtag:      true,
 800                 },
 801                 {
 802                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 803                         method:       "GET",
 804                         responseCode: http.StatusNotFound,
 805                 },
 806                 {
 807                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 808                         method:       "PUT",
 809                         body:         "boop",
 810                         responseCode: http.StatusOK,
 811                 },
 812                 {
 813                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 814                         method:         "GET",
 815                         responseCode:   http.StatusOK,
 816                         responseRegexp: []string{`boop`},
 817                         checkEtag:      true,
 818                 },
 819         } {
 820                 url, err := url.Parse(trial.url)
 821                 c.Assert(err, check.IsNil)
 822                 req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
 823                 c.Assert(err, check.IsNil)
 824                 s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
 825                 rr := httptest.NewRecorder()
 826                 s.handler.ServeHTTP(rr, req)
 827                 resp := rr.Result()
 828                 c.Check(resp.StatusCode, check.Equals, trial.responseCode)
 829                 body, err := ioutil.ReadAll(resp.Body)
 830                 c.Assert(err, check.IsNil)
 831                 for _, re := range trial.responseRegexp {
 832                         c.Check(string(body), check.Matches, re)
 833                 }
 834                 if trial.checkEtag {
 835                         c.Check(resp.Header.Get("Etag"), check.Matches, `"[\da-f]{32}\+\d+"`)
 836                 }
 837         }
 838 }
 839
 840 func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
 841         stage := s.s3setup(c)
 842         defer stage.teardown(c)
 843         for _, trial := range []struct {
 844                 rawPath        string
 845                 normalizedPath string
 846         }{
 847                 {"/foo", "/foo"},                           // boring case
 848                 {"/foo%5fbar", "/foo_bar"},                 // _ must not be escaped
 849                 {"/foo%2fbar", "/foo/bar"},                 // / must not be escaped
 850                 {"/(foo)/[];,", "/%28foo%29/%5B%5D%3B%2C"}, // ()[];, must be escaped
 851                 {"/foo%5bbar", "/foo%5Bbar"},               // %XX must be uppercase
 852                 // unicode chars must be UTF-8 encoded and escaped
 853                 {"/\u26f5", "/%E2%9B%B5"},
 854                 // "//" and "///" must not be squashed -- see example,
 855                 // https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
 856                 {"//foo///.bar", "//foo///.bar"},
 857         } {
 858                 c.Logf("trial %q", trial)
 859
 860                 date := time.Now().UTC().Format("20060102T150405Z")
 861                 scope := "20200202/zzzzz/S3/aws4_request"
 862                 canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
 863                 c.Logf("canonicalRequest %q", canonicalRequest)
 864                 expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
 865                 c.Logf("expected stringToSign %q", expect)
 866
 867                 req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
 868                 req.Header.Set("X-Amz-Date", date)
 869                 req.Host = "host.example.com"
 870                 c.Assert(err, check.IsNil)
 871
 872                 obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
 873                 if !c.Check(err, check.IsNil) {
 874                         continue
 875                 }
 876                 c.Check(obtained, check.Equals, expect)
 877         }
 878 }
 879
 880 func (s *IntegrationSuite) TestS3GetBucketLocation(c *check.C) {
 881         stage := s.s3setup(c)
 882         defer stage.teardown(c)
 883         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 884                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 885                 c.Check(err, check.IsNil)
 886                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 887                 req.URL.RawQuery = "location"
 888                 resp, err := http.DefaultClient.Do(req)
 889                 c.Assert(err, check.IsNil)
 890                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 891                 buf, err := ioutil.ReadAll(resp.Body)
 892                 c.Assert(err, check.IsNil)
 893                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<LocationConstraint><LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">zzzzz</LocationConstraint></LocationConstraint>\n")
 894         }
 895 }
 896
 897 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
 898         stage := s.s3setup(c)
 899         defer stage.teardown(c)
 900         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 901                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 902                 c.Check(err, check.IsNil)
 903                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 904                 req.URL.RawQuery = "versioning"
 905                 resp, err := http.DefaultClient.Do(req)
 906                 c.Assert(err, check.IsNil)
 907                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 908                 buf, err := ioutil.ReadAll(resp.Body)
 909                 c.Assert(err, check.IsNil)
 910                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>\n")
 911         }
 912 }
 913
 914 func (s *IntegrationSuite) TestS3UnsupportedAPIs(c *check.C) {
 915         stage := s.s3setup(c)
 916         defer stage.teardown(c)
 917         for _, trial := range []struct {
 918                 method   string
 919                 path     string
 920                 rawquery string
 921         }{
 922                 {"GET", "/", "acl&versionId=1234"},    // GetBucketAcl
 923                 {"GET", "/foo", "acl&versionId=1234"}, // GetObjectAcl
 924                 {"PUT", "/", "acl"},                   // PutBucketAcl
 925                 {"PUT", "/foo", "acl"},                // PutObjectAcl
 926                 {"DELETE", "/", "tagging"},            // DeleteBucketTagging
 927                 {"DELETE", "/foo", "tagging"},         // DeleteObjectTagging
 928         } {
 929                 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 930                         c.Logf("trial %v bucket %v", trial, bucket)
 931                         req, err := http.NewRequest(trial.method, bucket.URL(trial.path), nil)
 932                         c.Check(err, check.IsNil)
 933                         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 934                         req.URL.RawQuery = trial.rawquery
 935                         resp, err := http.DefaultClient.Do(req)
 936                         c.Assert(err, check.IsNil)
 937                         c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 938                         buf, err := ioutil.ReadAll(resp.Body)
 939                         c.Assert(err, check.IsNil)
 940                         c.Check(string(buf), check.Matches, "(?ms).*InvalidRequest.*API not supported.*")
 941                 }
 942         }
 943 }
 944
 945 // If there are no CommonPrefixes entries, the CommonPrefixes XML tag
 946 // should not appear at all.
 947 func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
 948         stage := s.s3setup(c)
 949         defer stage.teardown(c)
 950
 951         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 952         c.Assert(err, check.IsNil)
 953         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 954         req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
 955         resp, err := http.DefaultClient.Do(req)
 956         c.Assert(err, check.IsNil)
 957         buf, err := ioutil.ReadAll(resp.Body)
 958         c.Assert(err, check.IsNil)
 959         c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
 960 }
 961
 962 // If there is no delimiter in the request, or the results are not
 963 // truncated, the NextMarker XML tag should not appear in the response
 964 // body.
 965 func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
 966         stage := s.s3setup(c)
 967         defer stage.teardown(c)
 968
 969         for _, query := range []string{"prefix=e&delimiter=/", ""} {
 970                 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 971                 c.Assert(err, check.IsNil)
 972                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 973                 req.URL.RawQuery = query
 974                 resp, err := http.DefaultClient.Do(req)
 975                 c.Assert(err, check.IsNil)
 976                 buf, err := ioutil.ReadAll(resp.Body)
 977                 c.Assert(err, check.IsNil)
 978                 c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`)
 979         }
 980 }
 981
 982 // List response should include KeyCount field.
 983 func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
 984         stage := s.s3setup(c)
 985         defer stage.teardown(c)
 986
 987         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 988         c.Assert(err, check.IsNil)
 989         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 990         req.URL.RawQuery = "prefix=&delimiter=/"
 991         resp, err := http.DefaultClient.Do(req)
 992         c.Assert(err, check.IsNil)
 993         buf, err := ioutil.ReadAll(resp.Body)
 994         c.Assert(err, check.IsNil)
 995         c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
 996 }
 997
 998 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
 999         stage := s.s3setup(c)
1000         defer stage.teardown(c)
1001
1002         var markers int
1003         for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
1004                 dirs := 2000
1005                 filesPerDir := 2
1006                 stage.writeBigDirs(c, dirs, filesPerDir)
1007                 // Total # objects is:
1008                 //                 2 file entries from s3setup (emptyfile and sailboat.txt)
1009                 //                +1 fake "directory" marker from s3setup (emptydir) (if enabled)
1010                 //             +dirs fake "directory" marker from writeBigDirs (dir0/, dir1/) (if enabled)
1011                 // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
1012                 s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
1013                 s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
1014                 s.testS3List(c, stage.collbucket, "", 51, markers+2+(filesPerDir+markers)*dirs)
1015                 s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
1016         }
1017 }
1018 func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
1019         c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.handler.Cluster.Collections.S3FolderObjects)
1020         expectPageSize := pageSize
1021         if expectPageSize > 1000 {
1022                 expectPageSize = 1000
1023         }
1024         gotKeys := map[string]s3.Key{}
1025         nextMarker := ""
1026         pages := 0
1027         for {
1028                 resp, err := bucket.List(prefix, "", nextMarker, pageSize)
1029                 if !c.Check(err, check.IsNil) {
1030                         break
1031                 }
1032                 c.Check(len(resp.Contents) <= expectPageSize, check.Equals, true)
1033                 if pages++; !c.Check(pages <= (expectFiles/expectPageSize)+1, check.Equals, true) {
1034                         break
1035                 }
1036                 for _, key := range resp.Contents {
1037                         if _, dup := gotKeys[key.Key]; dup {
1038                                 c.Errorf("got duplicate key %q on page %d", key.Key, pages)
1039                         }
1040                         gotKeys[key.Key] = key
1041                         if strings.Contains(key.Key, "sailboat.txt") {
1042                                 c.Check(key.Size, check.Equals, int64(4))
1043                         }
1044                 }
1045                 if !resp.IsTruncated {
1046                         c.Check(resp.NextMarker, check.Equals, "")
1047                         break
1048                 }
1049                 if !c.Check(resp.NextMarker, check.Not(check.Equals), "") {
1050                         break
1051                 }
1052                 nextMarker = resp.NextMarker
1053         }
1054         if !c.Check(len(gotKeys), check.Equals, expectFiles) {
1055                 var sorted []string
1056                 for k := range gotKeys {
1057                         sorted = append(sorted, k)
1058                 }
1059                 sort.Strings(sorted)
1060                 for _, k := range sorted {
1061                         c.Logf("got %s", k)
1062                 }
1063         }
1064 }
1065
1066 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
1067         for _, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
1068                 s.testS3CollectionListRollup(c)
1069         }
1070 }
1071
1072 func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
1073         stage := s.s3setup(c)
1074         defer stage.teardown(c)
1075
1076         dirs := 2
1077         filesPerDir := 500
1078         stage.writeBigDirs(c, dirs, filesPerDir)
1079         err := stage.collbucket.PutReader("dingbats", &bytes.Buffer{}, 0, "application/octet-stream", s3.Private, s3.Options{})
1080         c.Assert(err, check.IsNil)
1081         var allfiles []string
1082         for marker := ""; ; {
1083                 resp, err := stage.collbucket.List("", "", marker, 20000)
1084                 c.Check(err, check.IsNil)
1085                 for _, key := range resp.Contents {
1086                         if len(allfiles) == 0 || allfiles[len(allfiles)-1] != key.Key {
1087                                 allfiles = append(allfiles, key.Key)
1088                         }
1089                 }
1090                 marker = resp.NextMarker
1091                 if marker == "" {
1092                         break
1093                 }
1094         }
1095         markers := 0
1096         if s.handler.Cluster.Collections.S3FolderObjects {
1097                 markers = 1
1098         }
1099         c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
1100
1101         gotDirMarker := map[string]bool{}
1102         for _, name := range allfiles {
1103                 isDirMarker := strings.HasSuffix(name, "/")
1104                 if markers == 0 {
1105                         c.Check(isDirMarker, check.Equals, false, check.Commentf("name %q", name))
1106                 } else if isDirMarker {
1107                         gotDirMarker[name] = true
1108                 } else if i := strings.LastIndex(name, "/"); i >= 0 {
1109                         c.Check(gotDirMarker[name[:i+1]], check.Equals, true, check.Commentf("name %q", name))
1110                         gotDirMarker[name[:i+1]] = true // skip redundant complaints about this dir marker
1111                 }
1112         }
1113
1114         for _, trial := range []struct {
1115                 prefix    string
1116                 delimiter string
1117                 marker    string
1118         }{
1119                 {"", "", ""},
1120                 {"di", "/", ""},
1121                 {"di", "r", ""},
1122                 {"di", "n", ""},
1123                 {"dir0", "/", ""},
1124                 {"dir0/", "/", ""},
1125                 {"dir0/f", "/", ""},
1126                 {"dir0", "", ""},
1127                 {"dir0/", "", ""},
1128                 {"dir0/f", "", ""},
1129                 {"dir0", "/", "dir0/file14.txt"},       // one commonprefix, "dir0/"
1130                 {"dir0", "/", "dir0/zzzzfile.txt"},     // no commonprefixes
1131                 {"", "", "dir0/file14.txt"},            // middle page, skip walking dir1
1132                 {"", "", "dir1/file14.txt"},            // middle page, skip walking dir0
1133                 {"", "", "dir1/file498.txt"},           // last page of results
1134                 {"dir1/file", "", "dir1/file498.txt"},  // last page of results, with prefix
1135                 {"dir1/file", "/", "dir1/file498.txt"}, // last page of results, with prefix + delimiter
1136                 {"dir1", "Z", "dir1/file498.txt"},      // delimiter "Z" never appears
1137                 {"dir2", "/", ""},                      // prefix "dir2" does not exist
1138                 {"", "/", ""},
1139         } {
1140                 c.Logf("\n\n=== trial %+v markers=%d", trial, markers)
1141
1142                 maxKeys := 20
1143                 resp, err := stage.collbucket.List(trial.prefix, trial.delimiter, trial.marker, maxKeys)
1144                 c.Check(err, check.IsNil)
1145                 if resp.IsTruncated && trial.delimiter == "" {
1146                         // goamz List method fills in the missing
1147                         // NextMarker field if resp.IsTruncated, so
1148                         // now we can't really tell whether it was
1149                         // sent by the server or by goamz. In cases
1150                         // where it should be empty but isn't, assume
1151                         // it's goamz's fault.
1152                         resp.NextMarker = ""
1153                 }
1154
1155                 var expectKeys []string
1156                 var expectPrefixes []string
1157                 var expectNextMarker string
1158                 var expectTruncated bool
1159                 for _, key := range allfiles {
1160                         full := len(expectKeys)+len(expectPrefixes) >= maxKeys
1161                         if !strings.HasPrefix(key, trial.prefix) || key <= trial.marker {
1162                                 continue
1163                         } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
1164                                 prefix := key[:len(trial.prefix)+idx+1]
1165                                 if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
1166                                         // same prefix as previous key
1167                                 } else if full {
1168                                         expectTruncated = true
1169                                 } else {
1170                                         expectPrefixes = append(expectPrefixes, prefix)
1171                                         expectNextMarker = prefix
1172                                 }
1173                         } else if full {
1174                                 expectTruncated = true
1175                                 break
1176                         } else {
1177                                 expectKeys = append(expectKeys, key)
1178                                 if trial.delimiter != "" {
1179                                         expectNextMarker = key
1180                                 }
1181                         }
1182                 }
1183                 if !expectTruncated {
1184                         expectNextMarker = ""
1185                 }
1186
1187                 var gotKeys []string
1188                 for _, key := range resp.Contents {
1189                         gotKeys = append(gotKeys, key.Key)
1190                 }
1191                 var gotPrefixes []string
1192                 for _, prefix := range resp.CommonPrefixes {
1193                         gotPrefixes = append(gotPrefixes, prefix)
1194                 }
1195                 commentf := check.Commentf("trial %+v markers=%d", trial, markers)
1196                 c.Check(gotKeys, check.DeepEquals, expectKeys, commentf)
1197                 c.Check(gotPrefixes, check.DeepEquals, expectPrefixes, commentf)
1198                 c.Check(resp.NextMarker, check.Equals, expectNextMarker, commentf)
1199                 c.Check(resp.IsTruncated, check.Equals, expectTruncated, commentf)
1200                 c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
1201         }
1202 }
1203
1204 func (s *IntegrationSuite) TestS3ListObjectsV2ManySubprojects(c *check.C) {
1205         stage := s.s3setup(c)
1206         defer stage.teardown(c)
1207         projects := 50
1208         collectionsPerProject := 2
1209         for i := 0; i < projects; i++ {
1210                 var subproj arvados.Group
1211                 err := stage.arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
1212                         "group": map[string]interface{}{
1213                                 "owner_uuid":  stage.subproj.UUID,
1214                                 "group_class": "project",
1215                                 "name":        fmt.Sprintf("keep-web s3 test subproject %d", i),
1216                         },
1217                 })
1218                 c.Assert(err, check.IsNil)
1219                 for j := 0; j < collectionsPerProject; j++ {
1220                         err = stage.arv.RequestAndDecode(nil, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
1221                                 "owner_uuid":    subproj.UUID,
1222                                 "name":          fmt.Sprintf("keep-web s3 test collection %d", j),
1223                                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
1224                         }})
1225                         c.Assert(err, check.IsNil)
1226                 }
1227         }
1228         c.Logf("setup complete")
1229
1230         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1231                 Region:           aws_aws.String("auto"),
1232                 Endpoint:         aws_aws.String(s.testServer.URL),
1233                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1234                 S3ForcePathStyle: aws_aws.Bool(true),
1235         }))
1236         client := aws_s3.New(sess)
1237         ctx := context.Background()
1238         params := aws_s3.ListObjectsV2Input{
1239                 Bucket:    aws_aws.String(stage.proj.UUID),
1240                 Delimiter: aws_aws.String("/"),
1241                 Prefix:    aws_aws.String("keep-web s3 test subproject/"),
1242                 MaxKeys:   aws_aws.Int64(int64(projects / 2)),
1243         }
1244         for page := 1; ; page++ {
1245                 t0 := time.Now()
1246                 result, err := client.ListObjectsV2WithContext(ctx, &params)
1247                 if !c.Check(err, check.IsNil) {
1248                         break
1249                 }
1250                 c.Logf("got page %d in %v with len(Contents) == %d, len(CommonPrefixes) == %d", page, time.Since(t0), len(result.Contents), len(result.CommonPrefixes))
1251                 if !*result.IsTruncated {
1252                         break
1253                 }
1254                 params.ContinuationToken = result.NextContinuationToken
1255                 *params.MaxKeys = *params.MaxKeys/2 + 1
1256         }
1257 }
1258
1259 func (s *IntegrationSuite) TestS3ListObjectsV2(c *check.C) {
1260         stage := s.s3setup(c)
1261         defer stage.teardown(c)
1262         dirs := 2
1263         filesPerDir := 40
1264         stage.writeBigDirs(c, dirs, filesPerDir)
1265
1266         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1267                 Region:           aws_aws.String("auto"),
1268                 Endpoint:         aws_aws.String(s.testServer.URL),
1269                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1270                 S3ForcePathStyle: aws_aws.Bool(true),
1271         }))
1272
1273         stringOrNil := func(s string) *string {
1274                 if s == "" {
1275                         return nil
1276                 } else {
1277                         return &s
1278                 }
1279         }
1280
1281         client := aws_s3.New(sess)
1282         ctx := context.Background()
1283
1284         for _, trial := range []struct {
1285                 prefix               string
1286                 delimiter            string
1287                 startAfter           string
1288                 maxKeys              int
1289                 expectKeys           int
1290                 expectCommonPrefixes map[string]bool
1291         }{
1292                 {
1293                         // Expect {filesPerDir plus the dir itself}
1294                         // for each dir, plus emptydir, emptyfile, and
1295                         // sailboat.txt.
1296                         expectKeys: (filesPerDir+1)*dirs + 3,
1297                 },
1298                 {
1299                         maxKeys:    15,
1300                         expectKeys: (filesPerDir+1)*dirs + 3,
1301                 },
1302                 {
1303                         startAfter: "dir0/z",
1304                         maxKeys:    15,
1305                         // Expect {filesPerDir plus the dir itself}
1306                         // for each dir except dir0, plus emptydir,
1307                         // emptyfile, and sailboat.txt.
1308                         expectKeys: (filesPerDir+1)*(dirs-1) + 3,
1309                 },
1310                 {
1311                         maxKeys:              1,
1312                         delimiter:            "/",
1313                         expectKeys:           2, // emptyfile, sailboat.txt
1314                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1315                 },
1316                 {
1317                         startAfter:           "dir0/z",
1318                         maxKeys:              15,
1319                         delimiter:            "/",
1320                         expectKeys:           2, // emptyfile, sailboat.txt
1321                         expectCommonPrefixes: map[string]bool{"dir1/": true, "emptydir/": true},
1322                 },
1323                 {
1324                         startAfter:           "dir0/file10.txt",
1325                         maxKeys:              15,
1326                         delimiter:            "/",
1327                         expectKeys:           2,
1328                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1329                 },
1330                 {
1331                         startAfter:           "dir0/file10.txt",
1332                         maxKeys:              15,
1333                         prefix:               "d",
1334                         delimiter:            "/",
1335                         expectKeys:           0,
1336                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true},
1337                 },
1338         } {
1339                 c.Logf("[trial %+v]", trial)
1340                 params := aws_s3.ListObjectsV2Input{
1341                         Bucket:     aws_aws.String(stage.collbucket.Name),
1342                         Prefix:     stringOrNil(trial.prefix),
1343                         Delimiter:  stringOrNil(trial.delimiter),
1344                         StartAfter: stringOrNil(trial.startAfter),
1345                         MaxKeys:    aws_aws.Int64(int64(trial.maxKeys)),
1346                 }
1347                 keySeen := map[string]bool{}
1348                 prefixSeen := map[string]bool{}
1349                 for {
1350                         result, err := client.ListObjectsV2WithContext(ctx, &params)
1351                         if !c.Check(err, check.IsNil) {
1352                                 break
1353                         }
1354                         c.Check(result.Name, check.DeepEquals, aws_aws.String(stage.collbucket.Name))
1355                         c.Check(result.Prefix, check.DeepEquals, aws_aws.String(trial.prefix))
1356                         c.Check(result.Delimiter, check.DeepEquals, aws_aws.String(trial.delimiter))
1357                         // The following two fields are expected to be
1358                         // nil (i.e., no tag in XML response) rather
1359                         // than "" when the corresponding request
1360                         // field was empty or nil.
1361                         c.Check(result.StartAfter, check.DeepEquals, stringOrNil(trial.startAfter))
1362                         c.Check(result.ContinuationToken, check.DeepEquals, params.ContinuationToken)
1363
1364                         if trial.maxKeys > 0 {
1365                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(trial.maxKeys)))
1366                                 c.Check(len(result.Contents)+len(result.CommonPrefixes) <= trial.maxKeys, check.Equals, true)
1367                         } else {
1368                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(s3MaxKeys)))
1369                         }
1370
1371                         for _, ent := range result.Contents {
1372                                 c.Assert(ent.Key, check.NotNil)
1373                                 c.Check(*ent.Key > trial.startAfter, check.Equals, true)
1374                                 c.Check(keySeen[*ent.Key], check.Equals, false, check.Commentf("dup key %q", *ent.Key))
1375                                 keySeen[*ent.Key] = true
1376                         }
1377                         for _, ent := range result.CommonPrefixes {
1378                                 c.Assert(ent.Prefix, check.NotNil)
1379                                 c.Check(strings.HasSuffix(*ent.Prefix, trial.delimiter), check.Equals, true, check.Commentf("bad CommonPrefix %q", *ent.Prefix))
1380                                 if strings.HasPrefix(trial.startAfter, *ent.Prefix) {
1381                                         // If we asked for
1382                                         // startAfter=dir0/file10.txt,
1383                                         // we expect dir0/ to be
1384                                         // returned as a common prefix
1385                                 } else {
1386                                         c.Check(*ent.Prefix > trial.startAfter, check.Equals, true)
1387                                 }
1388                                 c.Check(prefixSeen[*ent.Prefix], check.Equals, false, check.Commentf("dup common prefix %q", *ent.Prefix))
1389                                 prefixSeen[*ent.Prefix] = true
1390                         }
1391                         if *result.IsTruncated && c.Check(result.NextContinuationToken, check.Not(check.Equals), "") {
1392                                 params.ContinuationToken = aws_aws.String(*result.NextContinuationToken)
1393                         } else {
1394                                 break
1395                         }
1396                 }
1397                 c.Check(keySeen, check.HasLen, trial.expectKeys)
1398                 c.Check(prefixSeen, check.HasLen, len(trial.expectCommonPrefixes))
1399                 if len(trial.expectCommonPrefixes) > 0 {
1400                         c.Check(prefixSeen, check.DeepEquals, trial.expectCommonPrefixes)
1401                 }
1402         }
1403 }
1404
1405 func (s *IntegrationSuite) TestS3ListObjectsV2EncodingTypeURL(c *check.C) {
1406         stage := s.s3setup(c)
1407         defer stage.teardown(c)
1408         dirs := 2
1409         filesPerDir := 40
1410         stage.writeBigDirs(c, dirs, filesPerDir)
1411
1412         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1413                 Region:           aws_aws.String("auto"),
1414                 Endpoint:         aws_aws.String(s.testServer.URL),
1415                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1416                 S3ForcePathStyle: aws_aws.Bool(true),
1417         }))
1418
1419         client := aws_s3.New(sess)
1420         ctx := context.Background()
1421
1422         result, err := client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1423                 Bucket:       aws_aws.String(stage.collbucket.Name),
1424                 Prefix:       aws_aws.String("dir0/"),
1425                 Delimiter:    aws_aws.String("/"),
1426                 StartAfter:   aws_aws.String("dir0/"),
1427                 EncodingType: aws_aws.String("url"),
1428         })
1429         c.Assert(err, check.IsNil)
1430         c.Check(*result.Prefix, check.Equals, "dir0%2F")
1431         c.Check(*result.Delimiter, check.Equals, "%2F")
1432         c.Check(*result.StartAfter, check.Equals, "dir0%2F")
1433         for _, ent := range result.Contents {
1434                 c.Check(*ent.Key, check.Matches, "dir0%2F.*")
1435         }
1436         result, err = client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1437                 Bucket:       aws_aws.String(stage.collbucket.Name),
1438                 Delimiter:    aws_aws.String("/"),
1439                 EncodingType: aws_aws.String("url"),
1440         })
1441         c.Assert(err, check.IsNil)
1442         c.Check(*result.Delimiter, check.Equals, "%2F")
1443         c.Check(result.CommonPrefixes, check.HasLen, dirs+1)
1444         for _, ent := range result.CommonPrefixes {
1445                 c.Check(*ent.Prefix, check.Matches, ".*%2F")
1446         }
1447 }
1448
1449 // TestS3cmd checks compatibility with the s3cmd command line tool, if
1450 // it's installed. As of Debian buster, s3cmd is only in backports, so
1451 // `arvados-server install` don't install it, and this test skips if
1452 // it's not installed.
1453 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
1454         if _, err := exec.LookPath("s3cmd"); err != nil {
1455                 c.Skip("s3cmd not found")
1456                 return
1457         }
1458
1459         stage := s.s3setup(c)
1460         defer stage.teardown(c)
1461
1462         cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
1463         buf, err := cmd.CombinedOutput()
1464         c.Check(err, check.IsNil)
1465         c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
1466
1467         // This tests whether s3cmd's path normalization agrees with
1468         // keep-web's signature verification wrt chars like "|"
1469         // (neither reserved nor unreserved) and "," (not normally
1470         // percent-encoded in a path).
1471         tmpfile := c.MkDir() + "/dstfile"
1472         cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar", tmpfile)
1473         buf, err = cmd.CombinedOutput()
1474         c.Check(err, check.NotNil)
1475         // As of commit b7520e5c25e1bf25c1a8bf5aa2eadb299be8f606
1476         // (between debian bullseye and bookworm versions), s3cmd
1477         // started catching the NoSuchKey error code and replacing it
1478         // with "Source object '%s' does not exist.".
1479         c.Check(string(buf), check.Matches, `(?ms).*(NoSuchKey|Source object.*does not exist).*\n`)
1480
1481         tmpfile = c.MkDir() + "/foo"
1482         cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo", tmpfile)
1483         buf, err = cmd.CombinedOutput()
1484         c.Logf("%s", buf)
1485         if c.Check(err, check.IsNil) {
1486                 checkcontent, err := os.ReadFile(tmpfile)
1487                 c.Check(err, check.IsNil)
1488                 c.Check(string(checkcontent), check.Equals, "foo")
1489         }
1490 }
1491
1492 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
1493         stage := s.s3setup(c)
1494         defer stage.teardown(c)
1495
1496         hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
1497         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
1498         c.Check(body, check.Equals, "⛵\n")
1499 }