services/keep-web/s3_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/rand"
  11         "crypto/sha256"
  12         "fmt"
  13         "io"
  14         "io/fs"
  15         "io/ioutil"
  16         "mime"
  17         "net/http"
  18         "net/http/httptest"
  19         "net/url"
  20         "os"
  21         "os/exec"
  22         "sort"
  23         "strings"
  24         "sync"
  25         "time"
  26
  27         "git.arvados.org/arvados.git/sdk/go/arvados"
  28         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  29         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  30         "git.arvados.org/arvados.git/sdk/go/keepclient"
  31         "github.com/AdRoll/goamz/aws"
  32         "github.com/AdRoll/goamz/s3"
  33         aws_aws "github.com/aws/aws-sdk-go/aws"
  34         aws_credentials "github.com/aws/aws-sdk-go/aws/credentials"
  35         aws_session "github.com/aws/aws-sdk-go/aws/session"
  36         aws_s3 "github.com/aws/aws-sdk-go/service/s3"
  37         check "gopkg.in/check.v1"
  38 )
  39
  40 type CachedS3SecretSuite struct{}
  41
  42 var _ = check.Suite(&CachedS3SecretSuite{})
  43
  44 func (s *CachedS3SecretSuite) activeACA(expiresAt time.Time) *arvados.APIClientAuthorization {
  45         return &arvados.APIClientAuthorization{
  46                 UUID:      arvadostest.ActiveTokenUUID,
  47                 APIToken:  arvadostest.ActiveToken,
  48                 ExpiresAt: expiresAt,
  49         }
  50 }
  51
  52 func (s *CachedS3SecretSuite) TestNewCachedS3SecretExpiresBeforeTTL(c *check.C) {
  53         expected := time.Unix(1<<29, 0)
  54         aca := s.activeACA(expected)
  55         actual := newCachedS3Secret(aca, time.Unix(1<<30, 0))
  56         c.Check(actual.expiry, check.Equals, expected)
  57 }
  58
  59 func (s *CachedS3SecretSuite) TestNewCachedS3SecretExpiresAfterTTL(c *check.C) {
  60         expected := time.Unix(1<<29, 0)
  61         aca := s.activeACA(time.Unix(1<<30, 0))
  62         actual := newCachedS3Secret(aca, expected)
  63         c.Check(actual.expiry, check.Equals, expected)
  64 }
  65
  66 func (s *CachedS3SecretSuite) TestNewCachedS3SecretWithoutExpiry(c *check.C) {
  67         expected := time.Unix(1<<29, 0)
  68         aca := s.activeACA(time.Time{})
  69         actual := newCachedS3Secret(aca, expected)
  70         c.Check(actual.expiry, check.Equals, expected)
  71 }
  72
  73 func (s *CachedS3SecretSuite) cachedSecretWithExpiry(expiry time.Time) *cachedS3Secret {
  74         return &cachedS3Secret{
  75                 auth:   s.activeACA(expiry),
  76                 expiry: expiry,
  77         }
  78 }
  79
  80 func (s *CachedS3SecretSuite) TestIsValidAtEmpty(c *check.C) {
  81         cache := &cachedS3Secret{}
  82         c.Check(cache.isValidAt(time.Unix(0, 0)), check.Equals, false)
  83         c.Check(cache.isValidAt(time.Unix(1<<31, 0)), check.Equals, false)
  84 }
  85
  86 func (s *CachedS3SecretSuite) TestIsValidAtNoAuth(c *check.C) {
  87         cache := &cachedS3Secret{expiry: time.Unix(3, 0)}
  88         c.Check(cache.isValidAt(time.Unix(2, 0)), check.Equals, false)
  89         c.Check(cache.isValidAt(time.Unix(4, 0)), check.Equals, false)
  90 }
  91
  92 func (s *CachedS3SecretSuite) TestIsValidAtNoExpiry(c *check.C) {
  93         cache := &cachedS3Secret{auth: s.activeACA(time.Unix(3, 0))}
  94         c.Check(cache.isValidAt(time.Unix(2, 0)), check.Equals, false)
  95         c.Check(cache.isValidAt(time.Unix(4, 0)), check.Equals, false)
  96 }
  97
  98 func (s *CachedS3SecretSuite) TestIsValidAtTimeAfterExpiry(c *check.C) {
  99         expiry := time.Unix(10, 0)
 100         cache := s.cachedSecretWithExpiry(expiry)
 101         c.Check(cache.isValidAt(expiry), check.Equals, false)
 102         c.Check(cache.isValidAt(time.Unix(1<<25, 0)), check.Equals, false)
 103         c.Check(cache.isValidAt(time.Unix(1<<30, 0)), check.Equals, false)
 104 }
 105
 106 func (s *CachedS3SecretSuite) TestIsValidAtTimeBeforeExpiry(c *check.C) {
 107         cache := s.cachedSecretWithExpiry(time.Unix(1<<30, 0))
 108         c.Check(cache.isValidAt(time.Unix(1<<25, 0)), check.Equals, true)
 109         c.Check(cache.isValidAt(time.Unix(1<<27, 0)), check.Equals, true)
 110         c.Check(cache.isValidAt(time.Unix(1<<29, 0)), check.Equals, true)
 111 }
 112
 113 func (s *CachedS3SecretSuite) TestIsValidAtZeroTime(c *check.C) {
 114         cache := s.cachedSecretWithExpiry(time.Unix(10, 0))
 115         c.Check(cache.isValidAt(time.Time{}), check.Equals, false)
 116 }
 117
 118 type s3stage struct {
 119         arv        *arvados.Client
 120         ac         *arvadosclient.ArvadosClient
 121         kc         *keepclient.KeepClient
 122         proj       arvados.Group
 123         projbucket *s3.Bucket
 124         subproj    arvados.Group
 125         coll       arvados.Collection
 126         collbucket *s3.Bucket
 127 }
 128
 129 func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
 130         var proj, subproj arvados.Group
 131         var coll arvados.Collection
 132         arv := arvados.NewClientFromEnv()
 133         arv.AuthToken = arvadostest.ActiveToken
 134         err := arv.RequestAndDecode(&proj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
 135                 "group": map[string]interface{}{
 136                         "group_class": "project",
 137                         "name":        "keep-web s3 test",
 138                         "properties": map[string]interface{}{
 139                                 "project-properties-key": "project properties value",
 140                         },
 141                 },
 142                 "ensure_unique_name": true,
 143         })
 144         c.Assert(err, check.IsNil)
 145         err = arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
 146                 "group": map[string]interface{}{
 147                         "owner_uuid":  proj.UUID,
 148                         "group_class": "project",
 149                         "name":        "keep-web s3 test subproject",
 150                         "properties": map[string]interface{}{
 151                                 "subproject_properties_key": "subproject properties value",
 152                                 "invalid header key":        "this value will not be returned because key contains spaces",
 153                         },
 154                 },
 155         })
 156         c.Assert(err, check.IsNil)
 157         err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
 158                 "owner_uuid":    proj.UUID,
 159                 "name":          "keep-web s3 test collection",
 160                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
 161                 "properties": map[string]interface{}{
 162                         "string":   "string value",
 163                         "array":    []string{"element1", "element2"},
 164                         "object":   map[string]interface{}{"key": map[string]interface{}{"key2": "value⛵"}},
 165                         "nonascii": "⛵",
 166                         "newline":  "foo\r\nX-Bad: header",
 167                         // This key cannot be expressed as a MIME
 168                         // header key, so it will be silently skipped
 169                         // (see "Inject" in PropertiesAsMetadata test)
 170                         "a: a\r\nInject": "bogus",
 171                 },
 172         }})
 173         c.Assert(err, check.IsNil)
 174         ac, err := arvadosclient.New(arv)
 175         c.Assert(err, check.IsNil)
 176         kc, err := keepclient.MakeKeepClient(ac)
 177         c.Assert(err, check.IsNil)
 178         fs, err := coll.FileSystem(arv, kc)
 179         c.Assert(err, check.IsNil)
 180         f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
 181         c.Assert(err, check.IsNil)
 182         _, err = f.Write([]byte("⛵\n"))
 183         c.Assert(err, check.IsNil)
 184         err = f.Close()
 185         c.Assert(err, check.IsNil)
 186         err = fs.Sync()
 187         c.Assert(err, check.IsNil)
 188         err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
 189         c.Assert(err, check.IsNil)
 190
 191         auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
 192         region := aws.Region{
 193                 Name:       "zzzzz",
 194                 S3Endpoint: s.testServer.URL,
 195         }
 196         client := s3.New(*auth, region)
 197         client.Signature = aws.V4Signature
 198         return s3stage{
 199                 arv:  arv,
 200                 ac:   ac,
 201                 kc:   kc,
 202                 proj: proj,
 203                 projbucket: &s3.Bucket{
 204                         S3:   client,
 205                         Name: proj.UUID,
 206                 },
 207                 subproj: subproj,
 208                 coll:    coll,
 209                 collbucket: &s3.Bucket{
 210                         S3:   client,
 211                         Name: coll.UUID,
 212                 },
 213         }
 214 }
 215
 216 func (stage s3stage) teardown(c *check.C) {
 217         if stage.coll.UUID != "" {
 218                 err := stage.arv.RequestAndDecode(&stage.coll, "DELETE", "arvados/v1/collections/"+stage.coll.UUID, nil, nil)
 219                 c.Check(err, check.IsNil)
 220         }
 221         if stage.proj.UUID != "" {
 222                 err := stage.arv.RequestAndDecode(&stage.proj, "DELETE", "arvados/v1/groups/"+stage.proj.UUID, nil, nil)
 223                 c.Check(err, check.IsNil)
 224         }
 225 }
 226
 227 func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
 228         stage := s.s3setup(c)
 229         defer stage.teardown(c)
 230
 231         bucket := stage.collbucket
 232         for _, trial := range []struct {
 233                 success   bool
 234                 signature int
 235                 accesskey string
 236                 secretkey string
 237         }{
 238                 {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
 239                 {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
 240                 {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
 241                 {false, aws.V2Signature, "none", "none"},
 242                 {false, aws.V2Signature, "none", arvadostest.ActiveToken},
 243
 244                 {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
 245                 {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
 246                 {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
 247                 {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
 248                 {false, aws.V4Signature, arvadostest.ActiveToken, ""},
 249                 {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
 250                 {false, aws.V4Signature, "none", arvadostest.ActiveToken},
 251                 {false, aws.V4Signature, "none", "none"},
 252         } {
 253                 c.Logf("%#v", trial)
 254                 bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
 255                 bucket.S3.Signature = trial.signature
 256                 _, err := bucket.GetReader("emptyfile")
 257                 if trial.success {
 258                         c.Check(err, check.IsNil)
 259                 } else {
 260                         c.Check(err, check.NotNil)
 261                 }
 262         }
 263 }
 264
 265 func (s *IntegrationSuite) TestS3SecretCacheUpdates(c *check.C) {
 266         stage := s.s3setup(c)
 267         defer stage.teardown(c)
 268         reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
 269         c.Assert(err, check.IsNil)
 270
 271         for trialName, trialAuth := range map[string]string{
 272                 "v1 token":                    arvadostest.ActiveToken,
 273                 "token UUID":                  arvadostest.ActiveTokenUUID,
 274                 "v2 token query escaped":      url.QueryEscape(arvadostest.ActiveTokenV2),
 275                 "v2 token underscore escaped": strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1),
 276         } {
 277                 s.handler.s3SecretCache = nil
 278                 req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
 279                 if !c.Check(err, check.IsNil) {
 280                         continue
 281                 }
 282                 secret := trialAuth
 283                 if secret[5:12] == "-gj3su-" {
 284                         secret = arvadostest.ActiveToken
 285                 }
 286                 s.sign(c, req, trialAuth, secret)
 287                 rec := httptest.NewRecorder()
 288                 s.handler.ServeHTTP(rec, req)
 289                 if !c.Check(rec.Result().StatusCode, check.Equals, http.StatusOK,
 290                         check.Commentf("%s auth did not get 200 OK response: %v", trialName, req)) {
 291                         continue
 292                 }
 293
 294                 for name, key := range map[string]string{
 295                         "v1 token":   arvadostest.ActiveToken,
 296                         "token UUID": arvadostest.ActiveTokenUUID,
 297                         "v2 token":   arvadostest.ActiveTokenV2,
 298                 } {
 299                         actual, ok := s.handler.s3SecretCache[key]
 300                         if c.Check(ok, check.Equals, true, check.Commentf("%s not cached from %s", name, trialName)) {
 301                                 c.Check(actual.auth.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 302                         }
 303                 }
 304         }
 305 }
 306
 307 func (s *IntegrationSuite) TestS3SecretCacheUsed(c *check.C) {
 308         stage := s.s3setup(c)
 309         defer stage.teardown(c)
 310
 311         token := arvadostest.ActiveToken
 312         // Step 1: Make a request to get the active token in the cache.
 313         reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
 314         c.Assert(err, check.IsNil)
 315         req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
 316         s.sign(c, req, token, token)
 317         rec := httptest.NewRecorder()
 318         s.handler.ServeHTTP(rec, req)
 319         resp := rec.Result()
 320         c.Assert(resp.StatusCode, check.Equals, http.StatusOK,
 321                 check.Commentf("first request did not get 200 OK response"))
 322
 323         // Step 2: Remove some cache keys our request doesn't rely upon.
 324         c.Assert(s.handler.s3SecretCache[arvadostest.ActiveTokenUUID], check.NotNil)
 325         delete(s.handler.s3SecretCache, arvadostest.ActiveTokenUUID)
 326         c.Assert(s.handler.s3SecretCache[arvadostest.ActiveTokenV2], check.NotNil)
 327         delete(s.handler.s3SecretCache, arvadostest.ActiveTokenV2)
 328
 329         // Step 3: Repeat the original request.
 330         rec = httptest.NewRecorder()
 331         s.handler.ServeHTTP(rec, req)
 332         resp = rec.Result()
 333         c.Assert(resp.StatusCode, check.Equals, http.StatusOK,
 334                 check.Commentf("cached auth request did not get 200 OK response"))
 335
 336         // Step 4: Confirm the deleted cache keys were not re-added
 337         // (which would imply the authorization was re-requested and cached).
 338         c.Check(s.handler.s3SecretCache[arvadostest.ActiveTokenUUID], check.IsNil,
 339                 check.Commentf("token UUID re-added to cache after removal"))
 340         c.Check(s.handler.s3SecretCache[arvadostest.ActiveTokenV2], check.IsNil,
 341                 check.Commentf("v2 token re-added to cache after removal"))
 342 }
 343
 344 func (s *IntegrationSuite) TestS3SecretCacheCleanup(c *check.C) {
 345         stage := s.s3setup(c)
 346         defer stage.teardown(c)
 347         td := -2 * s3SecretCacheTidyInterval
 348         startTidied := time.Now().Add(td)
 349         s.handler.s3SecretCacheNextTidy = startTidied
 350         s.handler.s3SecretCache = make(map[string]*cachedS3Secret)
 351         s.handler.s3SecretCache["old"] = &cachedS3Secret{expiry: startTidied.Add(td)}
 352
 353         reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
 354         c.Assert(err, check.IsNil)
 355         req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
 356         token := arvadostest.ActiveToken
 357         s.sign(c, req, token, token)
 358         rec := httptest.NewRecorder()
 359         s.handler.ServeHTTP(rec, req)
 360
 361         c.Check(s.handler.s3SecretCache["old"], check.IsNil,
 362                 check.Commentf("expired token not removed from cache"))
 363         c.Check(s.handler.s3SecretCacheNextTidy.After(startTidied), check.Equals, true,
 364                 check.Commentf("s3SecretCacheNextTidy not updated"))
 365         c.Check(s.handler.s3SecretCache[token], check.NotNil,
 366                 check.Commentf("just-used token not found in cache"))
 367 }
 368
 369 func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
 370         stage := s.s3setup(c)
 371         defer stage.teardown(c)
 372
 373         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 374                 c.Logf("bucket %s", bucket.Name)
 375                 exists, err := bucket.Exists("")
 376                 c.Check(err, check.IsNil)
 377                 c.Check(exists, check.Equals, true)
 378         }
 379 }
 380
 381 func (s *IntegrationSuite) TestS3CollectionGetObject(c *check.C) {
 382         stage := s.s3setup(c)
 383         defer stage.teardown(c)
 384         s.testS3GetObject(c, stage.collbucket, "")
 385 }
 386 func (s *IntegrationSuite) TestS3ProjectGetObject(c *check.C) {
 387         stage := s.s3setup(c)
 388         defer stage.teardown(c)
 389         s.testS3GetObject(c, stage.projbucket, stage.coll.Name+"/")
 390 }
 391 func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix string) {
 392         rdr, err := bucket.GetReader(prefix + "emptyfile")
 393         c.Assert(err, check.IsNil)
 394         buf, err := ioutil.ReadAll(rdr)
 395         c.Check(err, check.IsNil)
 396         c.Check(len(buf), check.Equals, 0)
 397         err = rdr.Close()
 398         c.Check(err, check.IsNil)
 399
 400         // GetObject
 401         rdr, err = bucket.GetReader(prefix + "missingfile")
 402         c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 403         c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 404         c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
 405
 406         // HeadObject
 407         exists, err := bucket.Exists(prefix + "missingfile")
 408         c.Check(err, check.IsNil)
 409         c.Check(exists, check.Equals, false)
 410
 411         // GetObject
 412         rdr, err = bucket.GetReader(prefix + "sailboat.txt")
 413         c.Assert(err, check.IsNil)
 414         buf, err = ioutil.ReadAll(rdr)
 415         c.Check(err, check.IsNil)
 416         c.Check(buf, check.DeepEquals, []byte("⛵\n"))
 417         err = rdr.Close()
 418         c.Check(err, check.IsNil)
 419
 420         // HeadObject
 421         resp, err := bucket.Head(prefix+"sailboat.txt", nil)
 422         c.Check(err, check.IsNil)
 423         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 424         c.Check(resp.ContentLength, check.Equals, int64(4))
 425
 426         // HeadObject with superfluous leading slashes
 427         exists, err = bucket.Exists(prefix + "//sailboat.txt")
 428         c.Check(err, check.IsNil)
 429         c.Check(exists, check.Equals, false)
 430 }
 431
 432 func (s *IntegrationSuite) checkMetaEquals(c *check.C, hdr http.Header, expect map[string]string) {
 433         got := map[string]string{}
 434         for hk, hv := range hdr {
 435                 if k := strings.TrimPrefix(hk, "X-Amz-Meta-"); k != hk && len(hv) == 1 {
 436                         got[k] = hv[0]
 437                 }
 438         }
 439         c.Check(got, check.DeepEquals, expect)
 440 }
 441
 442 func (s *IntegrationSuite) TestS3PropertiesAsMetadata(c *check.C) {
 443         stage := s.s3setup(c)
 444         defer stage.teardown(c)
 445
 446         expectCollectionTags := map[string]string{
 447                 "String":   "string value",
 448                 "Array":    `["element1","element2"]`,
 449                 "Object":   mime.BEncoding.Encode("UTF-8", `{"key":{"key2":"value⛵"}}`),
 450                 "Nonascii": "=?UTF-8?b?4pu1?=",
 451                 "Newline":  mime.BEncoding.Encode("UTF-8", "foo\r\nX-Bad: header"),
 452         }
 453         expectSubprojectTags := map[string]string{
 454                 "Subproject_properties_key": "subproject properties value",
 455         }
 456         expectProjectTags := map[string]string{
 457                 "Project-Properties-Key": "project properties value",
 458         }
 459
 460         c.Log("HEAD object with metadata from collection")
 461         resp, err := stage.collbucket.Head("sailboat.txt", nil)
 462         c.Assert(err, check.IsNil)
 463         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 464
 465         c.Log("GET object with metadata from collection")
 466         rdr, hdr, err := stage.collbucket.GetReaderWithHeaders("sailboat.txt")
 467         c.Assert(err, check.IsNil)
 468         content, err := ioutil.ReadAll(rdr)
 469         c.Check(err, check.IsNil)
 470         rdr.Close()
 471         c.Check(content, check.HasLen, 4)
 472         s.checkMetaEquals(c, hdr, expectCollectionTags)
 473         c.Check(hdr["Inject"], check.IsNil)
 474
 475         c.Log("HEAD bucket with metadata from collection")
 476         resp, err = stage.collbucket.Head("/", nil)
 477         c.Assert(err, check.IsNil)
 478         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 479
 480         c.Log("HEAD directory placeholder with metadata from collection")
 481         resp, err = stage.projbucket.Head("keep-web s3 test collection/", nil)
 482         c.Assert(err, check.IsNil)
 483         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 484
 485         c.Log("HEAD file with metadata from collection")
 486         resp, err = stage.projbucket.Head("keep-web s3 test collection/sailboat.txt", nil)
 487         c.Assert(err, check.IsNil)
 488         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 489
 490         c.Log("HEAD directory placeholder with metadata from subproject")
 491         resp, err = stage.projbucket.Head("keep-web s3 test subproject/", nil)
 492         c.Assert(err, check.IsNil)
 493         s.checkMetaEquals(c, resp.Header, expectSubprojectTags)
 494
 495         c.Log("HEAD bucket with metadata from project")
 496         resp, err = stage.projbucket.Head("/", nil)
 497         c.Assert(err, check.IsNil)
 498         s.checkMetaEquals(c, resp.Header, expectProjectTags)
 499 }
 500
 501 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
 502         stage := s.s3setup(c)
 503         defer stage.teardown(c)
 504         s.testS3PutObjectSuccess(c, stage.collbucket, "", stage.coll.UUID)
 505 }
 506 func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) {
 507         stage := s.s3setup(c)
 508         defer stage.teardown(c)
 509         s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/", stage.coll.UUID)
 510 }
 511 func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string, collUUID string) {
 512         // We insert a delay between test cases to ensure we exercise
 513         // rollover of expired sessions.
 514         sleep := time.Second / 100
 515         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(sleep * 3)
 516
 517         for _, trial := range []struct {
 518                 path        string
 519                 size        int
 520                 contentType string
 521         }{
 522                 {
 523                         path:        "newfile",
 524                         size:        128000000,
 525                         contentType: "application/octet-stream",
 526                 }, {
 527                         path:        "newdir/newfile",
 528                         size:        1 << 26,
 529                         contentType: "application/octet-stream",
 530                 }, {
 531                         path:        "ccc/",
 532                         size:        0,
 533                         contentType: "application/x-directory",
 534                 }, {
 535                         path:        "newdir1/newdir2/newfile",
 536                         size:        0,
 537                         contentType: "application/octet-stream",
 538                 }, {
 539                         path:        "newdir1/newdir2/newdir3/",
 540                         size:        0,
 541                         contentType: "application/x-directory",
 542                 },
 543         } {
 544                 time.Sleep(sleep)
 545                 c.Logf("=== %v", trial)
 546
 547                 objname := prefix + trial.path
 548
 549                 _, err := bucket.GetReader(objname)
 550                 if !c.Check(err, check.NotNil) {
 551                         continue
 552                 }
 553                 c.Check(err.(*s3.Error).StatusCode, check.Equals, http.StatusNotFound)
 554                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 555                 if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
 556                         continue
 557                 }
 558
 559                 buf := make([]byte, trial.size)
 560                 rand.Read(buf)
 561
 562                 err = bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 563                 c.Check(err, check.IsNil)
 564
 565                 rdr, err := bucket.GetReader(objname)
 566                 if strings.HasSuffix(trial.path, "/") && !s.handler.Cluster.Collections.S3FolderObjects {
 567                         c.Check(err, check.NotNil)
 568                         continue
 569                 } else if !c.Check(err, check.IsNil) {
 570                         continue
 571                 }
 572                 buf2, err := ioutil.ReadAll(rdr)
 573                 c.Check(err, check.IsNil)
 574                 c.Check(buf2, check.HasLen, len(buf))
 575                 c.Check(bytes.Equal(buf, buf2), check.Equals, true)
 576
 577                 // Check that the change is immediately visible via
 578                 // (non-S3) webdav request.
 579                 _, resp := s.do("GET", "http://"+collUUID+".keep-web.example/"+trial.path, arvadostest.ActiveTokenV2, nil, nil)
 580                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 581                 if !strings.HasSuffix(trial.path, "/") {
 582                         buf, _ := io.ReadAll(resp.Body)
 583                         c.Check(len(buf), check.Equals, trial.size)
 584                 }
 585         }
 586 }
 587
 588 func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
 589         stage := s.s3setup(c)
 590         defer stage.teardown(c)
 591         bucket := stage.projbucket
 592
 593         for _, trial := range []struct {
 594                 path         string
 595                 size         int
 596                 contentType  string
 597                 errorMatches string
 598         }{
 599                 {
 600                         path:         "newfile",
 601                         size:         1234,
 602                         contentType:  "application/octet-stream",
 603                         errorMatches: `invalid argument: path is not in a collection`,
 604                 }, {
 605                         path:         "newdir/newfile",
 606                         size:         1234,
 607                         contentType:  "application/octet-stream",
 608                         errorMatches: `invalid argument: path is not in a collection`,
 609                 }, {
 610                         path:         "newdir2/",
 611                         size:         0,
 612                         contentType:  "application/x-directory",
 613                         errorMatches: `mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2" failed: invalid operation`,
 614                 },
 615         } {
 616                 c.Logf("=== %v", trial)
 617
 618                 _, err := bucket.GetReader(trial.path)
 619                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 620                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 621                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 622
 623                 buf := make([]byte, trial.size)
 624                 rand.Read(buf)
 625
 626                 err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 627                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
 628                 c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
 629                 c.Check(err, check.ErrorMatches, trial.errorMatches)
 630
 631                 _, err = bucket.GetReader(trial.path)
 632                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 633                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 634                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 635         }
 636 }
 637
 638 func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) {
 639         stage := s.s3setup(c)
 640         defer stage.teardown(c)
 641         s.testS3DeleteObject(c, stage.collbucket, "")
 642 }
 643 func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) {
 644         stage := s.s3setup(c)
 645         defer stage.teardown(c)
 646         s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
 647 }
 648 func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
 649         s.handler.Cluster.Collections.S3FolderObjects = true
 650         for _, trial := range []struct {
 651                 path string
 652         }{
 653                 {"/"},
 654                 {"nonexistentfile"},
 655                 {"emptyfile"},
 656                 {"sailboat.txt"},
 657                 {"sailboat.txt/"},
 658                 {"emptydir"},
 659                 {"emptydir/"},
 660         } {
 661                 objname := prefix + trial.path
 662                 comment := check.Commentf("objname %q", objname)
 663
 664                 err := bucket.Del(objname)
 665                 if trial.path == "/" {
 666                         c.Check(err, check.NotNil)
 667                         continue
 668                 }
 669                 c.Check(err, check.IsNil, comment)
 670                 _, err = bucket.GetReader(objname)
 671                 c.Check(err, check.NotNil, comment)
 672         }
 673 }
 674
 675 func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) {
 676         stage := s.s3setup(c)
 677         defer stage.teardown(c)
 678         s.testS3PutObjectFailure(c, stage.collbucket, "")
 679 }
 680 func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
 681         stage := s.s3setup(c)
 682         defer stage.teardown(c)
 683         s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
 684 }
 685 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
 686         s.handler.Cluster.Collections.S3FolderObjects = false
 687
 688         var wg sync.WaitGroup
 689         for _, trial := range []struct {
 690                 path string
 691         }{
 692                 {
 693                         path: "emptyfile/newname", // emptyfile exists, see s3setup()
 694                 }, {
 695                         path: "emptyfile/", // emptyfile exists, see s3setup()
 696                 }, {
 697                         path: "emptydir", // dir already exists, see s3setup()
 698                 }, {
 699                         path: "emptydir/",
 700                 }, {
 701                         path: "newdir/",
 702                 }, {
 703                         path: "",
 704                 },
 705         } {
 706                 trial := trial
 707                 wg.Add(1)
 708                 go func() {
 709                         defer wg.Done()
 710                         c.Logf("=== %v", trial)
 711
 712                         objname := prefix + trial.path
 713
 714                         buf := make([]byte, 1234)
 715                         rand.Read(buf)
 716
 717                         err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
 718                         if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing (directory|object)|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
 719                                 return
 720                         }
 721
 722                         if objname != "" && objname != "/" {
 723                                 _, err = bucket.GetReader(objname)
 724                                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 725                                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 726                                 c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
 727                         }
 728                 }()
 729         }
 730         wg.Wait()
 731 }
 732
 733 func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
 734         fs, err := stage.coll.FileSystem(stage.arv, stage.kc)
 735         c.Assert(err, check.IsNil)
 736         for d := 0; d < dirs; d++ {
 737                 dir := fmt.Sprintf("dir%d", d)
 738                 c.Assert(fs.Mkdir(dir, 0755), check.IsNil)
 739                 for i := 0; i < filesPerDir; i++ {
 740                         f, err := fs.OpenFile(fmt.Sprintf("%s/file%d.txt", dir, i), os.O_CREATE|os.O_WRONLY, 0644)
 741                         c.Assert(err, check.IsNil)
 742                         c.Assert(f.Close(), check.IsNil)
 743                 }
 744         }
 745         c.Assert(fs.Sync(), check.IsNil)
 746 }
 747
 748 func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
 749         scope := "20200202/zzzzz/service/aws4_request"
 750         signedHeaders := "date"
 751         req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
 752         stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
 753         c.Assert(err, check.IsNil)
 754         sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
 755         c.Assert(err, check.IsNil)
 756         req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
 757 }
 758
 759 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
 760         stage := s.s3setup(c)
 761         defer stage.teardown(c)
 762         for _, trial := range []struct {
 763                 url            string
 764                 method         string
 765                 body           string
 766                 responseCode   int
 767                 responseRegexp []string
 768                 checkEtag      bool
 769         }{
 770                 {
 771                         url:            "https://" + stage.collbucket.Name + ".example.com/",
 772                         method:         "GET",
 773                         responseCode:   http.StatusOK,
 774                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 775                 },
 776                 {
 777                         url:            "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
 778                         method:         "GET",
 779                         responseCode:   http.StatusOK,
 780                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 781                 },
 782                 {
 783                         url:            "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
 784                         method:         "GET",
 785                         responseCode:   http.StatusOK,
 786                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 787                 },
 788                 {
 789                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
 790                         method:         "GET",
 791                         responseCode:   http.StatusOK,
 792                         responseRegexp: []string{`⛵\n`},
 793                         checkEtag:      true,
 794                 },
 795                 {
 796                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 797                         method:       "PUT",
 798                         body:         "boop",
 799                         responseCode: http.StatusOK,
 800                 },
 801                 {
 802                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 803                         method:         "GET",
 804                         responseCode:   http.StatusOK,
 805                         responseRegexp: []string{`boop`},
 806                         checkEtag:      true,
 807                 },
 808                 {
 809                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 810                         method:       "GET",
 811                         responseCode: http.StatusNotFound,
 812                 },
 813                 {
 814                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 815                         method:       "PUT",
 816                         body:         "boop",
 817                         responseCode: http.StatusOK,
 818                 },
 819                 {
 820                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 821                         method:         "GET",
 822                         responseCode:   http.StatusOK,
 823                         responseRegexp: []string{`boop`},
 824                         checkEtag:      true,
 825                 },
 826         } {
 827                 c.Logf("=== %s %s", trial.method, trial.url)
 828                 url, err := url.Parse(trial.url)
 829                 c.Assert(err, check.IsNil)
 830                 req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
 831                 c.Assert(err, check.IsNil)
 832                 s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
 833                 rr := httptest.NewRecorder()
 834                 s.handler.ServeHTTP(rr, req)
 835                 resp := rr.Result()
 836                 c.Check(resp.StatusCode, check.Equals, trial.responseCode)
 837                 body, err := ioutil.ReadAll(resp.Body)
 838                 c.Assert(err, check.IsNil)
 839                 for _, re := range trial.responseRegexp {
 840                         c.Check(string(body), check.Matches, re)
 841                 }
 842                 if trial.checkEtag {
 843                         c.Check(resp.Header.Get("Etag"), check.Matches, `"[\da-f]{32}\+\d+"`)
 844                 }
 845         }
 846 }
 847
 848 func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
 849         stage := s.s3setup(c)
 850         defer stage.teardown(c)
 851         for _, trial := range []struct {
 852                 rawPath        string
 853                 normalizedPath string
 854         }{
 855                 {"/foo", "/foo"},                           // boring case
 856                 {"/foo%5fbar", "/foo_bar"},                 // _ must not be escaped
 857                 {"/foo%2fbar", "/foo/bar"},                 // / must not be escaped
 858                 {"/(foo)/[];,", "/%28foo%29/%5B%5D%3B%2C"}, // ()[];, must be escaped
 859                 {"/foo%5bbar", "/foo%5Bbar"},               // %XX must be uppercase
 860                 // unicode chars must be UTF-8 encoded and escaped
 861                 {"/\u26f5", "/%E2%9B%B5"},
 862                 // "//" and "///" must not be squashed -- see example,
 863                 // https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
 864                 {"//foo///.bar", "//foo///.bar"},
 865         } {
 866                 c.Logf("trial %q", trial)
 867
 868                 date := time.Now().UTC().Format("20060102T150405Z")
 869                 scope := "20200202/zzzzz/S3/aws4_request"
 870                 canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
 871                 c.Logf("canonicalRequest %q", canonicalRequest)
 872                 expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
 873                 c.Logf("expected stringToSign %q", expect)
 874
 875                 req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
 876                 req.Header.Set("X-Amz-Date", date)
 877                 req.Host = "host.example.com"
 878                 c.Assert(err, check.IsNil)
 879
 880                 obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
 881                 if !c.Check(err, check.IsNil) {
 882                         continue
 883                 }
 884                 c.Check(obtained, check.Equals, expect)
 885         }
 886 }
 887
 888 func (s *IntegrationSuite) TestS3GetBucketLocation(c *check.C) {
 889         stage := s.s3setup(c)
 890         defer stage.teardown(c)
 891         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 892                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 893                 c.Check(err, check.IsNil)
 894                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 895                 req.URL.RawQuery = "location"
 896                 resp, err := http.DefaultClient.Do(req)
 897                 c.Assert(err, check.IsNil)
 898                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 899                 buf, err := ioutil.ReadAll(resp.Body)
 900                 c.Assert(err, check.IsNil)
 901                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<LocationConstraint><LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">zzzzz</LocationConstraint></LocationConstraint>\n")
 902         }
 903 }
 904
 905 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
 906         stage := s.s3setup(c)
 907         defer stage.teardown(c)
 908         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 909                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 910                 c.Check(err, check.IsNil)
 911                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 912                 req.URL.RawQuery = "versioning"
 913                 resp, err := http.DefaultClient.Do(req)
 914                 c.Assert(err, check.IsNil)
 915                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 916                 buf, err := ioutil.ReadAll(resp.Body)
 917                 c.Assert(err, check.IsNil)
 918                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>\n")
 919         }
 920 }
 921
 922 func (s *IntegrationSuite) TestS3UnsupportedAPIs(c *check.C) {
 923         stage := s.s3setup(c)
 924         defer stage.teardown(c)
 925         for _, trial := range []struct {
 926                 method   string
 927                 path     string
 928                 rawquery string
 929         }{
 930                 {"GET", "/", "acl&versionId=1234"},    // GetBucketAcl
 931                 {"GET", "/foo", "acl&versionId=1234"}, // GetObjectAcl
 932                 {"PUT", "/", "acl"},                   // PutBucketAcl
 933                 {"PUT", "/foo", "acl"},                // PutObjectAcl
 934                 {"DELETE", "/", "tagging"},            // DeleteBucketTagging
 935                 {"DELETE", "/foo", "tagging"},         // DeleteObjectTagging
 936         } {
 937                 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 938                         c.Logf("trial %v bucket %v", trial, bucket)
 939                         req, err := http.NewRequest(trial.method, bucket.URL(trial.path), nil)
 940                         c.Check(err, check.IsNil)
 941                         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 942                         req.URL.RawQuery = trial.rawquery
 943                         resp, err := http.DefaultClient.Do(req)
 944                         c.Assert(err, check.IsNil)
 945                         c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 946                         buf, err := ioutil.ReadAll(resp.Body)
 947                         c.Assert(err, check.IsNil)
 948                         c.Check(string(buf), check.Matches, "(?ms).*InvalidRequest.*API not supported.*")
 949                 }
 950         }
 951 }
 952
 953 // If there are no CommonPrefixes entries, the CommonPrefixes XML tag
 954 // should not appear at all.
 955 func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
 956         stage := s.s3setup(c)
 957         defer stage.teardown(c)
 958
 959         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 960         c.Assert(err, check.IsNil)
 961         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 962         req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
 963         resp, err := http.DefaultClient.Do(req)
 964         c.Assert(err, check.IsNil)
 965         buf, err := ioutil.ReadAll(resp.Body)
 966         c.Assert(err, check.IsNil)
 967         c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
 968 }
 969
 970 // If there is no delimiter in the request, or the results are not
 971 // truncated, the NextMarker XML tag should not appear in the response
 972 // body.
 973 func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
 974         stage := s.s3setup(c)
 975         defer stage.teardown(c)
 976
 977         for _, query := range []string{"prefix=e&delimiter=/", ""} {
 978                 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 979                 c.Assert(err, check.IsNil)
 980                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 981                 req.URL.RawQuery = query
 982                 resp, err := http.DefaultClient.Do(req)
 983                 c.Assert(err, check.IsNil)
 984                 buf, err := ioutil.ReadAll(resp.Body)
 985                 c.Assert(err, check.IsNil)
 986                 c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`)
 987         }
 988 }
 989
 990 // List response should include KeyCount field.
 991 func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
 992         stage := s.s3setup(c)
 993         defer stage.teardown(c)
 994
 995         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 996         c.Assert(err, check.IsNil)
 997         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 998         req.URL.RawQuery = "prefix=&delimiter=/"
 999         resp, err := http.DefaultClient.Do(req)
1000         c.Assert(err, check.IsNil)
1001         buf, err := ioutil.ReadAll(resp.Body)
1002         c.Assert(err, check.IsNil)
1003         c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
1004 }
1005
1006 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
1007         stage := s.s3setup(c)
1008         defer stage.teardown(c)
1009
1010         var markers int
1011         for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
1012                 dirs := 2000
1013                 filesPerDir := 2
1014                 stage.writeBigDirs(c, dirs, filesPerDir)
1015                 // Total # objects is:
1016                 //                 2 file entries from s3setup (emptyfile and sailboat.txt)
1017                 //                +1 fake "directory" marker from s3setup (emptydir) (if enabled)
1018                 //             +dirs fake "directory" marker from writeBigDirs (dir0/, dir1/) (if enabled)
1019                 // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
1020                 s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
1021                 s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
1022                 s.testS3List(c, stage.collbucket, "", 51, markers+2+(filesPerDir+markers)*dirs)
1023                 s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
1024         }
1025 }
1026 func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
1027         c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.handler.Cluster.Collections.S3FolderObjects)
1028         expectPageSize := pageSize
1029         if expectPageSize > 1000 {
1030                 expectPageSize = 1000
1031         }
1032         gotKeys := map[string]s3.Key{}
1033         nextMarker := ""
1034         pages := 0
1035         for {
1036                 resp, err := bucket.List(prefix, "", nextMarker, pageSize)
1037                 if !c.Check(err, check.IsNil) {
1038                         break
1039                 }
1040                 c.Check(len(resp.Contents) <= expectPageSize, check.Equals, true)
1041                 if pages++; !c.Check(pages <= (expectFiles/expectPageSize)+1, check.Equals, true) {
1042                         break
1043                 }
1044                 for _, key := range resp.Contents {
1045                         if _, dup := gotKeys[key.Key]; dup {
1046                                 c.Errorf("got duplicate key %q on page %d", key.Key, pages)
1047                         }
1048                         gotKeys[key.Key] = key
1049                         if strings.Contains(key.Key, "sailboat.txt") {
1050                                 c.Check(key.Size, check.Equals, int64(4))
1051                         }
1052                 }
1053                 if !resp.IsTruncated {
1054                         c.Check(resp.NextMarker, check.Equals, "")
1055                         break
1056                 }
1057                 if !c.Check(resp.NextMarker, check.Not(check.Equals), "") {
1058                         break
1059                 }
1060                 nextMarker = resp.NextMarker
1061         }
1062         if !c.Check(len(gotKeys), check.Equals, expectFiles) {
1063                 var sorted []string
1064                 for k := range gotKeys {
1065                         sorted = append(sorted, k)
1066                 }
1067                 sort.Strings(sorted)
1068                 for _, k := range sorted {
1069                         c.Logf("got %s", k)
1070                 }
1071         }
1072 }
1073
1074 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
1075         for _, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
1076                 s.testS3CollectionListRollup(c)
1077         }
1078 }
1079
1080 func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
1081         stage := s.s3setup(c)
1082         defer stage.teardown(c)
1083
1084         dirs := 2
1085         filesPerDir := 500
1086         stage.writeBigDirs(c, dirs, filesPerDir)
1087         err := stage.collbucket.PutReader("dingbats", &bytes.Buffer{}, 0, "application/octet-stream", s3.Private, s3.Options{})
1088         c.Assert(err, check.IsNil)
1089         var allfiles []string
1090         for marker := ""; ; {
1091                 resp, err := stage.collbucket.List("", "", marker, 20000)
1092                 c.Check(err, check.IsNil)
1093                 for _, key := range resp.Contents {
1094                         if len(allfiles) == 0 || allfiles[len(allfiles)-1] != key.Key {
1095                                 allfiles = append(allfiles, key.Key)
1096                         }
1097                 }
1098                 marker = resp.NextMarker
1099                 if marker == "" {
1100                         break
1101                 }
1102         }
1103         markers := 0
1104         if s.handler.Cluster.Collections.S3FolderObjects {
1105                 markers = 1
1106         }
1107         c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
1108
1109         gotDirMarker := map[string]bool{}
1110         for _, name := range allfiles {
1111                 isDirMarker := strings.HasSuffix(name, "/")
1112                 if markers == 0 {
1113                         c.Check(isDirMarker, check.Equals, false, check.Commentf("name %q", name))
1114                 } else if isDirMarker {
1115                         gotDirMarker[name] = true
1116                 } else if i := strings.LastIndex(name, "/"); i >= 0 {
1117                         c.Check(gotDirMarker[name[:i+1]], check.Equals, true, check.Commentf("name %q", name))
1118                         gotDirMarker[name[:i+1]] = true // skip redundant complaints about this dir marker
1119                 }
1120         }
1121
1122         for _, trial := range []struct {
1123                 prefix    string
1124                 delimiter string
1125                 marker    string
1126         }{
1127                 {"", "", ""},
1128                 {"di", "/", ""},
1129                 {"di", "r", ""},
1130                 {"di", "n", ""},
1131                 {"dir0", "/", ""},
1132                 {"dir0/", "/", ""},
1133                 {"dir0/f", "/", ""},
1134                 {"dir0", "", ""},
1135                 {"dir0/", "", ""},
1136                 {"dir0/f", "", ""},
1137                 {"dir0", "/", "dir0/file14.txt"},       // one commonprefix, "dir0/"
1138                 {"dir0", "/", "dir0/zzzzfile.txt"},     // no commonprefixes
1139                 {"", "", "dir0/file14.txt"},            // middle page, skip walking dir1
1140                 {"", "", "dir1/file14.txt"},            // middle page, skip walking dir0
1141                 {"", "", "dir1/file498.txt"},           // last page of results
1142                 {"dir1/file", "", "dir1/file498.txt"},  // last page of results, with prefix
1143                 {"dir1/file", "/", "dir1/file498.txt"}, // last page of results, with prefix + delimiter
1144                 {"dir1", "Z", "dir1/file498.txt"},      // delimiter "Z" never appears
1145                 {"dir2", "/", ""},                      // prefix "dir2" does not exist
1146                 {"", "/", ""},
1147         } {
1148                 c.Logf("\n\n=== trial %+v markers=%d", trial, markers)
1149
1150                 maxKeys := 20
1151                 resp, err := stage.collbucket.List(trial.prefix, trial.delimiter, trial.marker, maxKeys)
1152                 c.Check(err, check.IsNil)
1153                 if resp.IsTruncated && trial.delimiter == "" {
1154                         // goamz List method fills in the missing
1155                         // NextMarker field if resp.IsTruncated, so
1156                         // now we can't really tell whether it was
1157                         // sent by the server or by goamz. In cases
1158                         // where it should be empty but isn't, assume
1159                         // it's goamz's fault.
1160                         resp.NextMarker = ""
1161                 }
1162
1163                 var expectKeys []string
1164                 var expectPrefixes []string
1165                 var expectNextMarker string
1166                 var expectTruncated bool
1167                 for _, key := range allfiles {
1168                         full := len(expectKeys)+len(expectPrefixes) >= maxKeys
1169                         if !strings.HasPrefix(key, trial.prefix) || key <= trial.marker {
1170                                 continue
1171                         } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
1172                                 prefix := key[:len(trial.prefix)+idx+1]
1173                                 if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
1174                                         // same prefix as previous key
1175                                 } else if full {
1176                                         expectTruncated = true
1177                                 } else {
1178                                         expectPrefixes = append(expectPrefixes, prefix)
1179                                         expectNextMarker = prefix
1180                                 }
1181                         } else if full {
1182                                 expectTruncated = true
1183                                 break
1184                         } else {
1185                                 expectKeys = append(expectKeys, key)
1186                                 if trial.delimiter != "" {
1187                                         expectNextMarker = key
1188                                 }
1189                         }
1190                 }
1191                 if !expectTruncated {
1192                         expectNextMarker = ""
1193                 }
1194
1195                 var gotKeys []string
1196                 for _, key := range resp.Contents {
1197                         gotKeys = append(gotKeys, key.Key)
1198                 }
1199                 var gotPrefixes []string
1200                 for _, prefix := range resp.CommonPrefixes {
1201                         gotPrefixes = append(gotPrefixes, prefix)
1202                 }
1203                 commentf := check.Commentf("trial %+v markers=%d", trial, markers)
1204                 c.Check(gotKeys, check.DeepEquals, expectKeys, commentf)
1205                 c.Check(gotPrefixes, check.DeepEquals, expectPrefixes, commentf)
1206                 c.Check(resp.NextMarker, check.Equals, expectNextMarker, commentf)
1207                 c.Check(resp.IsTruncated, check.Equals, expectTruncated, commentf)
1208                 c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
1209         }
1210 }
1211
1212 func (s *IntegrationSuite) TestS3ListObjectsV2ManySubprojects(c *check.C) {
1213         stage := s.s3setup(c)
1214         defer stage.teardown(c)
1215         projects := 50
1216         collectionsPerProject := 2
1217         for i := 0; i < projects; i++ {
1218                 var subproj arvados.Group
1219                 err := stage.arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
1220                         "group": map[string]interface{}{
1221                                 "owner_uuid":  stage.subproj.UUID,
1222                                 "group_class": "project",
1223                                 "name":        fmt.Sprintf("keep-web s3 test subproject %d", i),
1224                         },
1225                 })
1226                 c.Assert(err, check.IsNil)
1227                 for j := 0; j < collectionsPerProject; j++ {
1228                         err = stage.arv.RequestAndDecode(nil, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
1229                                 "owner_uuid":    subproj.UUID,
1230                                 "name":          fmt.Sprintf("keep-web s3 test collection %d", j),
1231                                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
1232                         }})
1233                         c.Assert(err, check.IsNil)
1234                 }
1235         }
1236         c.Logf("setup complete")
1237
1238         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1239                 Region:           aws_aws.String("auto"),
1240                 Endpoint:         aws_aws.String(s.testServer.URL),
1241                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1242                 S3ForcePathStyle: aws_aws.Bool(true),
1243         }))
1244         client := aws_s3.New(sess)
1245         ctx := context.Background()
1246         params := aws_s3.ListObjectsV2Input{
1247                 Bucket:    aws_aws.String(stage.proj.UUID),
1248                 Delimiter: aws_aws.String("/"),
1249                 Prefix:    aws_aws.String("keep-web s3 test subproject/"),
1250                 MaxKeys:   aws_aws.Int64(int64(projects / 2)),
1251         }
1252         for page := 1; ; page++ {
1253                 t0 := time.Now()
1254                 result, err := client.ListObjectsV2WithContext(ctx, &params)
1255                 if !c.Check(err, check.IsNil) {
1256                         break
1257                 }
1258                 c.Logf("got page %d in %v with len(Contents) == %d, len(CommonPrefixes) == %d", page, time.Since(t0), len(result.Contents), len(result.CommonPrefixes))
1259                 if !*result.IsTruncated {
1260                         break
1261                 }
1262                 params.ContinuationToken = result.NextContinuationToken
1263                 *params.MaxKeys = *params.MaxKeys/2 + 1
1264         }
1265 }
1266
1267 func (s *IntegrationSuite) TestS3ListObjectsV2(c *check.C) {
1268         stage := s.s3setup(c)
1269         defer stage.teardown(c)
1270         dirs := 2
1271         filesPerDir := 40
1272         stage.writeBigDirs(c, dirs, filesPerDir)
1273
1274         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1275                 Region:           aws_aws.String("auto"),
1276                 Endpoint:         aws_aws.String(s.testServer.URL),
1277                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1278                 S3ForcePathStyle: aws_aws.Bool(true),
1279         }))
1280
1281         stringOrNil := func(s string) *string {
1282                 if s == "" {
1283                         return nil
1284                 } else {
1285                         return &s
1286                 }
1287         }
1288
1289         client := aws_s3.New(sess)
1290         ctx := context.Background()
1291
1292         for _, trial := range []struct {
1293                 prefix               string
1294                 delimiter            string
1295                 startAfter           string
1296                 maxKeys              int
1297                 expectKeys           int
1298                 expectCommonPrefixes map[string]bool
1299         }{
1300                 {
1301                         // Expect {filesPerDir plus the dir itself}
1302                         // for each dir, plus emptydir, emptyfile, and
1303                         // sailboat.txt.
1304                         expectKeys: (filesPerDir+1)*dirs + 3,
1305                 },
1306                 {
1307                         maxKeys:    15,
1308                         expectKeys: (filesPerDir+1)*dirs + 3,
1309                 },
1310                 {
1311                         startAfter: "dir0/z",
1312                         maxKeys:    15,
1313                         // Expect {filesPerDir plus the dir itself}
1314                         // for each dir except dir0, plus emptydir,
1315                         // emptyfile, and sailboat.txt.
1316                         expectKeys: (filesPerDir+1)*(dirs-1) + 3,
1317                 },
1318                 {
1319                         maxKeys:              1,
1320                         delimiter:            "/",
1321                         expectKeys:           2, // emptyfile, sailboat.txt
1322                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1323                 },
1324                 {
1325                         startAfter:           "dir0/z",
1326                         maxKeys:              15,
1327                         delimiter:            "/",
1328                         expectKeys:           2, // emptyfile, sailboat.txt
1329                         expectCommonPrefixes: map[string]bool{"dir1/": true, "emptydir/": true},
1330                 },
1331                 {
1332                         startAfter:           "dir0/file10.txt",
1333                         maxKeys:              15,
1334                         delimiter:            "/",
1335                         expectKeys:           2,
1336                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1337                 },
1338                 {
1339                         startAfter:           "dir0/file10.txt",
1340                         maxKeys:              15,
1341                         prefix:               "d",
1342                         delimiter:            "/",
1343                         expectKeys:           0,
1344                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true},
1345                 },
1346         } {
1347                 c.Logf("[trial %+v]", trial)
1348                 params := aws_s3.ListObjectsV2Input{
1349                         Bucket:     aws_aws.String(stage.collbucket.Name),
1350                         Prefix:     stringOrNil(trial.prefix),
1351                         Delimiter:  stringOrNil(trial.delimiter),
1352                         StartAfter: stringOrNil(trial.startAfter),
1353                         MaxKeys:    aws_aws.Int64(int64(trial.maxKeys)),
1354                 }
1355                 keySeen := map[string]bool{}
1356                 prefixSeen := map[string]bool{}
1357                 for {
1358                         result, err := client.ListObjectsV2WithContext(ctx, &params)
1359                         if !c.Check(err, check.IsNil) {
1360                                 break
1361                         }
1362                         c.Check(result.Name, check.DeepEquals, aws_aws.String(stage.collbucket.Name))
1363                         c.Check(result.Prefix, check.DeepEquals, aws_aws.String(trial.prefix))
1364                         c.Check(result.Delimiter, check.DeepEquals, aws_aws.String(trial.delimiter))
1365                         // The following two fields are expected to be
1366                         // nil (i.e., no tag in XML response) rather
1367                         // than "" when the corresponding request
1368                         // field was empty or nil.
1369                         c.Check(result.StartAfter, check.DeepEquals, stringOrNil(trial.startAfter))
1370                         c.Check(result.ContinuationToken, check.DeepEquals, params.ContinuationToken)
1371
1372                         if trial.maxKeys > 0 {
1373                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(trial.maxKeys)))
1374                                 c.Check(len(result.Contents)+len(result.CommonPrefixes) <= trial.maxKeys, check.Equals, true)
1375                         } else {
1376                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(s3MaxKeys)))
1377                         }
1378
1379                         for _, ent := range result.Contents {
1380                                 c.Assert(ent.Key, check.NotNil)
1381                                 c.Check(*ent.Key > trial.startAfter, check.Equals, true)
1382                                 c.Check(keySeen[*ent.Key], check.Equals, false, check.Commentf("dup key %q", *ent.Key))
1383                                 keySeen[*ent.Key] = true
1384                         }
1385                         for _, ent := range result.CommonPrefixes {
1386                                 c.Assert(ent.Prefix, check.NotNil)
1387                                 c.Check(strings.HasSuffix(*ent.Prefix, trial.delimiter), check.Equals, true, check.Commentf("bad CommonPrefix %q", *ent.Prefix))
1388                                 if strings.HasPrefix(trial.startAfter, *ent.Prefix) {
1389                                         // If we asked for
1390                                         // startAfter=dir0/file10.txt,
1391                                         // we expect dir0/ to be
1392                                         // returned as a common prefix
1393                                 } else {
1394                                         c.Check(*ent.Prefix > trial.startAfter, check.Equals, true)
1395                                 }
1396                                 c.Check(prefixSeen[*ent.Prefix], check.Equals, false, check.Commentf("dup common prefix %q", *ent.Prefix))
1397                                 prefixSeen[*ent.Prefix] = true
1398                         }
1399                         if *result.IsTruncated && c.Check(result.NextContinuationToken, check.Not(check.Equals), "") {
1400                                 params.ContinuationToken = aws_aws.String(*result.NextContinuationToken)
1401                         } else {
1402                                 break
1403                         }
1404                 }
1405                 c.Check(keySeen, check.HasLen, trial.expectKeys)
1406                 c.Check(prefixSeen, check.HasLen, len(trial.expectCommonPrefixes))
1407                 if len(trial.expectCommonPrefixes) > 0 {
1408                         c.Check(prefixSeen, check.DeepEquals, trial.expectCommonPrefixes)
1409                 }
1410         }
1411 }
1412
1413 func (s *IntegrationSuite) TestS3ListObjectsV2EncodingTypeURL(c *check.C) {
1414         stage := s.s3setup(c)
1415         defer stage.teardown(c)
1416         dirs := 2
1417         filesPerDir := 40
1418         stage.writeBigDirs(c, dirs, filesPerDir)
1419
1420         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1421                 Region:           aws_aws.String("auto"),
1422                 Endpoint:         aws_aws.String(s.testServer.URL),
1423                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1424                 S3ForcePathStyle: aws_aws.Bool(true),
1425         }))
1426
1427         client := aws_s3.New(sess)
1428         ctx := context.Background()
1429
1430         result, err := client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1431                 Bucket:       aws_aws.String(stage.collbucket.Name),
1432                 Prefix:       aws_aws.String("dir0/"),
1433                 Delimiter:    aws_aws.String("/"),
1434                 StartAfter:   aws_aws.String("dir0/"),
1435                 EncodingType: aws_aws.String("url"),
1436         })
1437         c.Assert(err, check.IsNil)
1438         c.Check(*result.Prefix, check.Equals, "dir0%2F")
1439         c.Check(*result.Delimiter, check.Equals, "%2F")
1440         c.Check(*result.StartAfter, check.Equals, "dir0%2F")
1441         for _, ent := range result.Contents {
1442                 c.Check(*ent.Key, check.Matches, "dir0%2F.*")
1443         }
1444         result, err = client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1445                 Bucket:       aws_aws.String(stage.collbucket.Name),
1446                 Delimiter:    aws_aws.String("/"),
1447                 EncodingType: aws_aws.String("url"),
1448         })
1449         c.Assert(err, check.IsNil)
1450         c.Check(*result.Delimiter, check.Equals, "%2F")
1451         c.Check(result.CommonPrefixes, check.HasLen, dirs+1)
1452         for _, ent := range result.CommonPrefixes {
1453                 c.Check(*ent.Prefix, check.Matches, ".*%2F")
1454         }
1455 }
1456
1457 // TestS3cmd checks compatibility with the s3cmd command line tool, if
1458 // it's installed (run-tests normally takes care of that).
1459 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
1460         if _, err := exec.LookPath("s3cmd"); err != nil {
1461                 c.Skip("s3cmd not found")
1462                 return
1463         }
1464
1465         stage := s.s3setup(c)
1466         defer stage.teardown(c)
1467
1468         cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
1469         buf, err := cmd.CombinedOutput()
1470         c.Check(err, check.IsNil)
1471         c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
1472
1473         // This tests whether s3cmd's path normalization agrees with
1474         // keep-web's signature verification wrt chars like "|"
1475         // (neither reserved nor unreserved) and "," (not normally
1476         // percent-encoded in a path).
1477         tmpfile := c.MkDir() + "/dstfile"
1478         cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar", tmpfile)
1479         buf, err = cmd.CombinedOutput()
1480         c.Check(err, check.NotNil)
1481         // As of commit b7520e5c25e1bf25c1a8bf5aa2eadb299be8f606
1482         // (between debian bullseye and bookworm versions), s3cmd
1483         // started catching the NoSuchKey error code and replacing it
1484         // with "Source object '%s' does not exist.".
1485         c.Check(string(buf), check.Matches, `(?ms).*(NoSuchKey|Source object.*does not exist).*\n`)
1486
1487         tmpfile = c.MkDir() + "/foo"
1488         cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo", tmpfile)
1489         buf, err = cmd.CombinedOutput()
1490         c.Logf("%s", buf)
1491         if c.Check(err, check.IsNil) {
1492                 checkcontent, err := os.ReadFile(tmpfile)
1493                 c.Check(err, check.IsNil)
1494                 c.Check(string(checkcontent), check.Equals, "foo")
1495         }
1496 }
1497
1498 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
1499         stage := s.s3setup(c)
1500         defer stage.teardown(c)
1501
1502         hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
1503         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
1504         c.Check(body, check.Equals, "⛵\n")
1505 }
1506
1507 func (s *IntegrationSuite) TestS3ConcurrentPUT(c *check.C) {
1508         stage := s.s3setup(c)
1509         defer stage.teardown(c)
1510         ndirs, nfiles := 5, 5
1511         var wg sync.WaitGroup
1512         for di := 0; di < ndirs; di++ {
1513                 di := di
1514                 wg.Add(1)
1515                 go func() {
1516                         defer wg.Done()
1517                         for fi := 0; fi < nfiles; fi++ {
1518                                 fi := fi
1519                                 wg.Add(1)
1520                                 go func() {
1521                                         defer wg.Done()
1522                                         s.checkPut(c, stage, fmt.Sprintf("dir%d/file%d", di, fi), []byte("acbde"))
1523                                 }()
1524                         }
1525                 }()
1526         }
1527         wg.Wait()
1528         for di := 0; di < ndirs; di++ {
1529                 for fi := 0; fi < nfiles; fi++ {
1530                         s.checkGet(c, stage, fmt.Sprintf("dir%d/file%d", di, fi), 5)
1531                 }
1532         }
1533         if c.Failed() {
1534                 c.Log("preserved files:")
1535                 var saved arvados.Collection
1536                 err := stage.arv.RequestAndDecode(&saved, "GET", "arvados/v1/collections/"+stage.coll.UUID, nil, arvados.GetOptions{
1537                         Select: []string{"uuid", "manifest_text"}})
1538                 c.Assert(err, check.IsNil)
1539                 cfs, err := saved.FileSystem(stage.arv, stage.kc)
1540                 c.Assert(err, check.IsNil)
1541                 fs.WalkDir(arvados.FS(cfs), "", func(path string, _ fs.DirEntry, _ error) error {
1542                         c.Logf("%s", path)
1543                         return nil
1544                 })
1545         }
1546 }
1547
1548 func (s *IntegrationSuite) checkPut(c *check.C, stage s3stage, path string, data []byte) {
1549         url, err := url.Parse("https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/" + path)
1550         c.Assert(err, check.IsNil)
1551         req, err := http.NewRequest(http.MethodPut, url.String(), bytes.NewReader([]byte(data)))
1552         c.Assert(err, check.IsNil)
1553         s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
1554         rr := httptest.NewRecorder()
1555         s.handler.ServeHTTP(rr, req)
1556         resp := rr.Result()
1557         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
1558 }
1559
1560 func (s *IntegrationSuite) checkGet(c *check.C, stage s3stage, path string, expectLength int) {
1561         url, err := url.Parse("https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/" + path)
1562         c.Assert(err, check.IsNil)
1563         req, err := http.NewRequest(http.MethodGet, url.String(), nil)
1564         c.Assert(err, check.IsNil)
1565         s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
1566         rr := httptest.NewRecorder()
1567         s.handler.ServeHTTP(rr, req)
1568         resp := rr.Result()
1569         if !c.Check(resp.StatusCode, check.Equals, http.StatusOK, check.Commentf("%s", path)) {
1570                 return
1571         }
1572         body, err := ioutil.ReadAll(resp.Body)
1573         c.Assert(err, check.IsNil)
1574         c.Check(string(body), check.HasLen, expectLength)
1575 }