services/keep-web/s3_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/rand"
  11         "crypto/sha256"
  12         "fmt"
  13         "io/ioutil"
  14         "mime"
  15         "net/http"
  16         "net/http/httptest"
  17         "net/url"
  18         "os"
  19         "os/exec"
  20         "sort"
  21         "strings"
  22         "sync"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/keepclient"
  29         "github.com/AdRoll/goamz/aws"
  30         "github.com/AdRoll/goamz/s3"
  31         aws_aws "github.com/aws/aws-sdk-go/aws"
  32         aws_credentials "github.com/aws/aws-sdk-go/aws/credentials"
  33         aws_session "github.com/aws/aws-sdk-go/aws/session"
  34         aws_s3 "github.com/aws/aws-sdk-go/service/s3"
  35         check "gopkg.in/check.v1"
  36 )
  37
  38 type CachedS3SecretSuite struct{}
  39
  40 var _ = check.Suite(&CachedS3SecretSuite{})
  41
  42 func (s *CachedS3SecretSuite) activeACA(expiresAt time.Time) *arvados.APIClientAuthorization {
  43         return &arvados.APIClientAuthorization{
  44                 UUID:      arvadostest.ActiveTokenUUID,
  45                 APIToken:  arvadostest.ActiveToken,
  46                 ExpiresAt: expiresAt,
  47         }
  48 }
  49
  50 func (s *CachedS3SecretSuite) TestNewCachedS3SecretExpiresBeforeTTL(c *check.C) {
  51         expected := time.Unix(1<<29, 0)
  52         aca := s.activeACA(expected)
  53         actual := NewCachedS3Secret(aca, time.Unix(1<<30, 0))
  54         c.Check(actual.expiry, check.Equals, expected)
  55 }
  56
  57 func (s *CachedS3SecretSuite) TestNewCachedS3SecretExpiresAfterTTL(c *check.C) {
  58         expected := time.Unix(1<<29, 0)
  59         aca := s.activeACA(time.Unix(1<<30, 0))
  60         actual := NewCachedS3Secret(aca, expected)
  61         c.Check(actual.expiry, check.Equals, expected)
  62 }
  63
  64 func (s *CachedS3SecretSuite) TestNewCachedS3SecretWithoutExpiry(c *check.C) {
  65         expected := time.Unix(1<<29, 0)
  66         aca := s.activeACA(time.Time{})
  67         actual := NewCachedS3Secret(aca, expected)
  68         c.Check(actual.expiry, check.Equals, expected)
  69 }
  70
  71 func (s *CachedS3SecretSuite) cachedSecretWithExpiry(expiry time.Time) *cachedS3Secret {
  72         return &cachedS3Secret{
  73                 auth:   s.activeACA(expiry),
  74                 expiry: expiry,
  75         }
  76 }
  77
  78 func (s *CachedS3SecretSuite) TestIsValidAtEmpty(c *check.C) {
  79         cache := &cachedS3Secret{}
  80         c.Check(cache.IsValidAt(time.Unix(0, 0)), check.Equals, false)
  81         c.Check(cache.IsValidAt(time.Unix(1<<31, 0)), check.Equals, false)
  82 }
  83
  84 func (s *CachedS3SecretSuite) TestIsValidAtNoAuth(c *check.C) {
  85         cache := &cachedS3Secret{expiry: time.Unix(3, 0)}
  86         c.Check(cache.IsValidAt(time.Unix(2, 0)), check.Equals, false)
  87         c.Check(cache.IsValidAt(time.Unix(4, 0)), check.Equals, false)
  88 }
  89
  90 func (s *CachedS3SecretSuite) TestIsValidAtNoExpiry(c *check.C) {
  91         cache := &cachedS3Secret{auth: s.activeACA(time.Unix(3, 0))}
  92         c.Check(cache.IsValidAt(time.Unix(2, 0)), check.Equals, false)
  93         c.Check(cache.IsValidAt(time.Unix(4, 0)), check.Equals, false)
  94 }
  95
  96 func (s *CachedS3SecretSuite) TestIsValidAtTimeAfterExpiry(c *check.C) {
  97         expiry := time.Unix(10, 0)
  98         cache := s.cachedSecretWithExpiry(expiry)
  99         c.Check(cache.IsValidAt(expiry), check.Equals, false)
 100         c.Check(cache.IsValidAt(time.Unix(1<<25, 0)), check.Equals, false)
 101         c.Check(cache.IsValidAt(time.Unix(1<<30, 0)), check.Equals, false)
 102 }
 103
 104 func (s *CachedS3SecretSuite) TestIsValidAtTimeBeforeExpiry(c *check.C) {
 105         cache := s.cachedSecretWithExpiry(time.Unix(1<<30, 0))
 106         c.Check(cache.IsValidAt(time.Unix(1<<25, 0)), check.Equals, true)
 107         c.Check(cache.IsValidAt(time.Unix(1<<27, 0)), check.Equals, true)
 108         c.Check(cache.IsValidAt(time.Unix(1<<29, 0)), check.Equals, true)
 109 }
 110
 111 func (s *CachedS3SecretSuite) TestIsValidAtZeroTime(c *check.C) {
 112         cache := s.cachedSecretWithExpiry(time.Unix(10, 0))
 113         c.Check(cache.IsValidAt(time.Time{}), check.Equals, false)
 114 }
 115
 116 type s3stage struct {
 117         arv        *arvados.Client
 118         ac         *arvadosclient.ArvadosClient
 119         kc         *keepclient.KeepClient
 120         proj       arvados.Group
 121         projbucket *s3.Bucket
 122         subproj    arvados.Group
 123         coll       arvados.Collection
 124         collbucket *s3.Bucket
 125 }
 126
 127 func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
 128         var proj, subproj arvados.Group
 129         var coll arvados.Collection
 130         arv := arvados.NewClientFromEnv()
 131         arv.AuthToken = arvadostest.ActiveToken
 132         err := arv.RequestAndDecode(&proj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
 133                 "group": map[string]interface{}{
 134                         "group_class": "project",
 135                         "name":        "keep-web s3 test",
 136                         "properties": map[string]interface{}{
 137                                 "project-properties-key": "project properties value",
 138                         },
 139                 },
 140                 "ensure_unique_name": true,
 141         })
 142         c.Assert(err, check.IsNil)
 143         err = arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
 144                 "group": map[string]interface{}{
 145                         "owner_uuid":  proj.UUID,
 146                         "group_class": "project",
 147                         "name":        "keep-web s3 test subproject",
 148                         "properties": map[string]interface{}{
 149                                 "subproject_properties_key": "subproject properties value",
 150                                 "invalid header key":        "this value will not be returned because key contains spaces",
 151                         },
 152                 },
 153         })
 154         c.Assert(err, check.IsNil)
 155         err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
 156                 "owner_uuid":    proj.UUID,
 157                 "name":          "keep-web s3 test collection",
 158                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
 159                 "properties": map[string]interface{}{
 160                         "string":   "string value",
 161                         "array":    []string{"element1", "element2"},
 162                         "object":   map[string]interface{}{"key": map[string]interface{}{"key2": "value⛵"}},
 163                         "nonascii": "⛵",
 164                         "newline":  "foo\r\nX-Bad: header",
 165                         // This key cannot be expressed as a MIME
 166                         // header key, so it will be silently skipped
 167                         // (see "Inject" in PropertiesAsMetadata test)
 168                         "a: a\r\nInject": "bogus",
 169                 },
 170         }})
 171         c.Assert(err, check.IsNil)
 172         ac, err := arvadosclient.New(arv)
 173         c.Assert(err, check.IsNil)
 174         kc, err := keepclient.MakeKeepClient(ac)
 175         c.Assert(err, check.IsNil)
 176         fs, err := coll.FileSystem(arv, kc)
 177         c.Assert(err, check.IsNil)
 178         f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
 179         c.Assert(err, check.IsNil)
 180         _, err = f.Write([]byte("⛵\n"))
 181         c.Assert(err, check.IsNil)
 182         err = f.Close()
 183         c.Assert(err, check.IsNil)
 184         err = fs.Sync()
 185         c.Assert(err, check.IsNil)
 186         err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
 187         c.Assert(err, check.IsNil)
 188
 189         auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
 190         region := aws.Region{
 191                 Name:       "zzzzz",
 192                 S3Endpoint: s.testServer.URL,
 193         }
 194         client := s3.New(*auth, region)
 195         client.Signature = aws.V4Signature
 196         return s3stage{
 197                 arv:  arv,
 198                 ac:   ac,
 199                 kc:   kc,
 200                 proj: proj,
 201                 projbucket: &s3.Bucket{
 202                         S3:   client,
 203                         Name: proj.UUID,
 204                 },
 205                 subproj: subproj,
 206                 coll:    coll,
 207                 collbucket: &s3.Bucket{
 208                         S3:   client,
 209                         Name: coll.UUID,
 210                 },
 211         }
 212 }
 213
 214 func (stage s3stage) teardown(c *check.C) {
 215         if stage.coll.UUID != "" {
 216                 err := stage.arv.RequestAndDecode(&stage.coll, "DELETE", "arvados/v1/collections/"+stage.coll.UUID, nil, nil)
 217                 c.Check(err, check.IsNil)
 218         }
 219         if stage.proj.UUID != "" {
 220                 err := stage.arv.RequestAndDecode(&stage.proj, "DELETE", "arvados/v1/groups/"+stage.proj.UUID, nil, nil)
 221                 c.Check(err, check.IsNil)
 222         }
 223 }
 224
 225 func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
 226         stage := s.s3setup(c)
 227         defer stage.teardown(c)
 228
 229         bucket := stage.collbucket
 230         for _, trial := range []struct {
 231                 success   bool
 232                 signature int
 233                 accesskey string
 234                 secretkey string
 235         }{
 236                 {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
 237                 {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
 238                 {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
 239                 {false, aws.V2Signature, "none", "none"},
 240                 {false, aws.V2Signature, "none", arvadostest.ActiveToken},
 241
 242                 {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
 243                 {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
 244                 {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
 245                 {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
 246                 {false, aws.V4Signature, arvadostest.ActiveToken, ""},
 247                 {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
 248                 {false, aws.V4Signature, "none", arvadostest.ActiveToken},
 249                 {false, aws.V4Signature, "none", "none"},
 250         } {
 251                 c.Logf("%#v", trial)
 252                 bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
 253                 bucket.S3.Signature = trial.signature
 254                 _, err := bucket.GetReader("emptyfile")
 255                 if trial.success {
 256                         c.Check(err, check.IsNil)
 257                 } else {
 258                         c.Check(err, check.NotNil)
 259                 }
 260         }
 261 }
 262
 263 func (s *IntegrationSuite) TestS3SecretCacheUpdates(c *check.C) {
 264         stage := s.s3setup(c)
 265         defer stage.teardown(c)
 266         reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
 267         c.Assert(err, check.IsNil)
 268
 269         for trialName, trialAuth := range map[string]string{
 270                 "v1 token":                    arvadostest.ActiveToken,
 271                 "token UUID":                  arvadostest.ActiveTokenUUID,
 272                 "v2 token query escaped":      url.QueryEscape(arvadostest.ActiveTokenV2),
 273                 "v2 token underscore escaped": strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1),
 274         } {
 275                 s.handler.s3SecretCache = nil
 276                 req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
 277                 if !c.Check(err, check.IsNil) {
 278                         continue
 279                 }
 280                 secret := trialAuth
 281                 if secret[5:12] == "-gj3su-" {
 282                         secret = arvadostest.ActiveToken
 283                 }
 284                 s.sign(c, req, trialAuth, secret)
 285                 rec := httptest.NewRecorder()
 286                 s.handler.ServeHTTP(rec, req)
 287                 if !c.Check(rec.Result().StatusCode, check.Equals, http.StatusOK,
 288                         check.Commentf("%s auth did not get 200 OK response: %v", trialName, req)) {
 289                         continue
 290                 }
 291
 292                 for name, key := range map[string]string{
 293                         "v1 token":   arvadostest.ActiveToken,
 294                         "token UUID": arvadostest.ActiveTokenUUID,
 295                         "v2 token":   arvadostest.ActiveTokenV2,
 296                 } {
 297                         actual, ok := s.handler.s3SecretCache[key]
 298                         if c.Check(ok, check.Equals, true, check.Commentf("%s not cached from %s", name, trialName)) {
 299                                 c.Check(actual.auth.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 300                         }
 301                 }
 302         }
 303 }
 304
 305 func (s *IntegrationSuite) TestS3SecretCacheUsed(c *check.C) {
 306         stage := s.s3setup(c)
 307         defer stage.teardown(c)
 308
 309         token := arvadostest.ActiveToken
 310         // Step 1: Make a request to get the active token in the cache.
 311         reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
 312         c.Assert(err, check.IsNil)
 313         req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
 314         s.sign(c, req, token, token)
 315         rec := httptest.NewRecorder()
 316         s.handler.ServeHTTP(rec, req)
 317         resp := rec.Result()
 318         c.Assert(resp.StatusCode, check.Equals, http.StatusOK,
 319                 check.Commentf("first request did not get 200 OK response"))
 320
 321         // Step 2: Remove some cache keys our request doesn't rely upon.
 322         c.Assert(s.handler.s3SecretCache[arvadostest.ActiveTokenUUID], check.NotNil)
 323         delete(s.handler.s3SecretCache, arvadostest.ActiveTokenUUID)
 324         c.Assert(s.handler.s3SecretCache[arvadostest.ActiveTokenV2], check.NotNil)
 325         delete(s.handler.s3SecretCache, arvadostest.ActiveTokenV2)
 326
 327         // Step 3: Repeat the original request.
 328         rec = httptest.NewRecorder()
 329         s.handler.ServeHTTP(rec, req)
 330         resp = rec.Result()
 331         c.Assert(resp.StatusCode, check.Equals, http.StatusOK,
 332                 check.Commentf("cached auth request did not get 200 OK response"))
 333
 334         // Step 4: Confirm the deleted cache keys were not re-added
 335         // (which would imply the authorization was re-requested and cached).
 336         c.Check(s.handler.s3SecretCache[arvadostest.ActiveTokenUUID], check.IsNil,
 337                 check.Commentf("token UUID re-added to cache after removal"))
 338         c.Check(s.handler.s3SecretCache[arvadostest.ActiveTokenV2], check.IsNil,
 339                 check.Commentf("v2 token re-added to cache after removal"))
 340 }
 341
 342 func (s *IntegrationSuite) TestS3SecretCacheCleanup(c *check.C) {
 343         stage := s.s3setup(c)
 344         defer stage.teardown(c)
 345         td := -2 * s3SecretCacheTidyDuration
 346         startTidied := time.Now().Add(td)
 347         s.handler.s3SecretCacheNextTidy = startTidied
 348         s.handler.s3SecretCache = make(map[string]*cachedS3Secret)
 349         s.handler.s3SecretCache["old"] = &cachedS3Secret{expiry: startTidied.Add(td)}
 350
 351         reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
 352         c.Assert(err, check.IsNil)
 353         req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
 354         token := arvadostest.ActiveToken
 355         s.sign(c, req, token, token)
 356         rec := httptest.NewRecorder()
 357         s.handler.ServeHTTP(rec, req)
 358
 359         c.Check(s.handler.s3SecretCache["old"], check.IsNil,
 360                 check.Commentf("expired token not removed from cache"))
 361         c.Check(s.handler.s3SecretCacheNextTidy.After(startTidied), check.Equals, true,
 362                 check.Commentf("s3SecretCacheNextTidy not updated"))
 363         c.Check(s.handler.s3SecretCache[token], check.NotNil,
 364                 check.Commentf("just-used token not found in cache"))
 365 }
 366
 367 func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
 368         stage := s.s3setup(c)
 369         defer stage.teardown(c)
 370
 371         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 372                 c.Logf("bucket %s", bucket.Name)
 373                 exists, err := bucket.Exists("")
 374                 c.Check(err, check.IsNil)
 375                 c.Check(exists, check.Equals, true)
 376         }
 377 }
 378
 379 func (s *IntegrationSuite) TestS3CollectionGetObject(c *check.C) {
 380         stage := s.s3setup(c)
 381         defer stage.teardown(c)
 382         s.testS3GetObject(c, stage.collbucket, "")
 383 }
 384 func (s *IntegrationSuite) TestS3ProjectGetObject(c *check.C) {
 385         stage := s.s3setup(c)
 386         defer stage.teardown(c)
 387         s.testS3GetObject(c, stage.projbucket, stage.coll.Name+"/")
 388 }
 389 func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix string) {
 390         rdr, err := bucket.GetReader(prefix + "emptyfile")
 391         c.Assert(err, check.IsNil)
 392         buf, err := ioutil.ReadAll(rdr)
 393         c.Check(err, check.IsNil)
 394         c.Check(len(buf), check.Equals, 0)
 395         err = rdr.Close()
 396         c.Check(err, check.IsNil)
 397
 398         // GetObject
 399         rdr, err = bucket.GetReader(prefix + "missingfile")
 400         c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 401         c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 402         c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
 403
 404         // HeadObject
 405         exists, err := bucket.Exists(prefix + "missingfile")
 406         c.Check(err, check.IsNil)
 407         c.Check(exists, check.Equals, false)
 408
 409         // GetObject
 410         rdr, err = bucket.GetReader(prefix + "sailboat.txt")
 411         c.Assert(err, check.IsNil)
 412         buf, err = ioutil.ReadAll(rdr)
 413         c.Check(err, check.IsNil)
 414         c.Check(buf, check.DeepEquals, []byte("⛵\n"))
 415         err = rdr.Close()
 416         c.Check(err, check.IsNil)
 417
 418         // HeadObject
 419         resp, err := bucket.Head(prefix+"sailboat.txt", nil)
 420         c.Check(err, check.IsNil)
 421         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 422         c.Check(resp.ContentLength, check.Equals, int64(4))
 423
 424         // HeadObject with superfluous leading slashes
 425         exists, err = bucket.Exists(prefix + "//sailboat.txt")
 426         c.Check(err, check.IsNil)
 427         c.Check(exists, check.Equals, true)
 428 }
 429
 430 func (s *IntegrationSuite) checkMetaEquals(c *check.C, hdr http.Header, expect map[string]string) {
 431         got := map[string]string{}
 432         for hk, hv := range hdr {
 433                 if k := strings.TrimPrefix(hk, "X-Amz-Meta-"); k != hk && len(hv) == 1 {
 434                         got[k] = hv[0]
 435                 }
 436         }
 437         c.Check(got, check.DeepEquals, expect)
 438 }
 439
 440 func (s *IntegrationSuite) TestS3PropertiesAsMetadata(c *check.C) {
 441         stage := s.s3setup(c)
 442         defer stage.teardown(c)
 443
 444         expectCollectionTags := map[string]string{
 445                 "String":   "string value",
 446                 "Array":    `["element1","element2"]`,
 447                 "Object":   mime.BEncoding.Encode("UTF-8", `{"key":{"key2":"value⛵"}}`),
 448                 "Nonascii": "=?UTF-8?b?4pu1?=",
 449                 "Newline":  mime.BEncoding.Encode("UTF-8", "foo\r\nX-Bad: header"),
 450         }
 451         expectSubprojectTags := map[string]string{
 452                 "Subproject_properties_key": "subproject properties value",
 453         }
 454         expectProjectTags := map[string]string{
 455                 "Project-Properties-Key": "project properties value",
 456         }
 457
 458         c.Log("HEAD object with metadata from collection")
 459         resp, err := stage.collbucket.Head("sailboat.txt", nil)
 460         c.Assert(err, check.IsNil)
 461         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 462
 463         c.Log("GET object with metadata from collection")
 464         rdr, hdr, err := stage.collbucket.GetReaderWithHeaders("sailboat.txt")
 465         c.Assert(err, check.IsNil)
 466         content, err := ioutil.ReadAll(rdr)
 467         c.Check(err, check.IsNil)
 468         rdr.Close()
 469         c.Check(content, check.HasLen, 4)
 470         s.checkMetaEquals(c, hdr, expectCollectionTags)
 471         c.Check(hdr["Inject"], check.IsNil)
 472
 473         c.Log("HEAD bucket with metadata from collection")
 474         resp, err = stage.collbucket.Head("/", nil)
 475         c.Assert(err, check.IsNil)
 476         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 477
 478         c.Log("HEAD directory placeholder with metadata from collection")
 479         resp, err = stage.projbucket.Head("keep-web s3 test collection/", nil)
 480         c.Assert(err, check.IsNil)
 481         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 482
 483         c.Log("HEAD file with metadata from collection")
 484         resp, err = stage.projbucket.Head("keep-web s3 test collection/sailboat.txt", nil)
 485         c.Assert(err, check.IsNil)
 486         s.checkMetaEquals(c, resp.Header, expectCollectionTags)
 487
 488         c.Log("HEAD directory placeholder with metadata from subproject")
 489         resp, err = stage.projbucket.Head("keep-web s3 test subproject/", nil)
 490         c.Assert(err, check.IsNil)
 491         s.checkMetaEquals(c, resp.Header, expectSubprojectTags)
 492
 493         c.Log("HEAD bucket with metadata from project")
 494         resp, err = stage.projbucket.Head("/", nil)
 495         c.Assert(err, check.IsNil)
 496         s.checkMetaEquals(c, resp.Header, expectProjectTags)
 497 }
 498
 499 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
 500         stage := s.s3setup(c)
 501         defer stage.teardown(c)
 502         s.testS3PutObjectSuccess(c, stage.collbucket, "", stage.coll.UUID)
 503 }
 504 func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) {
 505         stage := s.s3setup(c)
 506         defer stage.teardown(c)
 507         s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/", stage.coll.UUID)
 508 }
 509 func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string, collUUID string) {
 510         // We insert a delay between test cases to ensure we exercise
 511         // rollover of expired sessions.
 512         sleep := time.Second / 100
 513         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(sleep * 3)
 514
 515         for _, trial := range []struct {
 516                 path        string
 517                 size        int
 518                 contentType string
 519         }{
 520                 {
 521                         path:        "newfile",
 522                         size:        128000000,
 523                         contentType: "application/octet-stream",
 524                 }, {
 525                         path:        "newdir/newfile",
 526                         size:        1 << 26,
 527                         contentType: "application/octet-stream",
 528                 }, {
 529                         path:        "/aaa",
 530                         size:        2,
 531                         contentType: "application/octet-stream",
 532                 }, {
 533                         path:        "//bbb",
 534                         size:        2,
 535                         contentType: "application/octet-stream",
 536                 }, {
 537                         path:        "ccc//",
 538                         size:        0,
 539                         contentType: "application/x-directory",
 540                 }, {
 541                         path:        "newdir1/newdir2/newfile",
 542                         size:        0,
 543                         contentType: "application/octet-stream",
 544                 }, {
 545                         path:        "newdir1/newdir2/newdir3/",
 546                         size:        0,
 547                         contentType: "application/x-directory",
 548                 },
 549         } {
 550                 time.Sleep(sleep)
 551                 c.Logf("=== %v", trial)
 552
 553                 objname := prefix + trial.path
 554
 555                 _, err := bucket.GetReader(objname)
 556                 if !c.Check(err, check.NotNil) {
 557                         continue
 558                 }
 559                 c.Check(err.(*s3.Error).StatusCode, check.Equals, http.StatusNotFound)
 560                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 561                 if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
 562                         continue
 563                 }
 564
 565                 buf := make([]byte, trial.size)
 566                 rand.Read(buf)
 567
 568                 err = bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 569                 c.Check(err, check.IsNil)
 570
 571                 rdr, err := bucket.GetReader(objname)
 572                 if strings.HasSuffix(trial.path, "/") && !s.handler.Cluster.Collections.S3FolderObjects {
 573                         c.Check(err, check.NotNil)
 574                         continue
 575                 } else if !c.Check(err, check.IsNil) {
 576                         continue
 577                 }
 578                 buf2, err := ioutil.ReadAll(rdr)
 579                 c.Check(err, check.IsNil)
 580                 c.Check(buf2, check.HasLen, len(buf))
 581                 c.Check(bytes.Equal(buf, buf2), check.Equals, true)
 582
 583                 // Check that the change is immediately visible via
 584                 // (non-S3) webdav request.
 585                 _, resp := s.do("GET", "http://"+collUUID+".keep-web.example/"+trial.path, arvadostest.ActiveTokenV2, nil)
 586                 c.Check(resp.Code, check.Equals, http.StatusOK)
 587                 if !strings.HasSuffix(trial.path, "/") {
 588                         c.Check(resp.Body.Len(), check.Equals, trial.size)
 589                 }
 590         }
 591 }
 592
 593 func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
 594         stage := s.s3setup(c)
 595         defer stage.teardown(c)
 596         bucket := stage.projbucket
 597
 598         for _, trial := range []struct {
 599                 path        string
 600                 size        int
 601                 contentType string
 602         }{
 603                 {
 604                         path:        "newfile",
 605                         size:        1234,
 606                         contentType: "application/octet-stream",
 607                 }, {
 608                         path:        "newdir/newfile",
 609                         size:        1234,
 610                         contentType: "application/octet-stream",
 611                 }, {
 612                         path:        "newdir2/",
 613                         size:        0,
 614                         contentType: "application/x-directory",
 615                 },
 616         } {
 617                 c.Logf("=== %v", trial)
 618
 619                 _, err := bucket.GetReader(trial.path)
 620                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 621                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 622                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 623
 624                 buf := make([]byte, trial.size)
 625                 rand.Read(buf)
 626
 627                 err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 628                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
 629                 c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
 630                 c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid (argument|operation)`)
 631
 632                 _, err = bucket.GetReader(trial.path)
 633                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 634                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 635                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 636         }
 637 }
 638
 639 func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) {
 640         stage := s.s3setup(c)
 641         defer stage.teardown(c)
 642         s.testS3DeleteObject(c, stage.collbucket, "")
 643 }
 644 func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) {
 645         stage := s.s3setup(c)
 646         defer stage.teardown(c)
 647         s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
 648 }
 649 func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
 650         s.handler.Cluster.Collections.S3FolderObjects = true
 651         for _, trial := range []struct {
 652                 path string
 653         }{
 654                 {"/"},
 655                 {"nonexistentfile"},
 656                 {"emptyfile"},
 657                 {"sailboat.txt"},
 658                 {"sailboat.txt/"},
 659                 {"emptydir"},
 660                 {"emptydir/"},
 661         } {
 662                 objname := prefix + trial.path
 663                 comment := check.Commentf("objname %q", objname)
 664
 665                 err := bucket.Del(objname)
 666                 if trial.path == "/" {
 667                         c.Check(err, check.NotNil)
 668                         continue
 669                 }
 670                 c.Check(err, check.IsNil, comment)
 671                 _, err = bucket.GetReader(objname)
 672                 c.Check(err, check.NotNil, comment)
 673         }
 674 }
 675
 676 func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) {
 677         stage := s.s3setup(c)
 678         defer stage.teardown(c)
 679         s.testS3PutObjectFailure(c, stage.collbucket, "")
 680 }
 681 func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
 682         stage := s.s3setup(c)
 683         defer stage.teardown(c)
 684         s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
 685 }
 686 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
 687         s.handler.Cluster.Collections.S3FolderObjects = false
 688
 689         var wg sync.WaitGroup
 690         for _, trial := range []struct {
 691                 path string
 692         }{
 693                 {
 694                         path: "emptyfile/newname", // emptyfile exists, see s3setup()
 695                 }, {
 696                         path: "emptyfile/", // emptyfile exists, see s3setup()
 697                 }, {
 698                         path: "emptydir", // dir already exists, see s3setup()
 699                 }, {
 700                         path: "emptydir/",
 701                 }, {
 702                         path: "emptydir//",
 703                 }, {
 704                         path: "newdir/",
 705                 }, {
 706                         path: "newdir//",
 707                 }, {
 708                         path: "/",
 709                 }, {
 710                         path: "//",
 711                 }, {
 712                         path: "",
 713                 },
 714         } {
 715                 trial := trial
 716                 wg.Add(1)
 717                 go func() {
 718                         defer wg.Done()
 719                         c.Logf("=== %v", trial)
 720
 721                         objname := prefix + trial.path
 722
 723                         buf := make([]byte, 1234)
 724                         rand.Read(buf)
 725
 726                         err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
 727                         if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
 728                                 return
 729                         }
 730
 731                         if objname != "" && objname != "/" {
 732                                 _, err = bucket.GetReader(objname)
 733                                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 734                                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 735                                 c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
 736                         }
 737                 }()
 738         }
 739         wg.Wait()
 740 }
 741
 742 func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
 743         fs, err := stage.coll.FileSystem(stage.arv, stage.kc)
 744         c.Assert(err, check.IsNil)
 745         for d := 0; d < dirs; d++ {
 746                 dir := fmt.Sprintf("dir%d", d)
 747                 c.Assert(fs.Mkdir(dir, 0755), check.IsNil)
 748                 for i := 0; i < filesPerDir; i++ {
 749                         f, err := fs.OpenFile(fmt.Sprintf("%s/file%d.txt", dir, i), os.O_CREATE|os.O_WRONLY, 0644)
 750                         c.Assert(err, check.IsNil)
 751                         c.Assert(f.Close(), check.IsNil)
 752                 }
 753         }
 754         c.Assert(fs.Sync(), check.IsNil)
 755 }
 756
 757 func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
 758         scope := "20200202/zzzzz/service/aws4_request"
 759         signedHeaders := "date"
 760         req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
 761         stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
 762         c.Assert(err, check.IsNil)
 763         sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
 764         c.Assert(err, check.IsNil)
 765         req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
 766 }
 767
 768 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
 769         stage := s.s3setup(c)
 770         defer stage.teardown(c)
 771         for _, trial := range []struct {
 772                 url            string
 773                 method         string
 774                 body           string
 775                 responseCode   int
 776                 responseRegexp []string
 777         }{
 778                 {
 779                         url:            "https://" + stage.collbucket.Name + ".example.com/",
 780                         method:         "GET",
 781                         responseCode:   http.StatusOK,
 782                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 783                 },
 784                 {
 785                         url:            "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
 786                         method:         "GET",
 787                         responseCode:   http.StatusOK,
 788                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 789                 },
 790                 {
 791                         url:            "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
 792                         method:         "GET",
 793                         responseCode:   http.StatusOK,
 794                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 795                 },
 796                 {
 797                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
 798                         method:         "GET",
 799                         responseCode:   http.StatusOK,
 800                         responseRegexp: []string{`⛵\n`},
 801                 },
 802                 {
 803                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 804                         method:       "PUT",
 805                         body:         "boop",
 806                         responseCode: http.StatusOK,
 807                 },
 808                 {
 809                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 810                         method:         "GET",
 811                         responseCode:   http.StatusOK,
 812                         responseRegexp: []string{`boop`},
 813                 },
 814                 {
 815                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 816                         method:       "GET",
 817                         responseCode: http.StatusNotFound,
 818                 },
 819                 {
 820                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 821                         method:       "PUT",
 822                         body:         "boop",
 823                         responseCode: http.StatusOK,
 824                 },
 825                 {
 826                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
 827                         method:         "GET",
 828                         responseCode:   http.StatusOK,
 829                         responseRegexp: []string{`boop`},
 830                 },
 831         } {
 832                 url, err := url.Parse(trial.url)
 833                 c.Assert(err, check.IsNil)
 834                 req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
 835                 c.Assert(err, check.IsNil)
 836                 s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
 837                 rr := httptest.NewRecorder()
 838                 s.handler.ServeHTTP(rr, req)
 839                 resp := rr.Result()
 840                 c.Check(resp.StatusCode, check.Equals, trial.responseCode)
 841                 body, err := ioutil.ReadAll(resp.Body)
 842                 c.Assert(err, check.IsNil)
 843                 for _, re := range trial.responseRegexp {
 844                         c.Check(string(body), check.Matches, re)
 845                 }
 846         }
 847 }
 848
 849 func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
 850         stage := s.s3setup(c)
 851         defer stage.teardown(c)
 852         for _, trial := range []struct {
 853                 rawPath        string
 854                 normalizedPath string
 855         }{
 856                 {"/foo", "/foo"},                           // boring case
 857                 {"/foo%5fbar", "/foo_bar"},                 // _ must not be escaped
 858                 {"/foo%2fbar", "/foo/bar"},                 // / must not be escaped
 859                 {"/(foo)/[];,", "/%28foo%29/%5B%5D%3B%2C"}, // ()[];, must be escaped
 860                 {"/foo%5bbar", "/foo%5Bbar"},               // %XX must be uppercase
 861                 {"//foo///.bar", "/foo/.bar"},              // "//" and "///" must be squashed to "/"
 862         } {
 863                 c.Logf("trial %q", trial)
 864
 865                 date := time.Now().UTC().Format("20060102T150405Z")
 866                 scope := "20200202/zzzzz/S3/aws4_request"
 867                 canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
 868                 c.Logf("canonicalRequest %q", canonicalRequest)
 869                 expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
 870                 c.Logf("expected stringToSign %q", expect)
 871
 872                 req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
 873                 req.Header.Set("X-Amz-Date", date)
 874                 req.Host = "host.example.com"
 875                 c.Assert(err, check.IsNil)
 876
 877                 obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
 878                 if !c.Check(err, check.IsNil) {
 879                         continue
 880                 }
 881                 c.Check(obtained, check.Equals, expect)
 882         }
 883 }
 884
 885 func (s *IntegrationSuite) TestS3GetBucketLocation(c *check.C) {
 886         stage := s.s3setup(c)
 887         defer stage.teardown(c)
 888         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 889                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 890                 c.Check(err, check.IsNil)
 891                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 892                 req.URL.RawQuery = "location"
 893                 resp, err := http.DefaultClient.Do(req)
 894                 c.Assert(err, check.IsNil)
 895                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 896                 buf, err := ioutil.ReadAll(resp.Body)
 897                 c.Assert(err, check.IsNil)
 898                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<LocationConstraint><LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">zzzzz</LocationConstraint></LocationConstraint>\n")
 899         }
 900 }
 901
 902 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
 903         stage := s.s3setup(c)
 904         defer stage.teardown(c)
 905         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 906                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 907                 c.Check(err, check.IsNil)
 908                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 909                 req.URL.RawQuery = "versioning"
 910                 resp, err := http.DefaultClient.Do(req)
 911                 c.Assert(err, check.IsNil)
 912                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 913                 buf, err := ioutil.ReadAll(resp.Body)
 914                 c.Assert(err, check.IsNil)
 915                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>\n")
 916         }
 917 }
 918
 919 func (s *IntegrationSuite) TestS3UnsupportedAPIs(c *check.C) {
 920         stage := s.s3setup(c)
 921         defer stage.teardown(c)
 922         for _, trial := range []struct {
 923                 method   string
 924                 path     string
 925                 rawquery string
 926         }{
 927                 {"GET", "/", "acl&versionId=1234"},    // GetBucketAcl
 928                 {"GET", "/foo", "acl&versionId=1234"}, // GetObjectAcl
 929                 {"PUT", "/", "acl"},                   // PutBucketAcl
 930                 {"PUT", "/foo", "acl"},                // PutObjectAcl
 931                 {"DELETE", "/", "tagging"},            // DeleteBucketTagging
 932                 {"DELETE", "/foo", "tagging"},         // DeleteObjectTagging
 933         } {
 934                 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 935                         c.Logf("trial %v bucket %v", trial, bucket)
 936                         req, err := http.NewRequest(trial.method, bucket.URL(trial.path), nil)
 937                         c.Check(err, check.IsNil)
 938                         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 939                         req.URL.RawQuery = trial.rawquery
 940                         resp, err := http.DefaultClient.Do(req)
 941                         c.Assert(err, check.IsNil)
 942                         c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 943                         buf, err := ioutil.ReadAll(resp.Body)
 944                         c.Assert(err, check.IsNil)
 945                         c.Check(string(buf), check.Matches, "(?ms).*InvalidRequest.*API not supported.*")
 946                 }
 947         }
 948 }
 949
 950 // If there are no CommonPrefixes entries, the CommonPrefixes XML tag
 951 // should not appear at all.
 952 func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
 953         stage := s.s3setup(c)
 954         defer stage.teardown(c)
 955
 956         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 957         c.Assert(err, check.IsNil)
 958         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 959         req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
 960         resp, err := http.DefaultClient.Do(req)
 961         c.Assert(err, check.IsNil)
 962         buf, err := ioutil.ReadAll(resp.Body)
 963         c.Assert(err, check.IsNil)
 964         c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
 965 }
 966
 967 // If there is no delimiter in the request, or the results are not
 968 // truncated, the NextMarker XML tag should not appear in the response
 969 // body.
 970 func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
 971         stage := s.s3setup(c)
 972         defer stage.teardown(c)
 973
 974         for _, query := range []string{"prefix=e&delimiter=/", ""} {
 975                 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 976                 c.Assert(err, check.IsNil)
 977                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 978                 req.URL.RawQuery = query
 979                 resp, err := http.DefaultClient.Do(req)
 980                 c.Assert(err, check.IsNil)
 981                 buf, err := ioutil.ReadAll(resp.Body)
 982                 c.Assert(err, check.IsNil)
 983                 c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`)
 984         }
 985 }
 986
 987 // List response should include KeyCount field.
 988 func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
 989         stage := s.s3setup(c)
 990         defer stage.teardown(c)
 991
 992         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 993         c.Assert(err, check.IsNil)
 994         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 995         req.URL.RawQuery = "prefix=&delimiter=/"
 996         resp, err := http.DefaultClient.Do(req)
 997         c.Assert(err, check.IsNil)
 998         buf, err := ioutil.ReadAll(resp.Body)
 999         c.Assert(err, check.IsNil)
1000         c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
1001 }
1002
1003 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
1004         stage := s.s3setup(c)
1005         defer stage.teardown(c)
1006
1007         var markers int
1008         for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
1009                 dirs := 2000
1010                 filesPerDir := 2
1011                 stage.writeBigDirs(c, dirs, filesPerDir)
1012                 // Total # objects is:
1013                 //                 2 file entries from s3setup (emptyfile and sailboat.txt)
1014                 //                +1 fake "directory" marker from s3setup (emptydir) (if enabled)
1015                 //             +dirs fake "directory" marker from writeBigDirs (dir0/, dir1/) (if enabled)
1016                 // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
1017                 s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
1018                 s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
1019                 s.testS3List(c, stage.collbucket, "", 51, markers+2+(filesPerDir+markers)*dirs)
1020                 s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
1021         }
1022 }
1023 func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
1024         c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.handler.Cluster.Collections.S3FolderObjects)
1025         expectPageSize := pageSize
1026         if expectPageSize > 1000 {
1027                 expectPageSize = 1000
1028         }
1029         gotKeys := map[string]s3.Key{}
1030         nextMarker := ""
1031         pages := 0
1032         for {
1033                 resp, err := bucket.List(prefix, "", nextMarker, pageSize)
1034                 if !c.Check(err, check.IsNil) {
1035                         break
1036                 }
1037                 c.Check(len(resp.Contents) <= expectPageSize, check.Equals, true)
1038                 if pages++; !c.Check(pages <= (expectFiles/expectPageSize)+1, check.Equals, true) {
1039                         break
1040                 }
1041                 for _, key := range resp.Contents {
1042                         if _, dup := gotKeys[key.Key]; dup {
1043                                 c.Errorf("got duplicate key %q on page %d", key.Key, pages)
1044                         }
1045                         gotKeys[key.Key] = key
1046                         if strings.Contains(key.Key, "sailboat.txt") {
1047                                 c.Check(key.Size, check.Equals, int64(4))
1048                         }
1049                 }
1050                 if !resp.IsTruncated {
1051                         c.Check(resp.NextMarker, check.Equals, "")
1052                         break
1053                 }
1054                 if !c.Check(resp.NextMarker, check.Not(check.Equals), "") {
1055                         break
1056                 }
1057                 nextMarker = resp.NextMarker
1058         }
1059         if !c.Check(len(gotKeys), check.Equals, expectFiles) {
1060                 var sorted []string
1061                 for k := range gotKeys {
1062                         sorted = append(sorted, k)
1063                 }
1064                 sort.Strings(sorted)
1065                 for _, k := range sorted {
1066                         c.Logf("got %s", k)
1067                 }
1068         }
1069 }
1070
1071 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
1072         for _, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
1073                 s.testS3CollectionListRollup(c)
1074         }
1075 }
1076
1077 func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
1078         stage := s.s3setup(c)
1079         defer stage.teardown(c)
1080
1081         dirs := 2
1082         filesPerDir := 500
1083         stage.writeBigDirs(c, dirs, filesPerDir)
1084         err := stage.collbucket.PutReader("dingbats", &bytes.Buffer{}, 0, "application/octet-stream", s3.Private, s3.Options{})
1085         c.Assert(err, check.IsNil)
1086         var allfiles []string
1087         for marker := ""; ; {
1088                 resp, err := stage.collbucket.List("", "", marker, 20000)
1089                 c.Check(err, check.IsNil)
1090                 for _, key := range resp.Contents {
1091                         if len(allfiles) == 0 || allfiles[len(allfiles)-1] != key.Key {
1092                                 allfiles = append(allfiles, key.Key)
1093                         }
1094                 }
1095                 marker = resp.NextMarker
1096                 if marker == "" {
1097                         break
1098                 }
1099         }
1100         markers := 0
1101         if s.handler.Cluster.Collections.S3FolderObjects {
1102                 markers = 1
1103         }
1104         c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
1105
1106         gotDirMarker := map[string]bool{}
1107         for _, name := range allfiles {
1108                 isDirMarker := strings.HasSuffix(name, "/")
1109                 if markers == 0 {
1110                         c.Check(isDirMarker, check.Equals, false, check.Commentf("name %q", name))
1111                 } else if isDirMarker {
1112                         gotDirMarker[name] = true
1113                 } else if i := strings.LastIndex(name, "/"); i >= 0 {
1114                         c.Check(gotDirMarker[name[:i+1]], check.Equals, true, check.Commentf("name %q", name))
1115                         gotDirMarker[name[:i+1]] = true // skip redundant complaints about this dir marker
1116                 }
1117         }
1118
1119         for _, trial := range []struct {
1120                 prefix    string
1121                 delimiter string
1122                 marker    string
1123         }{
1124                 {"", "", ""},
1125                 {"di", "/", ""},
1126                 {"di", "r", ""},
1127                 {"di", "n", ""},
1128                 {"dir0", "/", ""},
1129                 {"dir0/", "/", ""},
1130                 {"dir0/f", "/", ""},
1131                 {"dir0", "", ""},
1132                 {"dir0/", "", ""},
1133                 {"dir0/f", "", ""},
1134                 {"dir0", "/", "dir0/file14.txt"},       // one commonprefix, "dir0/"
1135                 {"dir0", "/", "dir0/zzzzfile.txt"},     // no commonprefixes
1136                 {"", "", "dir0/file14.txt"},            // middle page, skip walking dir1
1137                 {"", "", "dir1/file14.txt"},            // middle page, skip walking dir0
1138                 {"", "", "dir1/file498.txt"},           // last page of results
1139                 {"dir1/file", "", "dir1/file498.txt"},  // last page of results, with prefix
1140                 {"dir1/file", "/", "dir1/file498.txt"}, // last page of results, with prefix + delimiter
1141                 {"dir1", "Z", "dir1/file498.txt"},      // delimiter "Z" never appears
1142                 {"dir2", "/", ""},                      // prefix "dir2" does not exist
1143                 {"", "/", ""},
1144         } {
1145                 c.Logf("\n\n=== trial %+v markers=%d", trial, markers)
1146
1147                 maxKeys := 20
1148                 resp, err := stage.collbucket.List(trial.prefix, trial.delimiter, trial.marker, maxKeys)
1149                 c.Check(err, check.IsNil)
1150                 if resp.IsTruncated && trial.delimiter == "" {
1151                         // goamz List method fills in the missing
1152                         // NextMarker field if resp.IsTruncated, so
1153                         // now we can't really tell whether it was
1154                         // sent by the server or by goamz. In cases
1155                         // where it should be empty but isn't, assume
1156                         // it's goamz's fault.
1157                         resp.NextMarker = ""
1158                 }
1159
1160                 var expectKeys []string
1161                 var expectPrefixes []string
1162                 var expectNextMarker string
1163                 var expectTruncated bool
1164                 for _, key := range allfiles {
1165                         full := len(expectKeys)+len(expectPrefixes) >= maxKeys
1166                         if !strings.HasPrefix(key, trial.prefix) || key <= trial.marker {
1167                                 continue
1168                         } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
1169                                 prefix := key[:len(trial.prefix)+idx+1]
1170                                 if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
1171                                         // same prefix as previous key
1172                                 } else if full {
1173                                         expectTruncated = true
1174                                 } else {
1175                                         expectPrefixes = append(expectPrefixes, prefix)
1176                                         expectNextMarker = prefix
1177                                 }
1178                         } else if full {
1179                                 expectTruncated = true
1180                                 break
1181                         } else {
1182                                 expectKeys = append(expectKeys, key)
1183                                 if trial.delimiter != "" {
1184                                         expectNextMarker = key
1185                                 }
1186                         }
1187                 }
1188                 if !expectTruncated {
1189                         expectNextMarker = ""
1190                 }
1191
1192                 var gotKeys []string
1193                 for _, key := range resp.Contents {
1194                         gotKeys = append(gotKeys, key.Key)
1195                 }
1196                 var gotPrefixes []string
1197                 for _, prefix := range resp.CommonPrefixes {
1198                         gotPrefixes = append(gotPrefixes, prefix)
1199                 }
1200                 commentf := check.Commentf("trial %+v markers=%d", trial, markers)
1201                 c.Check(gotKeys, check.DeepEquals, expectKeys, commentf)
1202                 c.Check(gotPrefixes, check.DeepEquals, expectPrefixes, commentf)
1203                 c.Check(resp.NextMarker, check.Equals, expectNextMarker, commentf)
1204                 c.Check(resp.IsTruncated, check.Equals, expectTruncated, commentf)
1205                 c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
1206         }
1207 }
1208
1209 func (s *IntegrationSuite) TestS3ListObjectsV2ManySubprojects(c *check.C) {
1210         stage := s.s3setup(c)
1211         defer stage.teardown(c)
1212         projects := 50
1213         collectionsPerProject := 2
1214         for i := 0; i < projects; i++ {
1215                 var subproj arvados.Group
1216                 err := stage.arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
1217                         "group": map[string]interface{}{
1218                                 "owner_uuid":  stage.subproj.UUID,
1219                                 "group_class": "project",
1220                                 "name":        fmt.Sprintf("keep-web s3 test subproject %d", i),
1221                         },
1222                 })
1223                 c.Assert(err, check.IsNil)
1224                 for j := 0; j < collectionsPerProject; j++ {
1225                         err = stage.arv.RequestAndDecode(nil, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
1226                                 "owner_uuid":    subproj.UUID,
1227                                 "name":          fmt.Sprintf("keep-web s3 test collection %d", j),
1228                                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
1229                         }})
1230                         c.Assert(err, check.IsNil)
1231                 }
1232         }
1233         c.Logf("setup complete")
1234
1235         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1236                 Region:           aws_aws.String("auto"),
1237                 Endpoint:         aws_aws.String(s.testServer.URL),
1238                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1239                 S3ForcePathStyle: aws_aws.Bool(true),
1240         }))
1241         client := aws_s3.New(sess)
1242         ctx := context.Background()
1243         params := aws_s3.ListObjectsV2Input{
1244                 Bucket:    aws_aws.String(stage.proj.UUID),
1245                 Delimiter: aws_aws.String("/"),
1246                 Prefix:    aws_aws.String("keep-web s3 test subproject/"),
1247                 MaxKeys:   aws_aws.Int64(int64(projects / 2)),
1248         }
1249         for page := 1; ; page++ {
1250                 t0 := time.Now()
1251                 result, err := client.ListObjectsV2WithContext(ctx, &params)
1252                 if !c.Check(err, check.IsNil) {
1253                         break
1254                 }
1255                 c.Logf("got page %d in %v with len(Contents) == %d, len(CommonPrefixes) == %d", page, time.Since(t0), len(result.Contents), len(result.CommonPrefixes))
1256                 if !*result.IsTruncated {
1257                         break
1258                 }
1259                 params.ContinuationToken = result.NextContinuationToken
1260                 *params.MaxKeys = *params.MaxKeys/2 + 1
1261         }
1262 }
1263
1264 func (s *IntegrationSuite) TestS3ListObjectsV2(c *check.C) {
1265         stage := s.s3setup(c)
1266         defer stage.teardown(c)
1267         dirs := 2
1268         filesPerDir := 40
1269         stage.writeBigDirs(c, dirs, filesPerDir)
1270
1271         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1272                 Region:           aws_aws.String("auto"),
1273                 Endpoint:         aws_aws.String(s.testServer.URL),
1274                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1275                 S3ForcePathStyle: aws_aws.Bool(true),
1276         }))
1277
1278         stringOrNil := func(s string) *string {
1279                 if s == "" {
1280                         return nil
1281                 } else {
1282                         return &s
1283                 }
1284         }
1285
1286         client := aws_s3.New(sess)
1287         ctx := context.Background()
1288
1289         for _, trial := range []struct {
1290                 prefix               string
1291                 delimiter            string
1292                 startAfter           string
1293                 maxKeys              int
1294                 expectKeys           int
1295                 expectCommonPrefixes map[string]bool
1296         }{
1297                 {
1298                         // Expect {filesPerDir plus the dir itself}
1299                         // for each dir, plus emptydir, emptyfile, and
1300                         // sailboat.txt.
1301                         expectKeys: (filesPerDir+1)*dirs + 3,
1302                 },
1303                 {
1304                         maxKeys:    15,
1305                         expectKeys: (filesPerDir+1)*dirs + 3,
1306                 },
1307                 {
1308                         startAfter: "dir0/z",
1309                         maxKeys:    15,
1310                         // Expect {filesPerDir plus the dir itself}
1311                         // for each dir except dir0, plus emptydir,
1312                         // emptyfile, and sailboat.txt.
1313                         expectKeys: (filesPerDir+1)*(dirs-1) + 3,
1314                 },
1315                 {
1316                         maxKeys:              1,
1317                         delimiter:            "/",
1318                         expectKeys:           2, // emptyfile, sailboat.txt
1319                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1320                 },
1321                 {
1322                         startAfter:           "dir0/z",
1323                         maxKeys:              15,
1324                         delimiter:            "/",
1325                         expectKeys:           2, // emptyfile, sailboat.txt
1326                         expectCommonPrefixes: map[string]bool{"dir1/": true, "emptydir/": true},
1327                 },
1328                 {
1329                         startAfter:           "dir0/file10.txt",
1330                         maxKeys:              15,
1331                         delimiter:            "/",
1332                         expectKeys:           2,
1333                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1334                 },
1335                 {
1336                         startAfter:           "dir0/file10.txt",
1337                         maxKeys:              15,
1338                         prefix:               "d",
1339                         delimiter:            "/",
1340                         expectKeys:           0,
1341                         expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true},
1342                 },
1343         } {
1344                 c.Logf("[trial %+v]", trial)
1345                 params := aws_s3.ListObjectsV2Input{
1346                         Bucket:     aws_aws.String(stage.collbucket.Name),
1347                         Prefix:     stringOrNil(trial.prefix),
1348                         Delimiter:  stringOrNil(trial.delimiter),
1349                         StartAfter: stringOrNil(trial.startAfter),
1350                         MaxKeys:    aws_aws.Int64(int64(trial.maxKeys)),
1351                 }
1352                 keySeen := map[string]bool{}
1353                 prefixSeen := map[string]bool{}
1354                 for {
1355                         result, err := client.ListObjectsV2WithContext(ctx, &params)
1356                         if !c.Check(err, check.IsNil) {
1357                                 break
1358                         }
1359                         c.Check(result.Name, check.DeepEquals, aws_aws.String(stage.collbucket.Name))
1360                         c.Check(result.Prefix, check.DeepEquals, aws_aws.String(trial.prefix))
1361                         c.Check(result.Delimiter, check.DeepEquals, aws_aws.String(trial.delimiter))
1362                         // The following two fields are expected to be
1363                         // nil (i.e., no tag in XML response) rather
1364                         // than "" when the corresponding request
1365                         // field was empty or nil.
1366                         c.Check(result.StartAfter, check.DeepEquals, stringOrNil(trial.startAfter))
1367                         c.Check(result.ContinuationToken, check.DeepEquals, params.ContinuationToken)
1368
1369                         if trial.maxKeys > 0 {
1370                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(trial.maxKeys)))
1371                                 c.Check(len(result.Contents)+len(result.CommonPrefixes) <= trial.maxKeys, check.Equals, true)
1372                         } else {
1373                                 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(s3MaxKeys)))
1374                         }
1375
1376                         for _, ent := range result.Contents {
1377                                 c.Assert(ent.Key, check.NotNil)
1378                                 c.Check(*ent.Key > trial.startAfter, check.Equals, true)
1379                                 c.Check(keySeen[*ent.Key], check.Equals, false, check.Commentf("dup key %q", *ent.Key))
1380                                 keySeen[*ent.Key] = true
1381                         }
1382                         for _, ent := range result.CommonPrefixes {
1383                                 c.Assert(ent.Prefix, check.NotNil)
1384                                 c.Check(strings.HasSuffix(*ent.Prefix, trial.delimiter), check.Equals, true, check.Commentf("bad CommonPrefix %q", *ent.Prefix))
1385                                 if strings.HasPrefix(trial.startAfter, *ent.Prefix) {
1386                                         // If we asked for
1387                                         // startAfter=dir0/file10.txt,
1388                                         // we expect dir0/ to be
1389                                         // returned as a common prefix
1390                                 } else {
1391                                         c.Check(*ent.Prefix > trial.startAfter, check.Equals, true)
1392                                 }
1393                                 c.Check(prefixSeen[*ent.Prefix], check.Equals, false, check.Commentf("dup common prefix %q", *ent.Prefix))
1394                                 prefixSeen[*ent.Prefix] = true
1395                         }
1396                         if *result.IsTruncated && c.Check(result.NextContinuationToken, check.Not(check.Equals), "") {
1397                                 params.ContinuationToken = aws_aws.String(*result.NextContinuationToken)
1398                         } else {
1399                                 break
1400                         }
1401                 }
1402                 c.Check(keySeen, check.HasLen, trial.expectKeys)
1403                 c.Check(prefixSeen, check.HasLen, len(trial.expectCommonPrefixes))
1404                 if len(trial.expectCommonPrefixes) > 0 {
1405                         c.Check(prefixSeen, check.DeepEquals, trial.expectCommonPrefixes)
1406                 }
1407         }
1408 }
1409
1410 func (s *IntegrationSuite) TestS3ListObjectsV2EncodingTypeURL(c *check.C) {
1411         stage := s.s3setup(c)
1412         defer stage.teardown(c)
1413         dirs := 2
1414         filesPerDir := 40
1415         stage.writeBigDirs(c, dirs, filesPerDir)
1416
1417         sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1418                 Region:           aws_aws.String("auto"),
1419                 Endpoint:         aws_aws.String(s.testServer.URL),
1420                 Credentials:      aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1421                 S3ForcePathStyle: aws_aws.Bool(true),
1422         }))
1423
1424         client := aws_s3.New(sess)
1425         ctx := context.Background()
1426
1427         result, err := client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1428                 Bucket:       aws_aws.String(stage.collbucket.Name),
1429                 Prefix:       aws_aws.String("dir0/"),
1430                 Delimiter:    aws_aws.String("/"),
1431                 StartAfter:   aws_aws.String("dir0/"),
1432                 EncodingType: aws_aws.String("url"),
1433         })
1434         c.Assert(err, check.IsNil)
1435         c.Check(*result.Prefix, check.Equals, "dir0%2F")
1436         c.Check(*result.Delimiter, check.Equals, "%2F")
1437         c.Check(*result.StartAfter, check.Equals, "dir0%2F")
1438         for _, ent := range result.Contents {
1439                 c.Check(*ent.Key, check.Matches, "dir0%2F.*")
1440         }
1441         result, err = client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1442                 Bucket:       aws_aws.String(stage.collbucket.Name),
1443                 Delimiter:    aws_aws.String("/"),
1444                 EncodingType: aws_aws.String("url"),
1445         })
1446         c.Assert(err, check.IsNil)
1447         c.Check(*result.Delimiter, check.Equals, "%2F")
1448         c.Check(result.CommonPrefixes, check.HasLen, dirs+1)
1449         for _, ent := range result.CommonPrefixes {
1450                 c.Check(*ent.Prefix, check.Matches, ".*%2F")
1451         }
1452 }
1453
1454 // TestS3cmd checks compatibility with the s3cmd command line tool, if
1455 // it's installed. As of Debian buster, s3cmd is only in backports, so
1456 // `arvados-server install` don't install it, and this test skips if
1457 // it's not installed.
1458 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
1459         if _, err := exec.LookPath("s3cmd"); err != nil {
1460                 c.Skip("s3cmd not found")
1461                 return
1462         }
1463
1464         stage := s.s3setup(c)
1465         defer stage.teardown(c)
1466
1467         cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
1468         buf, err := cmd.CombinedOutput()
1469         c.Check(err, check.IsNil)
1470         c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
1471
1472         // This tests whether s3cmd's path normalization agrees with
1473         // keep-web's signature verification wrt chars like "|"
1474         // (neither reserved nor unreserved) and "," (not normally
1475         // percent-encoded in a path).
1476         tmpfile := c.MkDir() + "/dstfile"
1477         cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar", tmpfile)
1478         buf, err = cmd.CombinedOutput()
1479         c.Check(err, check.NotNil)
1480         // As of commit b7520e5c25e1bf25c1a8bf5aa2eadb299be8f606
1481         // (between debian bullseye and bookworm versions), s3cmd
1482         // started catching the NoSuchKey error code and replacing it
1483         // with "Source object '%s' does not exist.".
1484         c.Check(string(buf), check.Matches, `(?ms).*(NoSuchKey|Source object.*does not exist).*\n`)
1485 }
1486
1487 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
1488         stage := s.s3setup(c)
1489         defer stage.teardown(c)
1490
1491         hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
1492         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
1493         c.Check(body, check.Equals, "⛵\n")
1494 }