1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
25 "git.arvados.org/arvados.git/sdk/go/arvados"
26 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
27 "git.arvados.org/arvados.git/sdk/go/arvadostest"
28 "git.arvados.org/arvados.git/sdk/go/keepclient"
29 "github.com/AdRoll/goamz/aws"
30 "github.com/AdRoll/goamz/s3"
31 aws_aws "github.com/aws/aws-sdk-go/aws"
32 aws_credentials "github.com/aws/aws-sdk-go/aws/credentials"
33 aws_session "github.com/aws/aws-sdk-go/aws/session"
34 aws_s3 "github.com/aws/aws-sdk-go/service/s3"
35 check "gopkg.in/check.v1"
38 type CachedS3SecretSuite struct{}
40 var _ = check.Suite(&CachedS3SecretSuite{})
42 func (s *CachedS3SecretSuite) activeACA(expiresAt time.Time) *arvados.APIClientAuthorization {
43 return &arvados.APIClientAuthorization{
44 UUID: arvadostest.ActiveTokenUUID,
45 APIToken: arvadostest.ActiveToken,
50 func (s *CachedS3SecretSuite) TestNewCachedS3SecretExpiresBeforeTTL(c *check.C) {
51 expected := time.Unix(1<<29, 0)
52 aca := s.activeACA(expected)
53 actual := NewCachedS3Secret(aca, time.Unix(1<<30, 0))
54 c.Check(actual.expiry, check.Equals, expected)
57 func (s *CachedS3SecretSuite) TestNewCachedS3SecretExpiresAfterTTL(c *check.C) {
58 expected := time.Unix(1<<29, 0)
59 aca := s.activeACA(time.Unix(1<<30, 0))
60 actual := NewCachedS3Secret(aca, expected)
61 c.Check(actual.expiry, check.Equals, expected)
64 func (s *CachedS3SecretSuite) TestNewCachedS3SecretWithoutExpiry(c *check.C) {
65 expected := time.Unix(1<<29, 0)
66 aca := s.activeACA(time.Time{})
67 actual := NewCachedS3Secret(aca, expected)
68 c.Check(actual.expiry, check.Equals, expected)
71 func (s *CachedS3SecretSuite) cachedSecretWithExpiry(expiry time.Time) *cachedS3Secret {
72 return &cachedS3Secret{
73 auth: s.activeACA(expiry),
78 func (s *CachedS3SecretSuite) TestIsValidAtEmpty(c *check.C) {
79 cache := &cachedS3Secret{}
80 c.Check(cache.IsValidAt(time.Unix(0, 0)), check.Equals, false)
81 c.Check(cache.IsValidAt(time.Unix(1<<31, 0)), check.Equals, false)
84 func (s *CachedS3SecretSuite) TestIsValidAtNoAuth(c *check.C) {
85 cache := &cachedS3Secret{expiry: time.Unix(3, 0)}
86 c.Check(cache.IsValidAt(time.Unix(2, 0)), check.Equals, false)
87 c.Check(cache.IsValidAt(time.Unix(4, 0)), check.Equals, false)
90 func (s *CachedS3SecretSuite) TestIsValidAtNoExpiry(c *check.C) {
91 cache := &cachedS3Secret{auth: s.activeACA(time.Unix(3, 0))}
92 c.Check(cache.IsValidAt(time.Unix(2, 0)), check.Equals, false)
93 c.Check(cache.IsValidAt(time.Unix(4, 0)), check.Equals, false)
96 func (s *CachedS3SecretSuite) TestIsValidAtTimeAfterExpiry(c *check.C) {
97 expiry := time.Unix(10, 0)
98 cache := s.cachedSecretWithExpiry(expiry)
99 c.Check(cache.IsValidAt(expiry), check.Equals, false)
100 c.Check(cache.IsValidAt(time.Unix(1<<25, 0)), check.Equals, false)
101 c.Check(cache.IsValidAt(time.Unix(1<<30, 0)), check.Equals, false)
104 func (s *CachedS3SecretSuite) TestIsValidAtTimeBeforeExpiry(c *check.C) {
105 cache := s.cachedSecretWithExpiry(time.Unix(1<<30, 0))
106 c.Check(cache.IsValidAt(time.Unix(1<<25, 0)), check.Equals, true)
107 c.Check(cache.IsValidAt(time.Unix(1<<27, 0)), check.Equals, true)
108 c.Check(cache.IsValidAt(time.Unix(1<<29, 0)), check.Equals, true)
111 func (s *CachedS3SecretSuite) TestIsValidAtZeroTime(c *check.C) {
112 cache := s.cachedSecretWithExpiry(time.Unix(10, 0))
113 c.Check(cache.IsValidAt(time.Time{}), check.Equals, false)
116 type s3stage struct {
118 ac *arvadosclient.ArvadosClient
119 kc *keepclient.KeepClient
121 projbucket *s3.Bucket
122 subproj arvados.Group
123 coll arvados.Collection
124 collbucket *s3.Bucket
127 func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
128 var proj, subproj arvados.Group
129 var coll arvados.Collection
130 arv := arvados.NewClientFromEnv()
131 arv.AuthToken = arvadostest.ActiveToken
132 err := arv.RequestAndDecode(&proj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
133 "group": map[string]interface{}{
134 "group_class": "project",
135 "name": "keep-web s3 test",
136 "properties": map[string]interface{}{
137 "project-properties-key": "project properties value",
140 "ensure_unique_name": true,
142 c.Assert(err, check.IsNil)
143 err = arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
144 "group": map[string]interface{}{
145 "owner_uuid": proj.UUID,
146 "group_class": "project",
147 "name": "keep-web s3 test subproject",
148 "properties": map[string]interface{}{
149 "subproject_properties_key": "subproject properties value",
150 "invalid header key": "this value will not be returned because key contains spaces",
154 c.Assert(err, check.IsNil)
155 err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
156 "owner_uuid": proj.UUID,
157 "name": "keep-web s3 test collection",
158 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
159 "properties": map[string]interface{}{
160 "string": "string value",
161 "array": []string{"element1", "element2"},
162 "object": map[string]interface{}{"key": map[string]interface{}{"key2": "value⛵"}},
164 "newline": "foo\r\nX-Bad: header",
165 // This key cannot be expressed as a MIME
166 // header key, so it will be silently skipped
167 // (see "Inject" in PropertiesAsMetadata test)
168 "a: a\r\nInject": "bogus",
171 c.Assert(err, check.IsNil)
172 ac, err := arvadosclient.New(arv)
173 c.Assert(err, check.IsNil)
174 kc, err := keepclient.MakeKeepClient(ac)
175 c.Assert(err, check.IsNil)
176 fs, err := coll.FileSystem(arv, kc)
177 c.Assert(err, check.IsNil)
178 f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
179 c.Assert(err, check.IsNil)
180 _, err = f.Write([]byte("⛵\n"))
181 c.Assert(err, check.IsNil)
183 c.Assert(err, check.IsNil)
185 c.Assert(err, check.IsNil)
186 err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
187 c.Assert(err, check.IsNil)
189 auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
190 region := aws.Region{
192 S3Endpoint: s.testServer.URL,
194 client := s3.New(*auth, region)
195 client.Signature = aws.V4Signature
201 projbucket: &s3.Bucket{
207 collbucket: &s3.Bucket{
214 func (stage s3stage) teardown(c *check.C) {
215 if stage.coll.UUID != "" {
216 err := stage.arv.RequestAndDecode(&stage.coll, "DELETE", "arvados/v1/collections/"+stage.coll.UUID, nil, nil)
217 c.Check(err, check.IsNil)
219 if stage.proj.UUID != "" {
220 err := stage.arv.RequestAndDecode(&stage.proj, "DELETE", "arvados/v1/groups/"+stage.proj.UUID, nil, nil)
221 c.Check(err, check.IsNil)
225 func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
226 stage := s.s3setup(c)
227 defer stage.teardown(c)
229 bucket := stage.collbucket
230 for _, trial := range []struct {
236 {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
237 {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
238 {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
239 {false, aws.V2Signature, "none", "none"},
240 {false, aws.V2Signature, "none", arvadostest.ActiveToken},
242 {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
243 {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
244 {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
245 {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
246 {false, aws.V4Signature, arvadostest.ActiveToken, ""},
247 {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
248 {false, aws.V4Signature, "none", arvadostest.ActiveToken},
249 {false, aws.V4Signature, "none", "none"},
252 bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
253 bucket.S3.Signature = trial.signature
254 _, err := bucket.GetReader("emptyfile")
256 c.Check(err, check.IsNil)
258 c.Check(err, check.NotNil)
263 func (s *IntegrationSuite) TestS3SecretCacheUpdates(c *check.C) {
264 stage := s.s3setup(c)
265 defer stage.teardown(c)
266 reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
267 c.Assert(err, check.IsNil)
269 for trialName, trialAuth := range map[string]string{
270 "v1 token": arvadostest.ActiveToken,
271 "token UUID": arvadostest.ActiveTokenUUID,
272 "v2 token query escaped": url.QueryEscape(arvadostest.ActiveTokenV2),
273 "v2 token underscore escaped": strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1),
275 s.handler.s3SecretCache = nil
276 req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
277 if !c.Check(err, check.IsNil) {
281 if secret[5:12] == "-gj3su-" {
282 secret = arvadostest.ActiveToken
284 s.sign(c, req, trialAuth, secret)
285 rec := httptest.NewRecorder()
286 s.handler.ServeHTTP(rec, req)
287 if !c.Check(rec.Result().StatusCode, check.Equals, http.StatusOK,
288 check.Commentf("%s auth did not get 200 OK response: %v", trialName, req)) {
292 for name, key := range map[string]string{
293 "v1 token": arvadostest.ActiveToken,
294 "token UUID": arvadostest.ActiveTokenUUID,
295 "v2 token": arvadostest.ActiveTokenV2,
297 actual, ok := s.handler.s3SecretCache[key]
298 if c.Check(ok, check.Equals, true, check.Commentf("%s not cached from %s", name, trialName)) {
299 c.Check(actual.auth.UUID, check.Equals, arvadostest.ActiveTokenUUID)
305 func (s *IntegrationSuite) TestS3SecretCacheUsed(c *check.C) {
306 stage := s.s3setup(c)
307 defer stage.teardown(c)
309 token := arvadostest.ActiveToken
310 // Step 1: Make a request to get the active token in the cache.
311 reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
312 c.Assert(err, check.IsNil)
313 req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
314 s.sign(c, req, token, token)
315 rec := httptest.NewRecorder()
316 s.handler.ServeHTTP(rec, req)
318 c.Assert(resp.StatusCode, check.Equals, http.StatusOK,
319 check.Commentf("first request did not get 200 OK response"))
321 // Step 2: Remove some cache keys our request doesn't rely upon.
322 c.Assert(s.handler.s3SecretCache[arvadostest.ActiveTokenUUID], check.NotNil)
323 delete(s.handler.s3SecretCache, arvadostest.ActiveTokenUUID)
324 c.Assert(s.handler.s3SecretCache[arvadostest.ActiveTokenV2], check.NotNil)
325 delete(s.handler.s3SecretCache, arvadostest.ActiveTokenV2)
327 // Step 3: Repeat the original request.
328 rec = httptest.NewRecorder()
329 s.handler.ServeHTTP(rec, req)
331 c.Assert(resp.StatusCode, check.Equals, http.StatusOK,
332 check.Commentf("cached auth request did not get 200 OK response"))
334 // Step 4: Confirm the deleted cache keys were not re-added
335 // (which would imply the authorization was re-requested and cached).
336 c.Check(s.handler.s3SecretCache[arvadostest.ActiveTokenUUID], check.IsNil,
337 check.Commentf("token UUID re-added to cache after removal"))
338 c.Check(s.handler.s3SecretCache[arvadostest.ActiveTokenV2], check.IsNil,
339 check.Commentf("v2 token re-added to cache after removal"))
342 func (s *IntegrationSuite) TestS3SecretCacheCleanup(c *check.C) {
343 stage := s.s3setup(c)
344 defer stage.teardown(c)
345 td := -2 * s3SecretCacheTidyDuration
346 startTidied := time.Now().Add(td)
347 s.handler.s3SecretCacheNextTidy = startTidied
348 s.handler.s3SecretCache = make(map[string]*cachedS3Secret)
349 s.handler.s3SecretCache["old"] = &cachedS3Secret{expiry: startTidied.Add(td)}
351 reqUrl, err := url.Parse("https://" + stage.collbucket.Name + ".example.com/")
352 c.Assert(err, check.IsNil)
353 req, err := http.NewRequest("GET", reqUrl.String(), bytes.NewReader(nil))
354 token := arvadostest.ActiveToken
355 s.sign(c, req, token, token)
356 rec := httptest.NewRecorder()
357 s.handler.ServeHTTP(rec, req)
359 c.Check(s.handler.s3SecretCache["old"], check.IsNil,
360 check.Commentf("expired token not removed from cache"))
361 c.Check(s.handler.s3SecretCacheNextTidy.After(startTidied), check.Equals, true,
362 check.Commentf("s3SecretCacheNextTidy not updated"))
363 c.Check(s.handler.s3SecretCache[token], check.NotNil,
364 check.Commentf("just-used token not found in cache"))
367 func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
368 stage := s.s3setup(c)
369 defer stage.teardown(c)
371 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
372 c.Logf("bucket %s", bucket.Name)
373 exists, err := bucket.Exists("")
374 c.Check(err, check.IsNil)
375 c.Check(exists, check.Equals, true)
379 func (s *IntegrationSuite) TestS3CollectionGetObject(c *check.C) {
380 stage := s.s3setup(c)
381 defer stage.teardown(c)
382 s.testS3GetObject(c, stage.collbucket, "")
384 func (s *IntegrationSuite) TestS3ProjectGetObject(c *check.C) {
385 stage := s.s3setup(c)
386 defer stage.teardown(c)
387 s.testS3GetObject(c, stage.projbucket, stage.coll.Name+"/")
389 func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix string) {
390 rdr, err := bucket.GetReader(prefix + "emptyfile")
391 c.Assert(err, check.IsNil)
392 buf, err := ioutil.ReadAll(rdr)
393 c.Check(err, check.IsNil)
394 c.Check(len(buf), check.Equals, 0)
396 c.Check(err, check.IsNil)
399 rdr, err = bucket.GetReader(prefix + "missingfile")
400 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
401 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
402 c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
405 exists, err := bucket.Exists(prefix + "missingfile")
406 c.Check(err, check.IsNil)
407 c.Check(exists, check.Equals, false)
410 rdr, err = bucket.GetReader(prefix + "sailboat.txt")
411 c.Assert(err, check.IsNil)
412 buf, err = ioutil.ReadAll(rdr)
413 c.Check(err, check.IsNil)
414 c.Check(buf, check.DeepEquals, []byte("⛵\n"))
416 c.Check(err, check.IsNil)
419 resp, err := bucket.Head(prefix+"sailboat.txt", nil)
420 c.Check(err, check.IsNil)
421 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
422 c.Check(resp.ContentLength, check.Equals, int64(4))
424 // HeadObject with superfluous leading slashes
425 exists, err = bucket.Exists(prefix + "//sailboat.txt")
426 c.Check(err, check.IsNil)
427 c.Check(exists, check.Equals, true)
430 func (s *IntegrationSuite) checkMetaEquals(c *check.C, hdr http.Header, expect map[string]string) {
431 got := map[string]string{}
432 for hk, hv := range hdr {
433 if k := strings.TrimPrefix(hk, "X-Amz-Meta-"); k != hk && len(hv) == 1 {
437 c.Check(got, check.DeepEquals, expect)
440 func (s *IntegrationSuite) TestS3PropertiesAsMetadata(c *check.C) {
441 stage := s.s3setup(c)
442 defer stage.teardown(c)
444 expectCollectionTags := map[string]string{
445 "String": "string value",
446 "Array": `["element1","element2"]`,
447 "Object": mime.BEncoding.Encode("UTF-8", `{"key":{"key2":"value⛵"}}`),
448 "Nonascii": "=?UTF-8?b?4pu1?=",
449 "Newline": mime.BEncoding.Encode("UTF-8", "foo\r\nX-Bad: header"),
451 expectSubprojectTags := map[string]string{
452 "Subproject_properties_key": "subproject properties value",
454 expectProjectTags := map[string]string{
455 "Project-Properties-Key": "project properties value",
458 c.Log("HEAD object with metadata from collection")
459 resp, err := stage.collbucket.Head("sailboat.txt", nil)
460 c.Assert(err, check.IsNil)
461 s.checkMetaEquals(c, resp.Header, expectCollectionTags)
463 c.Log("GET object with metadata from collection")
464 rdr, hdr, err := stage.collbucket.GetReaderWithHeaders("sailboat.txt")
465 c.Assert(err, check.IsNil)
466 content, err := ioutil.ReadAll(rdr)
467 c.Check(err, check.IsNil)
469 c.Check(content, check.HasLen, 4)
470 s.checkMetaEquals(c, hdr, expectCollectionTags)
471 c.Check(hdr["Inject"], check.IsNil)
473 c.Log("HEAD bucket with metadata from collection")
474 resp, err = stage.collbucket.Head("/", nil)
475 c.Assert(err, check.IsNil)
476 s.checkMetaEquals(c, resp.Header, expectCollectionTags)
478 c.Log("HEAD directory placeholder with metadata from collection")
479 resp, err = stage.projbucket.Head("keep-web s3 test collection/", nil)
480 c.Assert(err, check.IsNil)
481 s.checkMetaEquals(c, resp.Header, expectCollectionTags)
483 c.Log("HEAD file with metadata from collection")
484 resp, err = stage.projbucket.Head("keep-web s3 test collection/sailboat.txt", nil)
485 c.Assert(err, check.IsNil)
486 s.checkMetaEquals(c, resp.Header, expectCollectionTags)
488 c.Log("HEAD directory placeholder with metadata from subproject")
489 resp, err = stage.projbucket.Head("keep-web s3 test subproject/", nil)
490 c.Assert(err, check.IsNil)
491 s.checkMetaEquals(c, resp.Header, expectSubprojectTags)
493 c.Log("HEAD bucket with metadata from project")
494 resp, err = stage.projbucket.Head("/", nil)
495 c.Assert(err, check.IsNil)
496 s.checkMetaEquals(c, resp.Header, expectProjectTags)
499 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
500 stage := s.s3setup(c)
501 defer stage.teardown(c)
502 s.testS3PutObjectSuccess(c, stage.collbucket, "", stage.coll.UUID)
504 func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) {
505 stage := s.s3setup(c)
506 defer stage.teardown(c)
507 s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/", stage.coll.UUID)
509 func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string, collUUID string) {
510 // We insert a delay between test cases to ensure we exercise
511 // rollover of expired sessions.
512 sleep := time.Second / 100
513 s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(sleep * 3)
515 for _, trial := range []struct {
523 contentType: "application/octet-stream",
525 path: "newdir/newfile",
527 contentType: "application/octet-stream",
531 contentType: "application/octet-stream",
535 contentType: "application/octet-stream",
539 contentType: "application/x-directory",
541 path: "newdir1/newdir2/newfile",
543 contentType: "application/octet-stream",
545 path: "newdir1/newdir2/newdir3/",
547 contentType: "application/x-directory",
551 c.Logf("=== %v", trial)
553 objname := prefix + trial.path
555 _, err := bucket.GetReader(objname)
556 if !c.Check(err, check.NotNil) {
559 c.Check(err.(*s3.Error).StatusCode, check.Equals, http.StatusNotFound)
560 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
561 if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
565 buf := make([]byte, trial.size)
568 err = bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
569 c.Check(err, check.IsNil)
571 rdr, err := bucket.GetReader(objname)
572 if strings.HasSuffix(trial.path, "/") && !s.handler.Cluster.Collections.S3FolderObjects {
573 c.Check(err, check.NotNil)
575 } else if !c.Check(err, check.IsNil) {
578 buf2, err := ioutil.ReadAll(rdr)
579 c.Check(err, check.IsNil)
580 c.Check(buf2, check.HasLen, len(buf))
581 c.Check(bytes.Equal(buf, buf2), check.Equals, true)
583 // Check that the change is immediately visible via
584 // (non-S3) webdav request.
585 _, resp := s.do("GET", "http://"+collUUID+".keep-web.example/"+trial.path, arvadostest.ActiveTokenV2, nil)
586 c.Check(resp.Code, check.Equals, http.StatusOK)
587 if !strings.HasSuffix(trial.path, "/") {
588 c.Check(resp.Body.Len(), check.Equals, trial.size)
593 func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
594 stage := s.s3setup(c)
595 defer stage.teardown(c)
596 bucket := stage.projbucket
598 for _, trial := range []struct {
606 contentType: "application/octet-stream",
608 path: "newdir/newfile",
610 contentType: "application/octet-stream",
614 contentType: "application/x-directory",
617 c.Logf("=== %v", trial)
619 _, err := bucket.GetReader(trial.path)
620 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
621 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
622 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
624 buf := make([]byte, trial.size)
627 err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
628 c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
629 c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
630 c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid (argument|operation)`)
632 _, err = bucket.GetReader(trial.path)
633 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
634 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
635 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
639 func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) {
640 stage := s.s3setup(c)
641 defer stage.teardown(c)
642 s.testS3DeleteObject(c, stage.collbucket, "")
644 func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) {
645 stage := s.s3setup(c)
646 defer stage.teardown(c)
647 s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
649 func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
650 s.handler.Cluster.Collections.S3FolderObjects = true
651 for _, trial := range []struct {
662 objname := prefix + trial.path
663 comment := check.Commentf("objname %q", objname)
665 err := bucket.Del(objname)
666 if trial.path == "/" {
667 c.Check(err, check.NotNil)
670 c.Check(err, check.IsNil, comment)
671 _, err = bucket.GetReader(objname)
672 c.Check(err, check.NotNil, comment)
676 func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) {
677 stage := s.s3setup(c)
678 defer stage.teardown(c)
679 s.testS3PutObjectFailure(c, stage.collbucket, "")
681 func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
682 stage := s.s3setup(c)
683 defer stage.teardown(c)
684 s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
686 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
687 s.handler.Cluster.Collections.S3FolderObjects = false
689 var wg sync.WaitGroup
690 for _, trial := range []struct {
694 path: "emptyfile/newname", // emptyfile exists, see s3setup()
696 path: "emptyfile/", // emptyfile exists, see s3setup()
698 path: "emptydir", // dir already exists, see s3setup()
719 c.Logf("=== %v", trial)
721 objname := prefix + trial.path
723 buf := make([]byte, 1234)
726 err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
727 if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
731 if objname != "" && objname != "/" {
732 _, err = bucket.GetReader(objname)
733 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
734 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
735 c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
742 func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
743 fs, err := stage.coll.FileSystem(stage.arv, stage.kc)
744 c.Assert(err, check.IsNil)
745 for d := 0; d < dirs; d++ {
746 dir := fmt.Sprintf("dir%d", d)
747 c.Assert(fs.Mkdir(dir, 0755), check.IsNil)
748 for i := 0; i < filesPerDir; i++ {
749 f, err := fs.OpenFile(fmt.Sprintf("%s/file%d.txt", dir, i), os.O_CREATE|os.O_WRONLY, 0644)
750 c.Assert(err, check.IsNil)
751 c.Assert(f.Close(), check.IsNil)
754 c.Assert(fs.Sync(), check.IsNil)
757 func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
758 scope := "20200202/zzzzz/service/aws4_request"
759 signedHeaders := "date"
760 req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
761 stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
762 c.Assert(err, check.IsNil)
763 sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
764 c.Assert(err, check.IsNil)
765 req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
768 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
769 stage := s.s3setup(c)
770 defer stage.teardown(c)
771 for _, trial := range []struct {
776 responseRegexp []string
779 url: "https://" + stage.collbucket.Name + ".example.com/",
781 responseCode: http.StatusOK,
782 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
785 url: "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
787 responseCode: http.StatusOK,
788 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
791 url: "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
793 responseCode: http.StatusOK,
794 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
797 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
799 responseCode: http.StatusOK,
800 responseRegexp: []string{`⛵\n`},
803 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
806 responseCode: http.StatusOK,
809 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
811 responseCode: http.StatusOK,
812 responseRegexp: []string{`boop`},
815 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
817 responseCode: http.StatusNotFound,
820 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
823 responseCode: http.StatusOK,
826 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
828 responseCode: http.StatusOK,
829 responseRegexp: []string{`boop`},
832 url, err := url.Parse(trial.url)
833 c.Assert(err, check.IsNil)
834 req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
835 c.Assert(err, check.IsNil)
836 s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
837 rr := httptest.NewRecorder()
838 s.handler.ServeHTTP(rr, req)
840 c.Check(resp.StatusCode, check.Equals, trial.responseCode)
841 body, err := ioutil.ReadAll(resp.Body)
842 c.Assert(err, check.IsNil)
843 for _, re := range trial.responseRegexp {
844 c.Check(string(body), check.Matches, re)
849 func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
850 stage := s.s3setup(c)
851 defer stage.teardown(c)
852 for _, trial := range []struct {
854 normalizedPath string
856 {"/foo", "/foo"}, // boring case
857 {"/foo%5fbar", "/foo_bar"}, // _ must not be escaped
858 {"/foo%2fbar", "/foo/bar"}, // / must not be escaped
859 {"/(foo)/[];,", "/%28foo%29/%5B%5D%3B%2C"}, // ()[];, must be escaped
860 {"/foo%5bbar", "/foo%5Bbar"}, // %XX must be uppercase
861 {"//foo///.bar", "/foo/.bar"}, // "//" and "///" must be squashed to "/"
863 c.Logf("trial %q", trial)
865 date := time.Now().UTC().Format("20060102T150405Z")
866 scope := "20200202/zzzzz/S3/aws4_request"
867 canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
868 c.Logf("canonicalRequest %q", canonicalRequest)
869 expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
870 c.Logf("expected stringToSign %q", expect)
872 req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
873 req.Header.Set("X-Amz-Date", date)
874 req.Host = "host.example.com"
875 c.Assert(err, check.IsNil)
877 obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
878 if !c.Check(err, check.IsNil) {
881 c.Check(obtained, check.Equals, expect)
885 func (s *IntegrationSuite) TestS3GetBucketLocation(c *check.C) {
886 stage := s.s3setup(c)
887 defer stage.teardown(c)
888 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
889 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
890 c.Check(err, check.IsNil)
891 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
892 req.URL.RawQuery = "location"
893 resp, err := http.DefaultClient.Do(req)
894 c.Assert(err, check.IsNil)
895 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
896 buf, err := ioutil.ReadAll(resp.Body)
897 c.Assert(err, check.IsNil)
898 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<LocationConstraint><LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">zzzzz</LocationConstraint></LocationConstraint>\n")
902 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
903 stage := s.s3setup(c)
904 defer stage.teardown(c)
905 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
906 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
907 c.Check(err, check.IsNil)
908 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
909 req.URL.RawQuery = "versioning"
910 resp, err := http.DefaultClient.Do(req)
911 c.Assert(err, check.IsNil)
912 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
913 buf, err := ioutil.ReadAll(resp.Body)
914 c.Assert(err, check.IsNil)
915 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>\n")
919 func (s *IntegrationSuite) TestS3UnsupportedAPIs(c *check.C) {
920 stage := s.s3setup(c)
921 defer stage.teardown(c)
922 for _, trial := range []struct {
927 {"GET", "/", "acl&versionId=1234"}, // GetBucketAcl
928 {"GET", "/foo", "acl&versionId=1234"}, // GetObjectAcl
929 {"PUT", "/", "acl"}, // PutBucketAcl
930 {"PUT", "/foo", "acl"}, // PutObjectAcl
931 {"DELETE", "/", "tagging"}, // DeleteBucketTagging
932 {"DELETE", "/foo", "tagging"}, // DeleteObjectTagging
934 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
935 c.Logf("trial %v bucket %v", trial, bucket)
936 req, err := http.NewRequest(trial.method, bucket.URL(trial.path), nil)
937 c.Check(err, check.IsNil)
938 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
939 req.URL.RawQuery = trial.rawquery
940 resp, err := http.DefaultClient.Do(req)
941 c.Assert(err, check.IsNil)
942 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
943 buf, err := ioutil.ReadAll(resp.Body)
944 c.Assert(err, check.IsNil)
945 c.Check(string(buf), check.Matches, "(?ms).*InvalidRequest.*API not supported.*")
950 // If there are no CommonPrefixes entries, the CommonPrefixes XML tag
951 // should not appear at all.
952 func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
953 stage := s.s3setup(c)
954 defer stage.teardown(c)
956 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
957 c.Assert(err, check.IsNil)
958 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
959 req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
960 resp, err := http.DefaultClient.Do(req)
961 c.Assert(err, check.IsNil)
962 buf, err := ioutil.ReadAll(resp.Body)
963 c.Assert(err, check.IsNil)
964 c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
967 // If there is no delimiter in the request, or the results are not
968 // truncated, the NextMarker XML tag should not appear in the response
970 func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
971 stage := s.s3setup(c)
972 defer stage.teardown(c)
974 for _, query := range []string{"prefix=e&delimiter=/", ""} {
975 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
976 c.Assert(err, check.IsNil)
977 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
978 req.URL.RawQuery = query
979 resp, err := http.DefaultClient.Do(req)
980 c.Assert(err, check.IsNil)
981 buf, err := ioutil.ReadAll(resp.Body)
982 c.Assert(err, check.IsNil)
983 c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`)
987 // List response should include KeyCount field.
988 func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
989 stage := s.s3setup(c)
990 defer stage.teardown(c)
992 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
993 c.Assert(err, check.IsNil)
994 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
995 req.URL.RawQuery = "prefix=&delimiter=/"
996 resp, err := http.DefaultClient.Do(req)
997 c.Assert(err, check.IsNil)
998 buf, err := ioutil.ReadAll(resp.Body)
999 c.Assert(err, check.IsNil)
1000 c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
1003 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
1004 stage := s.s3setup(c)
1005 defer stage.teardown(c)
1008 for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
1011 stage.writeBigDirs(c, dirs, filesPerDir)
1012 // Total # objects is:
1013 // 2 file entries from s3setup (emptyfile and sailboat.txt)
1014 // +1 fake "directory" marker from s3setup (emptydir) (if enabled)
1015 // +dirs fake "directory" marker from writeBigDirs (dir0/, dir1/) (if enabled)
1016 // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
1017 s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
1018 s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
1019 s.testS3List(c, stage.collbucket, "", 51, markers+2+(filesPerDir+markers)*dirs)
1020 s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
1023 func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
1024 c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.handler.Cluster.Collections.S3FolderObjects)
1025 expectPageSize := pageSize
1026 if expectPageSize > 1000 {
1027 expectPageSize = 1000
1029 gotKeys := map[string]s3.Key{}
1033 resp, err := bucket.List(prefix, "", nextMarker, pageSize)
1034 if !c.Check(err, check.IsNil) {
1037 c.Check(len(resp.Contents) <= expectPageSize, check.Equals, true)
1038 if pages++; !c.Check(pages <= (expectFiles/expectPageSize)+1, check.Equals, true) {
1041 for _, key := range resp.Contents {
1042 if _, dup := gotKeys[key.Key]; dup {
1043 c.Errorf("got duplicate key %q on page %d", key.Key, pages)
1045 gotKeys[key.Key] = key
1046 if strings.Contains(key.Key, "sailboat.txt") {
1047 c.Check(key.Size, check.Equals, int64(4))
1050 if !resp.IsTruncated {
1051 c.Check(resp.NextMarker, check.Equals, "")
1054 if !c.Check(resp.NextMarker, check.Not(check.Equals), "") {
1057 nextMarker = resp.NextMarker
1059 if !c.Check(len(gotKeys), check.Equals, expectFiles) {
1061 for k := range gotKeys {
1062 sorted = append(sorted, k)
1064 sort.Strings(sorted)
1065 for _, k := range sorted {
1071 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
1072 for _, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
1073 s.testS3CollectionListRollup(c)
1077 func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
1078 stage := s.s3setup(c)
1079 defer stage.teardown(c)
1083 stage.writeBigDirs(c, dirs, filesPerDir)
1084 err := stage.collbucket.PutReader("dingbats", &bytes.Buffer{}, 0, "application/octet-stream", s3.Private, s3.Options{})
1085 c.Assert(err, check.IsNil)
1086 var allfiles []string
1087 for marker := ""; ; {
1088 resp, err := stage.collbucket.List("", "", marker, 20000)
1089 c.Check(err, check.IsNil)
1090 for _, key := range resp.Contents {
1091 if len(allfiles) == 0 || allfiles[len(allfiles)-1] != key.Key {
1092 allfiles = append(allfiles, key.Key)
1095 marker = resp.NextMarker
1101 if s.handler.Cluster.Collections.S3FolderObjects {
1104 c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
1106 gotDirMarker := map[string]bool{}
1107 for _, name := range allfiles {
1108 isDirMarker := strings.HasSuffix(name, "/")
1110 c.Check(isDirMarker, check.Equals, false, check.Commentf("name %q", name))
1111 } else if isDirMarker {
1112 gotDirMarker[name] = true
1113 } else if i := strings.LastIndex(name, "/"); i >= 0 {
1114 c.Check(gotDirMarker[name[:i+1]], check.Equals, true, check.Commentf("name %q", name))
1115 gotDirMarker[name[:i+1]] = true // skip redundant complaints about this dir marker
1119 for _, trial := range []struct {
1130 {"dir0/f", "/", ""},
1134 {"dir0", "/", "dir0/file14.txt"}, // one commonprefix, "dir0/"
1135 {"dir0", "/", "dir0/zzzzfile.txt"}, // no commonprefixes
1136 {"", "", "dir0/file14.txt"}, // middle page, skip walking dir1
1137 {"", "", "dir1/file14.txt"}, // middle page, skip walking dir0
1138 {"", "", "dir1/file498.txt"}, // last page of results
1139 {"dir1/file", "", "dir1/file498.txt"}, // last page of results, with prefix
1140 {"dir1/file", "/", "dir1/file498.txt"}, // last page of results, with prefix + delimiter
1141 {"dir1", "Z", "dir1/file498.txt"}, // delimiter "Z" never appears
1142 {"dir2", "/", ""}, // prefix "dir2" does not exist
1145 c.Logf("\n\n=== trial %+v markers=%d", trial, markers)
1148 resp, err := stage.collbucket.List(trial.prefix, trial.delimiter, trial.marker, maxKeys)
1149 c.Check(err, check.IsNil)
1150 if resp.IsTruncated && trial.delimiter == "" {
1151 // goamz List method fills in the missing
1152 // NextMarker field if resp.IsTruncated, so
1153 // now we can't really tell whether it was
1154 // sent by the server or by goamz. In cases
1155 // where it should be empty but isn't, assume
1156 // it's goamz's fault.
1157 resp.NextMarker = ""
1160 var expectKeys []string
1161 var expectPrefixes []string
1162 var expectNextMarker string
1163 var expectTruncated bool
1164 for _, key := range allfiles {
1165 full := len(expectKeys)+len(expectPrefixes) >= maxKeys
1166 if !strings.HasPrefix(key, trial.prefix) || key <= trial.marker {
1168 } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
1169 prefix := key[:len(trial.prefix)+idx+1]
1170 if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
1171 // same prefix as previous key
1173 expectTruncated = true
1175 expectPrefixes = append(expectPrefixes, prefix)
1176 expectNextMarker = prefix
1179 expectTruncated = true
1182 expectKeys = append(expectKeys, key)
1183 if trial.delimiter != "" {
1184 expectNextMarker = key
1188 if !expectTruncated {
1189 expectNextMarker = ""
1192 var gotKeys []string
1193 for _, key := range resp.Contents {
1194 gotKeys = append(gotKeys, key.Key)
1196 var gotPrefixes []string
1197 for _, prefix := range resp.CommonPrefixes {
1198 gotPrefixes = append(gotPrefixes, prefix)
1200 commentf := check.Commentf("trial %+v markers=%d", trial, markers)
1201 c.Check(gotKeys, check.DeepEquals, expectKeys, commentf)
1202 c.Check(gotPrefixes, check.DeepEquals, expectPrefixes, commentf)
1203 c.Check(resp.NextMarker, check.Equals, expectNextMarker, commentf)
1204 c.Check(resp.IsTruncated, check.Equals, expectTruncated, commentf)
1205 c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
1209 func (s *IntegrationSuite) TestS3ListObjectsV2ManySubprojects(c *check.C) {
1210 stage := s.s3setup(c)
1211 defer stage.teardown(c)
1213 collectionsPerProject := 2
1214 for i := 0; i < projects; i++ {
1215 var subproj arvados.Group
1216 err := stage.arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
1217 "group": map[string]interface{}{
1218 "owner_uuid": stage.subproj.UUID,
1219 "group_class": "project",
1220 "name": fmt.Sprintf("keep-web s3 test subproject %d", i),
1223 c.Assert(err, check.IsNil)
1224 for j := 0; j < collectionsPerProject; j++ {
1225 err = stage.arv.RequestAndDecode(nil, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
1226 "owner_uuid": subproj.UUID,
1227 "name": fmt.Sprintf("keep-web s3 test collection %d", j),
1228 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
1230 c.Assert(err, check.IsNil)
1233 c.Logf("setup complete")
1235 sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1236 Region: aws_aws.String("auto"),
1237 Endpoint: aws_aws.String(s.testServer.URL),
1238 Credentials: aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1239 S3ForcePathStyle: aws_aws.Bool(true),
1241 client := aws_s3.New(sess)
1242 ctx := context.Background()
1243 params := aws_s3.ListObjectsV2Input{
1244 Bucket: aws_aws.String(stage.proj.UUID),
1245 Delimiter: aws_aws.String("/"),
1246 Prefix: aws_aws.String("keep-web s3 test subproject/"),
1247 MaxKeys: aws_aws.Int64(int64(projects / 2)),
1249 for page := 1; ; page++ {
1251 result, err := client.ListObjectsV2WithContext(ctx, ¶ms)
1252 if !c.Check(err, check.IsNil) {
1255 c.Logf("got page %d in %v with len(Contents) == %d, len(CommonPrefixes) == %d", page, time.Since(t0), len(result.Contents), len(result.CommonPrefixes))
1256 if !*result.IsTruncated {
1259 params.ContinuationToken = result.NextContinuationToken
1260 *params.MaxKeys = *params.MaxKeys/2 + 1
1264 func (s *IntegrationSuite) TestS3ListObjectsV2(c *check.C) {
1265 stage := s.s3setup(c)
1266 defer stage.teardown(c)
1269 stage.writeBigDirs(c, dirs, filesPerDir)
1271 sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1272 Region: aws_aws.String("auto"),
1273 Endpoint: aws_aws.String(s.testServer.URL),
1274 Credentials: aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1275 S3ForcePathStyle: aws_aws.Bool(true),
1278 stringOrNil := func(s string) *string {
1286 client := aws_s3.New(sess)
1287 ctx := context.Background()
1289 for _, trial := range []struct {
1295 expectCommonPrefixes map[string]bool
1298 // Expect {filesPerDir plus the dir itself}
1299 // for each dir, plus emptydir, emptyfile, and
1301 expectKeys: (filesPerDir+1)*dirs + 3,
1305 expectKeys: (filesPerDir+1)*dirs + 3,
1308 startAfter: "dir0/z",
1310 // Expect {filesPerDir plus the dir itself}
1311 // for each dir except dir0, plus emptydir,
1312 // emptyfile, and sailboat.txt.
1313 expectKeys: (filesPerDir+1)*(dirs-1) + 3,
1318 expectKeys: 2, // emptyfile, sailboat.txt
1319 expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1322 startAfter: "dir0/z",
1325 expectKeys: 2, // emptyfile, sailboat.txt
1326 expectCommonPrefixes: map[string]bool{"dir1/": true, "emptydir/": true},
1329 startAfter: "dir0/file10.txt",
1333 expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
1336 startAfter: "dir0/file10.txt",
1341 expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true},
1344 c.Logf("[trial %+v]", trial)
1345 params := aws_s3.ListObjectsV2Input{
1346 Bucket: aws_aws.String(stage.collbucket.Name),
1347 Prefix: stringOrNil(trial.prefix),
1348 Delimiter: stringOrNil(trial.delimiter),
1349 StartAfter: stringOrNil(trial.startAfter),
1350 MaxKeys: aws_aws.Int64(int64(trial.maxKeys)),
1352 keySeen := map[string]bool{}
1353 prefixSeen := map[string]bool{}
1355 result, err := client.ListObjectsV2WithContext(ctx, ¶ms)
1356 if !c.Check(err, check.IsNil) {
1359 c.Check(result.Name, check.DeepEquals, aws_aws.String(stage.collbucket.Name))
1360 c.Check(result.Prefix, check.DeepEquals, aws_aws.String(trial.prefix))
1361 c.Check(result.Delimiter, check.DeepEquals, aws_aws.String(trial.delimiter))
1362 // The following two fields are expected to be
1363 // nil (i.e., no tag in XML response) rather
1364 // than "" when the corresponding request
1365 // field was empty or nil.
1366 c.Check(result.StartAfter, check.DeepEquals, stringOrNil(trial.startAfter))
1367 c.Check(result.ContinuationToken, check.DeepEquals, params.ContinuationToken)
1369 if trial.maxKeys > 0 {
1370 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(trial.maxKeys)))
1371 c.Check(len(result.Contents)+len(result.CommonPrefixes) <= trial.maxKeys, check.Equals, true)
1373 c.Check(result.MaxKeys, check.DeepEquals, aws_aws.Int64(int64(s3MaxKeys)))
1376 for _, ent := range result.Contents {
1377 c.Assert(ent.Key, check.NotNil)
1378 c.Check(*ent.Key > trial.startAfter, check.Equals, true)
1379 c.Check(keySeen[*ent.Key], check.Equals, false, check.Commentf("dup key %q", *ent.Key))
1380 keySeen[*ent.Key] = true
1382 for _, ent := range result.CommonPrefixes {
1383 c.Assert(ent.Prefix, check.NotNil)
1384 c.Check(strings.HasSuffix(*ent.Prefix, trial.delimiter), check.Equals, true, check.Commentf("bad CommonPrefix %q", *ent.Prefix))
1385 if strings.HasPrefix(trial.startAfter, *ent.Prefix) {
1387 // startAfter=dir0/file10.txt,
1388 // we expect dir0/ to be
1389 // returned as a common prefix
1391 c.Check(*ent.Prefix > trial.startAfter, check.Equals, true)
1393 c.Check(prefixSeen[*ent.Prefix], check.Equals, false, check.Commentf("dup common prefix %q", *ent.Prefix))
1394 prefixSeen[*ent.Prefix] = true
1396 if *result.IsTruncated && c.Check(result.NextContinuationToken, check.Not(check.Equals), "") {
1397 params.ContinuationToken = aws_aws.String(*result.NextContinuationToken)
1402 c.Check(keySeen, check.HasLen, trial.expectKeys)
1403 c.Check(prefixSeen, check.HasLen, len(trial.expectCommonPrefixes))
1404 if len(trial.expectCommonPrefixes) > 0 {
1405 c.Check(prefixSeen, check.DeepEquals, trial.expectCommonPrefixes)
1410 func (s *IntegrationSuite) TestS3ListObjectsV2EncodingTypeURL(c *check.C) {
1411 stage := s.s3setup(c)
1412 defer stage.teardown(c)
1415 stage.writeBigDirs(c, dirs, filesPerDir)
1417 sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
1418 Region: aws_aws.String("auto"),
1419 Endpoint: aws_aws.String(s.testServer.URL),
1420 Credentials: aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
1421 S3ForcePathStyle: aws_aws.Bool(true),
1424 client := aws_s3.New(sess)
1425 ctx := context.Background()
1427 result, err := client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1428 Bucket: aws_aws.String(stage.collbucket.Name),
1429 Prefix: aws_aws.String("dir0/"),
1430 Delimiter: aws_aws.String("/"),
1431 StartAfter: aws_aws.String("dir0/"),
1432 EncodingType: aws_aws.String("url"),
1434 c.Assert(err, check.IsNil)
1435 c.Check(*result.Prefix, check.Equals, "dir0%2F")
1436 c.Check(*result.Delimiter, check.Equals, "%2F")
1437 c.Check(*result.StartAfter, check.Equals, "dir0%2F")
1438 for _, ent := range result.Contents {
1439 c.Check(*ent.Key, check.Matches, "dir0%2F.*")
1441 result, err = client.ListObjectsV2WithContext(ctx, &aws_s3.ListObjectsV2Input{
1442 Bucket: aws_aws.String(stage.collbucket.Name),
1443 Delimiter: aws_aws.String("/"),
1444 EncodingType: aws_aws.String("url"),
1446 c.Assert(err, check.IsNil)
1447 c.Check(*result.Delimiter, check.Equals, "%2F")
1448 c.Check(result.CommonPrefixes, check.HasLen, dirs+1)
1449 for _, ent := range result.CommonPrefixes {
1450 c.Check(*ent.Prefix, check.Matches, ".*%2F")
1454 // TestS3cmd checks compatibility with the s3cmd command line tool, if
1455 // it's installed. As of Debian buster, s3cmd is only in backports, so
1456 // `arvados-server install` don't install it, and this test skips if
1457 // it's not installed.
1458 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
1459 if _, err := exec.LookPath("s3cmd"); err != nil {
1460 c.Skip("s3cmd not found")
1464 stage := s.s3setup(c)
1465 defer stage.teardown(c)
1467 cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
1468 buf, err := cmd.CombinedOutput()
1469 c.Check(err, check.IsNil)
1470 c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
1472 // This tests whether s3cmd's path normalization agrees with
1473 // keep-web's signature verification wrt chars like "|"
1474 // (neither reserved nor unreserved) and "," (not normally
1475 // percent-encoded in a path).
1476 tmpfile := c.MkDir() + "/dstfile"
1477 cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar", tmpfile)
1478 buf, err = cmd.CombinedOutput()
1479 c.Check(err, check.NotNil)
1480 // As of commit b7520e5c25e1bf25c1a8bf5aa2eadb299be8f606
1481 // (between debian bullseye and bookworm versions), s3cmd
1482 // started catching the NoSuchKey error code and replacing it
1483 // with "Source object '%s' does not exist.".
1484 c.Check(string(buf), check.Matches, `(?ms).*(NoSuchKey|Source object.*does not exist).*\n`)
1487 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
1488 stage := s.s3setup(c)
1489 defer stage.teardown(c)
1491 hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
1492 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
1493 c.Check(body, check.Equals, "⛵\n")