1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
22 "git.arvados.org/arvados.git/sdk/go/arvados"
23 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
24 "git.arvados.org/arvados.git/sdk/go/arvadostest"
25 "git.arvados.org/arvados.git/sdk/go/keepclient"
26 "github.com/AdRoll/goamz/aws"
27 "github.com/AdRoll/goamz/s3"
28 check "gopkg.in/check.v1"
33 ac *arvadosclient.ArvadosClient
34 kc *keepclient.KeepClient
37 coll arvados.Collection
41 func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
42 var proj arvados.Group
43 var coll arvados.Collection
44 arv := arvados.NewClientFromEnv()
45 arv.AuthToken = arvadostest.ActiveToken
46 err := arv.RequestAndDecode(&proj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
47 "group": map[string]interface{}{
48 "group_class": "project",
49 "name": "keep-web s3 test",
51 "ensure_unique_name": true,
53 c.Assert(err, check.IsNil)
54 err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
55 "owner_uuid": proj.UUID,
56 "name": "keep-web s3 test collection",
57 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
59 c.Assert(err, check.IsNil)
60 ac, err := arvadosclient.New(arv)
61 c.Assert(err, check.IsNil)
62 kc, err := keepclient.MakeKeepClient(ac)
63 c.Assert(err, check.IsNil)
64 fs, err := coll.FileSystem(arv, kc)
65 c.Assert(err, check.IsNil)
66 f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
67 c.Assert(err, check.IsNil)
68 _, err = f.Write([]byte("⛵\n"))
69 c.Assert(err, check.IsNil)
71 c.Assert(err, check.IsNil)
73 c.Assert(err, check.IsNil)
74 err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
75 c.Assert(err, check.IsNil)
77 auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
79 Name: s.testServer.Addr,
80 S3Endpoint: "http://" + s.testServer.Addr,
82 client := s3.New(*auth, region)
83 client.Signature = aws.V4Signature
89 projbucket: &s3.Bucket{
94 collbucket: &s3.Bucket{
101 func (stage s3stage) teardown(c *check.C) {
102 if stage.coll.UUID != "" {
103 err := stage.arv.RequestAndDecode(&stage.coll, "DELETE", "arvados/v1/collections/"+stage.coll.UUID, nil, nil)
104 c.Check(err, check.IsNil)
106 if stage.proj.UUID != "" {
107 err := stage.arv.RequestAndDecode(&stage.proj, "DELETE", "arvados/v1/groups/"+stage.proj.UUID, nil, nil)
108 c.Check(err, check.IsNil)
112 func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
113 stage := s.s3setup(c)
114 defer stage.teardown(c)
116 bucket := stage.collbucket
117 for _, trial := range []struct {
123 {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
124 {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
125 {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
126 {false, aws.V2Signature, "none", "none"},
127 {false, aws.V2Signature, "none", arvadostest.ActiveToken},
129 {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
130 {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
131 {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
132 {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
133 {false, aws.V4Signature, arvadostest.ActiveToken, ""},
134 {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
135 {false, aws.V4Signature, "none", arvadostest.ActiveToken},
136 {false, aws.V4Signature, "none", "none"},
139 bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
140 bucket.S3.Signature = trial.signature
141 _, err := bucket.GetReader("emptyfile")
143 c.Check(err, check.IsNil)
145 c.Check(err, check.NotNil)
150 func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
151 stage := s.s3setup(c)
152 defer stage.teardown(c)
154 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
155 c.Logf("bucket %s", bucket.Name)
156 exists, err := bucket.Exists("")
157 c.Check(err, check.IsNil)
158 c.Check(exists, check.Equals, true)
162 func (s *IntegrationSuite) TestS3CollectionGetObject(c *check.C) {
163 stage := s.s3setup(c)
164 defer stage.teardown(c)
165 s.testS3GetObject(c, stage.collbucket, "")
167 func (s *IntegrationSuite) TestS3ProjectGetObject(c *check.C) {
168 stage := s.s3setup(c)
169 defer stage.teardown(c)
170 s.testS3GetObject(c, stage.projbucket, stage.coll.Name+"/")
172 func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix string) {
173 rdr, err := bucket.GetReader(prefix + "emptyfile")
174 c.Assert(err, check.IsNil)
175 buf, err := ioutil.ReadAll(rdr)
176 c.Check(err, check.IsNil)
177 c.Check(len(buf), check.Equals, 0)
179 c.Check(err, check.IsNil)
182 rdr, err = bucket.GetReader(prefix + "missingfile")
183 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
184 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
185 c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
188 exists, err := bucket.Exists(prefix + "missingfile")
189 c.Check(err, check.IsNil)
190 c.Check(exists, check.Equals, false)
193 rdr, err = bucket.GetReader(prefix + "sailboat.txt")
194 c.Assert(err, check.IsNil)
195 buf, err = ioutil.ReadAll(rdr)
196 c.Check(err, check.IsNil)
197 c.Check(buf, check.DeepEquals, []byte("⛵\n"))
199 c.Check(err, check.IsNil)
202 resp, err := bucket.Head(prefix+"sailboat.txt", nil)
203 c.Check(err, check.IsNil)
204 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
205 c.Check(resp.ContentLength, check.Equals, int64(4))
207 // HeadObject with superfluous leading slashes
208 exists, err = bucket.Exists(prefix + "//sailboat.txt")
209 c.Check(err, check.IsNil)
210 c.Check(exists, check.Equals, true)
213 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
214 stage := s.s3setup(c)
215 defer stage.teardown(c)
216 s.testS3PutObjectSuccess(c, stage.collbucket, "")
218 func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) {
219 stage := s.s3setup(c)
220 defer stage.teardown(c)
221 s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/")
223 func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string) {
224 for _, trial := range []struct {
232 contentType: "application/octet-stream",
234 path: "newdir/newfile",
236 contentType: "application/octet-stream",
240 contentType: "application/octet-stream",
244 contentType: "application/octet-stream",
248 contentType: "application/x-directory",
250 path: "newdir1/newdir2/newfile",
252 contentType: "application/octet-stream",
254 path: "newdir1/newdir2/newdir3/",
256 contentType: "application/x-directory",
259 c.Logf("=== %v", trial)
261 objname := prefix + trial.path
263 _, err := bucket.GetReader(objname)
264 if !c.Check(err, check.NotNil) {
267 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
268 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
269 if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
273 buf := make([]byte, trial.size)
276 err = bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
277 c.Check(err, check.IsNil)
279 rdr, err := bucket.GetReader(objname)
280 if strings.HasSuffix(trial.path, "/") && !s.testServer.Config.cluster.Collections.S3FolderObjects {
281 c.Check(err, check.NotNil)
283 } else if !c.Check(err, check.IsNil) {
286 buf2, err := ioutil.ReadAll(rdr)
287 c.Check(err, check.IsNil)
288 c.Check(buf2, check.HasLen, len(buf))
289 c.Check(bytes.Equal(buf, buf2), check.Equals, true)
293 func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
294 stage := s.s3setup(c)
295 defer stage.teardown(c)
296 bucket := stage.projbucket
298 for _, trial := range []struct {
306 contentType: "application/octet-stream",
308 path: "newdir/newfile",
310 contentType: "application/octet-stream",
314 contentType: "application/x-directory",
317 c.Logf("=== %v", trial)
319 _, err := bucket.GetReader(trial.path)
320 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
321 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
322 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
324 buf := make([]byte, trial.size)
327 err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
328 c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
329 c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
330 c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
332 _, err = bucket.GetReader(trial.path)
333 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
334 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
335 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
339 func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) {
340 stage := s.s3setup(c)
341 defer stage.teardown(c)
342 s.testS3DeleteObject(c, stage.collbucket, "")
344 func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) {
345 stage := s.s3setup(c)
346 defer stage.teardown(c)
347 s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
349 func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
350 s.testServer.Config.cluster.Collections.S3FolderObjects = true
351 for _, trial := range []struct {
362 objname := prefix + trial.path
363 comment := check.Commentf("objname %q", objname)
365 err := bucket.Del(objname)
366 if trial.path == "/" {
367 c.Check(err, check.NotNil)
370 c.Check(err, check.IsNil, comment)
371 _, err = bucket.GetReader(objname)
372 c.Check(err, check.NotNil, comment)
376 func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) {
377 stage := s.s3setup(c)
378 defer stage.teardown(c)
379 s.testS3PutObjectFailure(c, stage.collbucket, "")
381 func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
382 stage := s.s3setup(c)
383 defer stage.teardown(c)
384 s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
386 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
387 s.testServer.Config.cluster.Collections.S3FolderObjects = false
389 var wg sync.WaitGroup
390 for _, trial := range []struct {
394 path: "emptyfile/newname", // emptyfile exists, see s3setup()
396 path: "emptyfile/", // emptyfile exists, see s3setup()
398 path: "emptydir", // dir already exists, see s3setup()
419 c.Logf("=== %v", trial)
421 objname := prefix + trial.path
423 buf := make([]byte, 1234)
426 err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
427 if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
431 if objname != "" && objname != "/" {
432 _, err = bucket.GetReader(objname)
433 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
434 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
435 c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
442 func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
443 fs, err := stage.coll.FileSystem(stage.arv, stage.kc)
444 c.Assert(err, check.IsNil)
445 for d := 0; d < dirs; d++ {
446 dir := fmt.Sprintf("dir%d", d)
447 c.Assert(fs.Mkdir(dir, 0755), check.IsNil)
448 for i := 0; i < filesPerDir; i++ {
449 f, err := fs.OpenFile(fmt.Sprintf("%s/file%d.txt", dir, i), os.O_CREATE|os.O_WRONLY, 0644)
450 c.Assert(err, check.IsNil)
451 c.Assert(f.Close(), check.IsNil)
454 c.Assert(fs.Sync(), check.IsNil)
457 func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
458 scope := "20200202/region/service/aws4_request"
459 signedHeaders := "date"
460 req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
461 stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
462 c.Assert(err, check.IsNil)
463 sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
464 c.Assert(err, check.IsNil)
465 req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
468 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
469 stage := s.s3setup(c)
470 defer stage.teardown(c)
471 for _, trial := range []struct {
476 responseRegexp []string
479 url: "https://" + stage.collbucket.Name + ".example.com/",
481 responseCode: http.StatusOK,
482 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
485 url: "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
487 responseCode: http.StatusOK,
488 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
491 url: "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
493 responseCode: http.StatusOK,
494 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
497 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
499 responseCode: http.StatusOK,
500 responseRegexp: []string{`⛵\n`},
503 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
506 responseCode: http.StatusOK,
509 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
511 responseCode: http.StatusOK,
512 responseRegexp: []string{`boop`},
515 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
517 responseCode: http.StatusNotFound,
520 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
523 responseCode: http.StatusOK,
526 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
528 responseCode: http.StatusOK,
529 responseRegexp: []string{`boop`},
532 url, err := url.Parse(trial.url)
533 c.Assert(err, check.IsNil)
534 req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
535 c.Assert(err, check.IsNil)
536 s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
537 rr := httptest.NewRecorder()
538 s.testServer.Server.Handler.ServeHTTP(rr, req)
540 c.Check(resp.StatusCode, check.Equals, trial.responseCode)
541 body, err := ioutil.ReadAll(resp.Body)
542 c.Assert(err, check.IsNil)
543 for _, re := range trial.responseRegexp {
544 c.Check(string(body), check.Matches, re)
549 func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
550 stage := s.s3setup(c)
551 defer stage.teardown(c)
552 for _, trial := range []struct {
554 normalizedPath string
556 {"/foo", "/foo"}, // boring case
557 {"/foo%5fbar", "/foo_bar"}, // _ must not be escaped
558 {"/foo%2fbar", "/foo/bar"}, // / must not be escaped
559 {"/(foo)", "/%28foo%29"}, // () must be escaped
560 {"/foo%5bbar", "/foo%5Bbar"}, // %XX must be uppercase
562 date := time.Now().UTC().Format("20060102T150405Z")
563 scope := "20200202/fakeregion/S3/aws4_request"
564 canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
565 c.Logf("canonicalRequest %q", canonicalRequest)
566 expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
567 c.Logf("expected stringToSign %q", expect)
569 req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
570 req.Header.Set("X-Amz-Date", date)
571 req.Host = "host.example.com"
573 obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
574 if !c.Check(err, check.IsNil) {
577 c.Check(obtained, check.Equals, expect)
581 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
582 stage := s.s3setup(c)
583 defer stage.teardown(c)
584 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
585 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
586 c.Check(err, check.IsNil)
587 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
588 req.URL.RawQuery = "versioning"
589 resp, err := http.DefaultClient.Do(req)
590 c.Assert(err, check.IsNil)
591 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
592 buf, err := ioutil.ReadAll(resp.Body)
593 c.Assert(err, check.IsNil)
594 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>\n")
598 // If there are no CommonPrefixes entries, the CommonPrefixes XML tag
599 // should not appear at all.
600 func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
601 stage := s.s3setup(c)
602 defer stage.teardown(c)
604 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
605 c.Assert(err, check.IsNil)
606 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
607 req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
608 resp, err := http.DefaultClient.Do(req)
609 c.Assert(err, check.IsNil)
610 buf, err := ioutil.ReadAll(resp.Body)
611 c.Assert(err, check.IsNil)
612 c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
615 // If there is no delimiter in the request, or the results are not
616 // truncated, the NextMarker XML tag should not appear in the response
618 func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
619 stage := s.s3setup(c)
620 defer stage.teardown(c)
622 for _, query := range []string{"prefix=e&delimiter=/", ""} {
623 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
624 c.Assert(err, check.IsNil)
625 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
626 req.URL.RawQuery = query
627 resp, err := http.DefaultClient.Do(req)
628 c.Assert(err, check.IsNil)
629 buf, err := ioutil.ReadAll(resp.Body)
630 c.Assert(err, check.IsNil)
631 c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`)
635 // List response should include KeyCount field.
636 func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
637 stage := s.s3setup(c)
638 defer stage.teardown(c)
640 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
641 c.Assert(err, check.IsNil)
642 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
643 req.URL.RawQuery = "prefix=&delimiter=/"
644 resp, err := http.DefaultClient.Do(req)
645 c.Assert(err, check.IsNil)
646 buf, err := ioutil.ReadAll(resp.Body)
647 c.Assert(err, check.IsNil)
648 c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
651 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
652 stage := s.s3setup(c)
653 defer stage.teardown(c)
656 for markers, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
659 stage.writeBigDirs(c, dirs, filesPerDir)
660 // Total # objects is:
661 // 2 file entries from s3setup (emptyfile and sailboat.txt)
662 // +1 fake "directory" marker from s3setup (emptydir) (if enabled)
663 // +dirs fake "directory" marker from writeBigDirs (dir0/, dir1/) (if enabled)
664 // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
665 s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
666 s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
667 s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
670 func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
671 c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.testServer.Config.cluster.Collections.S3FolderObjects)
672 expectPageSize := pageSize
673 if expectPageSize > 1000 {
674 expectPageSize = 1000
676 gotKeys := map[string]s3.Key{}
680 resp, err := bucket.List(prefix, "", nextMarker, pageSize)
681 if !c.Check(err, check.IsNil) {
684 c.Check(len(resp.Contents) <= expectPageSize, check.Equals, true)
685 if pages++; !c.Check(pages <= (expectFiles/expectPageSize)+1, check.Equals, true) {
688 for _, key := range resp.Contents {
689 gotKeys[key.Key] = key
690 if strings.Contains(key.Key, "sailboat.txt") {
691 c.Check(key.Size, check.Equals, int64(4))
694 if !resp.IsTruncated {
695 c.Check(resp.NextMarker, check.Equals, "")
698 if !c.Check(resp.NextMarker, check.Not(check.Equals), "") {
701 nextMarker = resp.NextMarker
703 c.Check(len(gotKeys), check.Equals, expectFiles)
706 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
707 for _, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
708 s.testS3CollectionListRollup(c)
712 func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
713 stage := s.s3setup(c)
714 defer stage.teardown(c)
718 stage.writeBigDirs(c, dirs, filesPerDir)
719 err := stage.collbucket.PutReader("dingbats", &bytes.Buffer{}, 0, "application/octet-stream", s3.Private, s3.Options{})
720 c.Assert(err, check.IsNil)
721 var allfiles []string
722 for marker := ""; ; {
723 resp, err := stage.collbucket.List("", "", marker, 20000)
724 c.Check(err, check.IsNil)
725 for _, key := range resp.Contents {
726 if len(allfiles) == 0 || allfiles[len(allfiles)-1] != key.Key {
727 allfiles = append(allfiles, key.Key)
730 marker = resp.NextMarker
736 if s.testServer.Config.cluster.Collections.S3FolderObjects {
739 c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
741 gotDirMarker := map[string]bool{}
742 for _, name := range allfiles {
743 isDirMarker := strings.HasSuffix(name, "/")
745 c.Check(isDirMarker, check.Equals, false, check.Commentf("name %q", name))
746 } else if isDirMarker {
747 gotDirMarker[name] = true
748 } else if i := strings.LastIndex(name, "/"); i >= 0 {
749 c.Check(gotDirMarker[name[:i+1]], check.Equals, true, check.Commentf("name %q", name))
750 gotDirMarker[name[:i+1]] = true // skip redundant complaints about this dir marker
754 for _, trial := range []struct {
769 {"dir0", "/", "dir0/file14.txt"}, // no commonprefixes
770 {"", "", "dir0/file14.txt"}, // middle page, skip walking dir1
771 {"", "", "dir1/file14.txt"}, // middle page, skip walking dir0
772 {"", "", "dir1/file498.txt"}, // last page of results
773 {"dir1/file", "", "dir1/file498.txt"}, // last page of results, with prefix
774 {"dir1/file", "/", "dir1/file498.txt"}, // last page of results, with prefix + delimiter
775 {"dir1", "Z", "dir1/file498.txt"}, // delimiter "Z" never appears
776 {"dir2", "/", ""}, // prefix "dir2" does not exist
779 c.Logf("\n\n=== trial %+v markers=%d", trial, markers)
782 resp, err := stage.collbucket.List(trial.prefix, trial.delimiter, trial.marker, maxKeys)
783 c.Check(err, check.IsNil)
784 if resp.IsTruncated && trial.delimiter == "" {
785 // goamz List method fills in the missing
786 // NextMarker field if resp.IsTruncated, so
787 // now we can't really tell whether it was
788 // sent by the server or by goamz. In cases
789 // where it should be empty but isn't, assume
790 // it's goamz's fault.
794 var expectKeys []string
795 var expectPrefixes []string
796 var expectNextMarker string
797 var expectTruncated bool
798 for _, key := range allfiles {
799 full := len(expectKeys)+len(expectPrefixes) >= maxKeys
800 if !strings.HasPrefix(key, trial.prefix) || key < trial.marker {
802 } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
803 prefix := key[:len(trial.prefix)+idx+1]
804 if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
805 // same prefix as previous key
807 expectNextMarker = key
808 expectTruncated = true
810 expectPrefixes = append(expectPrefixes, prefix)
813 if trial.delimiter != "" {
814 expectNextMarker = key
816 expectTruncated = true
819 expectKeys = append(expectKeys, key)
824 for _, key := range resp.Contents {
825 gotKeys = append(gotKeys, key.Key)
827 var gotPrefixes []string
828 for _, prefix := range resp.CommonPrefixes {
829 gotPrefixes = append(gotPrefixes, prefix)
831 commentf := check.Commentf("trial %+v markers=%d", trial, markers)
832 c.Check(gotKeys, check.DeepEquals, expectKeys, commentf)
833 c.Check(gotPrefixes, check.DeepEquals, expectPrefixes, commentf)
834 c.Check(resp.NextMarker, check.Equals, expectNextMarker, commentf)
835 c.Check(resp.IsTruncated, check.Equals, expectTruncated, commentf)
836 c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
840 // TestS3cmd checks compatibility with the s3cmd command line tool, if
841 // it's installed. As of Debian buster, s3cmd is only in backports, so
842 // `arvados-server install` don't install it, and this test skips if
843 // it's not installed.
844 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
845 if _, err := exec.LookPath("s3cmd"); err != nil {
846 c.Skip("s3cmd not found")
850 stage := s.s3setup(c)
851 defer stage.teardown(c)
853 cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.Addr, "--host-bucket="+s.testServer.Addr, "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
854 buf, err := cmd.CombinedOutput()
855 c.Check(err, check.IsNil)
856 c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
859 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
860 stage := s.s3setup(c)
861 defer stage.teardown(c)
863 hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
864 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
865 c.Check(body, check.Equals, "⛵\n")