1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
22 "git.arvados.org/arvados.git/sdk/go/arvados"
23 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
24 "git.arvados.org/arvados.git/sdk/go/arvadostest"
25 "git.arvados.org/arvados.git/sdk/go/keepclient"
26 "github.com/AdRoll/goamz/aws"
27 "github.com/AdRoll/goamz/s3"
28 check "gopkg.in/check.v1"
33 ac *arvadosclient.ArvadosClient
34 kc *keepclient.KeepClient
37 coll arvados.Collection
41 func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
42 var proj arvados.Group
43 var coll arvados.Collection
44 arv := arvados.NewClientFromEnv()
45 arv.AuthToken = arvadostest.ActiveToken
46 err := arv.RequestAndDecode(&proj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
47 "group": map[string]interface{}{
48 "group_class": "project",
49 "name": "keep-web s3 test",
51 "ensure_unique_name": true,
53 c.Assert(err, check.IsNil)
54 err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
55 "owner_uuid": proj.UUID,
56 "name": "keep-web s3 test collection",
57 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
59 c.Assert(err, check.IsNil)
60 ac, err := arvadosclient.New(arv)
61 c.Assert(err, check.IsNil)
62 kc, err := keepclient.MakeKeepClient(ac)
63 c.Assert(err, check.IsNil)
64 fs, err := coll.FileSystem(arv, kc)
65 c.Assert(err, check.IsNil)
66 f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
67 c.Assert(err, check.IsNil)
68 _, err = f.Write([]byte("⛵\n"))
69 c.Assert(err, check.IsNil)
71 c.Assert(err, check.IsNil)
73 c.Assert(err, check.IsNil)
74 err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
75 c.Assert(err, check.IsNil)
77 auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
79 Name: s.testServer.Addr,
80 S3Endpoint: "http://" + s.testServer.Addr,
82 client := s3.New(*auth, region)
83 client.Signature = aws.V4Signature
89 projbucket: &s3.Bucket{
94 collbucket: &s3.Bucket{
101 func (stage s3stage) teardown(c *check.C) {
102 if stage.coll.UUID != "" {
103 err := stage.arv.RequestAndDecode(&stage.coll, "DELETE", "arvados/v1/collections/"+stage.coll.UUID, nil, nil)
104 c.Check(err, check.IsNil)
106 if stage.proj.UUID != "" {
107 err := stage.arv.RequestAndDecode(&stage.proj, "DELETE", "arvados/v1/groups/"+stage.proj.UUID, nil, nil)
108 c.Check(err, check.IsNil)
112 func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
113 stage := s.s3setup(c)
114 defer stage.teardown(c)
116 bucket := stage.collbucket
117 for _, trial := range []struct {
123 {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
124 {false, aws.V2Signature, "none", "none"},
125 {false, aws.V2Signature, "none", arvadostest.ActiveToken},
127 {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
128 {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
129 {false, aws.V4Signature, arvadostest.ActiveToken, ""},
130 {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
131 {false, aws.V4Signature, "none", arvadostest.ActiveToken},
132 {false, aws.V4Signature, "none", "none"},
135 bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
136 bucket.S3.Signature = trial.signature
137 _, err := bucket.GetReader("emptyfile")
139 c.Check(err, check.IsNil)
141 c.Check(err, check.NotNil)
146 func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
147 stage := s.s3setup(c)
148 defer stage.teardown(c)
150 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
151 c.Logf("bucket %s", bucket.Name)
152 exists, err := bucket.Exists("")
153 c.Check(err, check.IsNil)
154 c.Check(exists, check.Equals, true)
158 func (s *IntegrationSuite) TestS3CollectionGetObject(c *check.C) {
159 stage := s.s3setup(c)
160 defer stage.teardown(c)
161 s.testS3GetObject(c, stage.collbucket, "")
163 func (s *IntegrationSuite) TestS3ProjectGetObject(c *check.C) {
164 stage := s.s3setup(c)
165 defer stage.teardown(c)
166 s.testS3GetObject(c, stage.projbucket, stage.coll.Name+"/")
168 func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix string) {
169 rdr, err := bucket.GetReader(prefix + "emptyfile")
170 c.Assert(err, check.IsNil)
171 buf, err := ioutil.ReadAll(rdr)
172 c.Check(err, check.IsNil)
173 c.Check(len(buf), check.Equals, 0)
175 c.Check(err, check.IsNil)
178 rdr, err = bucket.GetReader(prefix + "missingfile")
179 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
180 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
181 c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
184 exists, err := bucket.Exists(prefix + "missingfile")
185 c.Check(err, check.IsNil)
186 c.Check(exists, check.Equals, false)
189 rdr, err = bucket.GetReader(prefix + "sailboat.txt")
190 c.Assert(err, check.IsNil)
191 buf, err = ioutil.ReadAll(rdr)
192 c.Check(err, check.IsNil)
193 c.Check(buf, check.DeepEquals, []byte("⛵\n"))
195 c.Check(err, check.IsNil)
198 resp, err := bucket.Head(prefix+"sailboat.txt", nil)
199 c.Check(err, check.IsNil)
200 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
201 c.Check(resp.ContentLength, check.Equals, int64(4))
203 // HeadObject with superfluous leading slashes
204 exists, err = bucket.Exists(prefix + "//sailboat.txt")
205 c.Check(err, check.IsNil)
206 c.Check(exists, check.Equals, true)
209 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
210 stage := s.s3setup(c)
211 defer stage.teardown(c)
212 s.testS3PutObjectSuccess(c, stage.collbucket, "")
214 func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) {
215 stage := s.s3setup(c)
216 defer stage.teardown(c)
217 s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/")
219 func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string) {
220 for _, trial := range []struct {
228 contentType: "application/octet-stream",
230 path: "newdir/newfile",
232 contentType: "application/octet-stream",
236 contentType: "application/octet-stream",
240 contentType: "application/octet-stream",
244 contentType: "application/x-directory",
246 path: "newdir1/newdir2/newfile",
248 contentType: "application/octet-stream",
250 path: "newdir1/newdir2/newdir3/",
252 contentType: "application/x-directory",
255 c.Logf("=== %v", trial)
257 objname := prefix + trial.path
259 _, err := bucket.GetReader(objname)
260 if !c.Check(err, check.NotNil) {
263 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
264 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
265 if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
269 buf := make([]byte, trial.size)
272 err = bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
273 c.Check(err, check.IsNil)
275 rdr, err := bucket.GetReader(objname)
276 if strings.HasSuffix(trial.path, "/") && !s.testServer.Config.cluster.Collections.S3FolderObjects {
277 c.Check(err, check.NotNil)
279 } else if !c.Check(err, check.IsNil) {
282 buf2, err := ioutil.ReadAll(rdr)
283 c.Check(err, check.IsNil)
284 c.Check(buf2, check.HasLen, len(buf))
285 c.Check(bytes.Equal(buf, buf2), check.Equals, true)
289 func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
290 stage := s.s3setup(c)
291 defer stage.teardown(c)
292 bucket := stage.projbucket
294 for _, trial := range []struct {
302 contentType: "application/octet-stream",
304 path: "newdir/newfile",
306 contentType: "application/octet-stream",
310 contentType: "application/x-directory",
313 c.Logf("=== %v", trial)
315 _, err := bucket.GetReader(trial.path)
316 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
317 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
318 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
320 buf := make([]byte, trial.size)
323 err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
324 c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
325 c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
326 c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
328 _, err = bucket.GetReader(trial.path)
329 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
330 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
331 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
335 func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) {
336 stage := s.s3setup(c)
337 defer stage.teardown(c)
338 s.testS3DeleteObject(c, stage.collbucket, "")
340 func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) {
341 stage := s.s3setup(c)
342 defer stage.teardown(c)
343 s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
345 func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
346 s.testServer.Config.cluster.Collections.S3FolderObjects = true
347 for _, trial := range []struct {
358 objname := prefix + trial.path
359 comment := check.Commentf("objname %q", objname)
361 err := bucket.Del(objname)
362 if trial.path == "/" {
363 c.Check(err, check.NotNil)
366 c.Check(err, check.IsNil, comment)
367 _, err = bucket.GetReader(objname)
368 c.Check(err, check.NotNil, comment)
372 func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) {
373 stage := s.s3setup(c)
374 defer stage.teardown(c)
375 s.testS3PutObjectFailure(c, stage.collbucket, "")
377 func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
378 stage := s.s3setup(c)
379 defer stage.teardown(c)
380 s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
382 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
383 s.testServer.Config.cluster.Collections.S3FolderObjects = false
385 var wg sync.WaitGroup
386 for _, trial := range []struct {
390 path: "emptyfile/newname", // emptyfile exists, see s3setup()
392 path: "emptyfile/", // emptyfile exists, see s3setup()
394 path: "emptydir", // dir already exists, see s3setup()
415 c.Logf("=== %v", trial)
417 objname := prefix + trial.path
419 buf := make([]byte, 1234)
422 err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
423 if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
427 if objname != "" && objname != "/" {
428 _, err = bucket.GetReader(objname)
429 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
430 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
431 c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
438 func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
439 fs, err := stage.coll.FileSystem(stage.arv, stage.kc)
440 c.Assert(err, check.IsNil)
441 for d := 0; d < dirs; d++ {
442 dir := fmt.Sprintf("dir%d", d)
443 c.Assert(fs.Mkdir(dir, 0755), check.IsNil)
444 for i := 0; i < filesPerDir; i++ {
445 f, err := fs.OpenFile(fmt.Sprintf("%s/file%d.txt", dir, i), os.O_CREATE|os.O_WRONLY, 0644)
446 c.Assert(err, check.IsNil)
447 c.Assert(f.Close(), check.IsNil)
450 c.Assert(fs.Sync(), check.IsNil)
453 func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
454 scope := "20200202/region/service/aws4_request"
455 signedHeaders := "date"
456 req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
457 stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
458 c.Assert(err, check.IsNil)
459 sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
460 c.Assert(err, check.IsNil)
461 req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
464 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
465 stage := s.s3setup(c)
466 defer stage.teardown(c)
467 for _, trial := range []struct {
472 responseRegexp []string
475 url: "https://" + stage.collbucket.Name + ".example.com/",
477 responseCode: http.StatusOK,
478 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
481 url: "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
483 responseCode: http.StatusOK,
484 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
487 url: "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
489 responseCode: http.StatusOK,
490 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
493 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
495 responseCode: http.StatusOK,
496 responseRegexp: []string{`⛵\n`},
499 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
502 responseCode: http.StatusOK,
505 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
507 responseCode: http.StatusOK,
508 responseRegexp: []string{`boop`},
511 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
513 responseCode: http.StatusNotFound,
516 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
519 responseCode: http.StatusOK,
522 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
524 responseCode: http.StatusOK,
525 responseRegexp: []string{`boop`},
528 url, err := url.Parse(trial.url)
529 c.Assert(err, check.IsNil)
530 req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
531 c.Assert(err, check.IsNil)
532 s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
533 rr := httptest.NewRecorder()
534 s.testServer.Server.Handler.ServeHTTP(rr, req)
536 c.Check(resp.StatusCode, check.Equals, trial.responseCode)
537 body, err := ioutil.ReadAll(resp.Body)
538 c.Assert(err, check.IsNil)
539 for _, re := range trial.responseRegexp {
540 c.Check(string(body), check.Matches, re)
545 func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
546 stage := s.s3setup(c)
547 defer stage.teardown(c)
548 for _, trial := range []struct {
550 normalizedPath string
552 {"/foo", "/foo"}, // boring case
553 {"/foo%5fbar", "/foo_bar"}, // _ must not be escaped
554 {"/foo%2fbar", "/foo/bar"}, // / must not be escaped
555 {"/(foo)", "/%28foo%29"}, // () must be escaped
556 {"/foo%5bbar", "/foo%5Bbar"}, // %XX must be uppercase
558 date := time.Now().UTC().Format("20060102T150405Z")
559 scope := "20200202/fakeregion/S3/aws4_request"
560 canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
561 c.Logf("canonicalRequest %q", canonicalRequest)
562 expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
563 c.Logf("expected stringToSign %q", expect)
565 req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
566 req.Header.Set("X-Amz-Date", date)
567 req.Host = "host.example.com"
569 obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
570 if !c.Check(err, check.IsNil) {
573 c.Check(obtained, check.Equals, expect)
577 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
578 stage := s.s3setup(c)
579 defer stage.teardown(c)
580 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
581 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
582 c.Check(err, check.IsNil)
583 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
584 req.URL.RawQuery = "versioning"
585 resp, err := http.DefaultClient.Do(req)
586 c.Assert(err, check.IsNil)
587 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
588 buf, err := ioutil.ReadAll(resp.Body)
589 c.Assert(err, check.IsNil)
590 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>\n")
594 // If there are no CommonPrefixes entries, the CommonPrefixes XML tag
595 // should not appear at all.
596 func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
597 stage := s.s3setup(c)
598 defer stage.teardown(c)
600 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
601 c.Assert(err, check.IsNil)
602 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
603 req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
604 resp, err := http.DefaultClient.Do(req)
605 c.Assert(err, check.IsNil)
606 buf, err := ioutil.ReadAll(resp.Body)
607 c.Assert(err, check.IsNil)
608 c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
611 // If there is no delimiter in the request, or the results are not
612 // truncated, the NextMarker XML tag should not appear in the response
614 func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
615 stage := s.s3setup(c)
616 defer stage.teardown(c)
618 for _, query := range []string{"prefix=e&delimiter=/", ""} {
619 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
620 c.Assert(err, check.IsNil)
621 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
622 req.URL.RawQuery = query
623 resp, err := http.DefaultClient.Do(req)
624 c.Assert(err, check.IsNil)
625 buf, err := ioutil.ReadAll(resp.Body)
626 c.Assert(err, check.IsNil)
627 c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`)
631 // List response should include KeyCount field.
632 func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
633 stage := s.s3setup(c)
634 defer stage.teardown(c)
636 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
637 c.Assert(err, check.IsNil)
638 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
639 req.URL.RawQuery = "prefix=&delimiter=/"
640 resp, err := http.DefaultClient.Do(req)
641 c.Assert(err, check.IsNil)
642 buf, err := ioutil.ReadAll(resp.Body)
643 c.Assert(err, check.IsNil)
644 c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
647 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
648 stage := s.s3setup(c)
649 defer stage.teardown(c)
652 for markers, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
655 stage.writeBigDirs(c, dirs, filesPerDir)
656 // Total # objects is:
657 // 2 file entries from s3setup (emptyfile and sailboat.txt)
658 // +1 fake "directory" marker from s3setup (emptydir) (if enabled)
659 // +dirs fake "directory" marker from writeBigDirs (dir0/, dir1/) (if enabled)
660 // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
661 s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
662 s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
663 s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
666 func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
667 c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.testServer.Config.cluster.Collections.S3FolderObjects)
668 expectPageSize := pageSize
669 if expectPageSize > 1000 {
670 expectPageSize = 1000
672 gotKeys := map[string]s3.Key{}
676 resp, err := bucket.List(prefix, "", nextMarker, pageSize)
677 if !c.Check(err, check.IsNil) {
680 c.Check(len(resp.Contents) <= expectPageSize, check.Equals, true)
681 if pages++; !c.Check(pages <= (expectFiles/expectPageSize)+1, check.Equals, true) {
684 for _, key := range resp.Contents {
685 gotKeys[key.Key] = key
686 if strings.Contains(key.Key, "sailboat.txt") {
687 c.Check(key.Size, check.Equals, int64(4))
690 if !resp.IsTruncated {
691 c.Check(resp.NextMarker, check.Equals, "")
694 if !c.Check(resp.NextMarker, check.Not(check.Equals), "") {
697 nextMarker = resp.NextMarker
699 c.Check(len(gotKeys), check.Equals, expectFiles)
702 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
703 for _, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
704 s.testS3CollectionListRollup(c)
708 func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
709 stage := s.s3setup(c)
710 defer stage.teardown(c)
714 stage.writeBigDirs(c, dirs, filesPerDir)
715 err := stage.collbucket.PutReader("dingbats", &bytes.Buffer{}, 0, "application/octet-stream", s3.Private, s3.Options{})
716 c.Assert(err, check.IsNil)
717 var allfiles []string
718 for marker := ""; ; {
719 resp, err := stage.collbucket.List("", "", marker, 20000)
720 c.Check(err, check.IsNil)
721 for _, key := range resp.Contents {
722 if len(allfiles) == 0 || allfiles[len(allfiles)-1] != key.Key {
723 allfiles = append(allfiles, key.Key)
726 marker = resp.NextMarker
732 if s.testServer.Config.cluster.Collections.S3FolderObjects {
735 c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
737 gotDirMarker := map[string]bool{}
738 for _, name := range allfiles {
739 isDirMarker := strings.HasSuffix(name, "/")
741 c.Check(isDirMarker, check.Equals, false, check.Commentf("name %q", name))
742 } else if isDirMarker {
743 gotDirMarker[name] = true
744 } else if i := strings.LastIndex(name, "/"); i >= 0 {
745 c.Check(gotDirMarker[name[:i+1]], check.Equals, true, check.Commentf("name %q", name))
746 gotDirMarker[name[:i+1]] = true // skip redundant complaints about this dir marker
750 for _, trial := range []struct {
765 {"dir0", "/", "dir0/file14.txt"}, // no commonprefixes
766 {"", "", "dir0/file14.txt"}, // middle page, skip walking dir1
767 {"", "", "dir1/file14.txt"}, // middle page, skip walking dir0
768 {"", "", "dir1/file498.txt"}, // last page of results
769 {"dir1/file", "", "dir1/file498.txt"}, // last page of results, with prefix
770 {"dir1/file", "/", "dir1/file498.txt"}, // last page of results, with prefix + delimiter
771 {"dir1", "Z", "dir1/file498.txt"}, // delimiter "Z" never appears
772 {"dir2", "/", ""}, // prefix "dir2" does not exist
775 c.Logf("\n\n=== trial %+v markers=%d", trial, markers)
778 resp, err := stage.collbucket.List(trial.prefix, trial.delimiter, trial.marker, maxKeys)
779 c.Check(err, check.IsNil)
780 if resp.IsTruncated && trial.delimiter == "" {
781 // goamz List method fills in the missing
782 // NextMarker field if resp.IsTruncated, so
783 // now we can't really tell whether it was
784 // sent by the server or by goamz. In cases
785 // where it should be empty but isn't, assume
786 // it's goamz's fault.
790 var expectKeys []string
791 var expectPrefixes []string
792 var expectNextMarker string
793 var expectTruncated bool
794 for _, key := range allfiles {
795 full := len(expectKeys)+len(expectPrefixes) >= maxKeys
796 if !strings.HasPrefix(key, trial.prefix) || key < trial.marker {
798 } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
799 prefix := key[:len(trial.prefix)+idx+1]
800 if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
801 // same prefix as previous key
803 expectNextMarker = key
804 expectTruncated = true
806 expectPrefixes = append(expectPrefixes, prefix)
809 if trial.delimiter != "" {
810 expectNextMarker = key
812 expectTruncated = true
815 expectKeys = append(expectKeys, key)
820 for _, key := range resp.Contents {
821 gotKeys = append(gotKeys, key.Key)
823 var gotPrefixes []string
824 for _, prefix := range resp.CommonPrefixes {
825 gotPrefixes = append(gotPrefixes, prefix)
827 commentf := check.Commentf("trial %+v markers=%d", trial, markers)
828 c.Check(gotKeys, check.DeepEquals, expectKeys, commentf)
829 c.Check(gotPrefixes, check.DeepEquals, expectPrefixes, commentf)
830 c.Check(resp.NextMarker, check.Equals, expectNextMarker, commentf)
831 c.Check(resp.IsTruncated, check.Equals, expectTruncated, commentf)
832 c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
836 // TestS3cmd checks compatibility with the s3cmd command line tool, if
837 // it's installed. As of Debian buster, s3cmd is only in backports, so
838 // `arvados-server install` don't install it, and this test skips if
839 // it's not installed.
840 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
841 if _, err := exec.LookPath("s3cmd"); err != nil {
842 c.Skip("s3cmd not found")
846 stage := s.s3setup(c)
847 defer stage.teardown(c)
849 cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.Addr, "--host-bucket="+s.testServer.Addr, "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
850 buf, err := cmd.CombinedOutput()
851 c.Check(err, check.IsNil)
852 c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
855 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
856 stage := s.s3setup(c)
857 defer stage.teardown(c)
859 hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
860 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
861 c.Check(body, check.Equals, "⛵\n")