1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
21 "git.arvados.org/arvados.git/sdk/go/arvados"
22 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
23 "git.arvados.org/arvados.git/sdk/go/arvadostest"
24 "git.arvados.org/arvados.git/sdk/go/keepclient"
25 "github.com/AdRoll/goamz/aws"
26 "github.com/AdRoll/goamz/s3"
27 check "gopkg.in/check.v1"
32 ac *arvadosclient.ArvadosClient
33 kc *keepclient.KeepClient
36 coll arvados.Collection
40 func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
41 var proj arvados.Group
42 var coll arvados.Collection
43 arv := arvados.NewClientFromEnv()
44 arv.AuthToken = arvadostest.ActiveToken
45 err := arv.RequestAndDecode(&proj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
46 "group": map[string]interface{}{
47 "group_class": "project",
48 "name": "keep-web s3 test",
50 "ensure_unique_name": true,
52 c.Assert(err, check.IsNil)
53 err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
54 "owner_uuid": proj.UUID,
55 "name": "keep-web s3 test collection",
56 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
58 c.Assert(err, check.IsNil)
59 ac, err := arvadosclient.New(arv)
60 c.Assert(err, check.IsNil)
61 kc, err := keepclient.MakeKeepClient(ac)
62 c.Assert(err, check.IsNil)
63 fs, err := coll.FileSystem(arv, kc)
64 c.Assert(err, check.IsNil)
65 f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
66 c.Assert(err, check.IsNil)
67 _, err = f.Write([]byte("⛵\n"))
68 c.Assert(err, check.IsNil)
70 c.Assert(err, check.IsNil)
72 c.Assert(err, check.IsNil)
73 err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
74 c.Assert(err, check.IsNil)
76 auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
78 Name: s.testServer.Addr,
79 S3Endpoint: "http://" + s.testServer.Addr,
81 client := s3.New(*auth, region)
82 client.Signature = aws.V4Signature
88 projbucket: &s3.Bucket{
93 collbucket: &s3.Bucket{
100 func (stage s3stage) teardown(c *check.C) {
101 if stage.coll.UUID != "" {
102 err := stage.arv.RequestAndDecode(&stage.coll, "DELETE", "arvados/v1/collections/"+stage.coll.UUID, nil, nil)
103 c.Check(err, check.IsNil)
105 if stage.proj.UUID != "" {
106 err := stage.arv.RequestAndDecode(&stage.proj, "DELETE", "arvados/v1/groups/"+stage.proj.UUID, nil, nil)
107 c.Check(err, check.IsNil)
111 func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
112 stage := s.s3setup(c)
113 defer stage.teardown(c)
115 bucket := stage.collbucket
116 for _, trial := range []struct {
122 {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
123 {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
124 {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
125 {false, aws.V2Signature, "none", "none"},
126 {false, aws.V2Signature, "none", arvadostest.ActiveToken},
128 {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
129 {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
130 {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
131 {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
132 {false, aws.V4Signature, arvadostest.ActiveToken, ""},
133 {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
134 {false, aws.V4Signature, "none", arvadostest.ActiveToken},
135 {false, aws.V4Signature, "none", "none"},
138 bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
139 bucket.S3.Signature = trial.signature
140 _, err := bucket.GetReader("emptyfile")
142 c.Check(err, check.IsNil)
144 c.Check(err, check.NotNil)
149 func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
150 stage := s.s3setup(c)
151 defer stage.teardown(c)
153 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
154 c.Logf("bucket %s", bucket.Name)
155 exists, err := bucket.Exists("")
156 c.Check(err, check.IsNil)
157 c.Check(exists, check.Equals, true)
161 func (s *IntegrationSuite) TestS3CollectionGetObject(c *check.C) {
162 stage := s.s3setup(c)
163 defer stage.teardown(c)
164 s.testS3GetObject(c, stage.collbucket, "")
166 func (s *IntegrationSuite) TestS3ProjectGetObject(c *check.C) {
167 stage := s.s3setup(c)
168 defer stage.teardown(c)
169 s.testS3GetObject(c, stage.projbucket, stage.coll.Name+"/")
171 func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix string) {
172 rdr, err := bucket.GetReader(prefix + "emptyfile")
173 c.Assert(err, check.IsNil)
174 buf, err := ioutil.ReadAll(rdr)
175 c.Check(err, check.IsNil)
176 c.Check(len(buf), check.Equals, 0)
178 c.Check(err, check.IsNil)
181 rdr, err = bucket.GetReader(prefix + "missingfile")
182 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
183 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
184 c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
187 exists, err := bucket.Exists(prefix + "missingfile")
188 c.Check(err, check.IsNil)
189 c.Check(exists, check.Equals, false)
192 rdr, err = bucket.GetReader(prefix + "sailboat.txt")
193 c.Assert(err, check.IsNil)
194 buf, err = ioutil.ReadAll(rdr)
195 c.Check(err, check.IsNil)
196 c.Check(buf, check.DeepEquals, []byte("⛵\n"))
198 c.Check(err, check.IsNil)
201 resp, err := bucket.Head(prefix+"sailboat.txt", nil)
202 c.Check(err, check.IsNil)
203 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
204 c.Check(resp.ContentLength, check.Equals, int64(4))
207 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
208 stage := s.s3setup(c)
209 defer stage.teardown(c)
210 s.testS3PutObjectSuccess(c, stage.collbucket, "")
212 func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) {
213 stage := s.s3setup(c)
214 defer stage.teardown(c)
215 s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/")
217 func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string) {
218 for _, trial := range []struct {
226 contentType: "application/octet-stream",
228 path: "newdir/newfile",
230 contentType: "application/octet-stream",
232 path: "newdir1/newdir2/newfile",
234 contentType: "application/octet-stream",
236 path: "newdir1/newdir2/newdir3/",
238 contentType: "application/x-directory",
241 c.Logf("=== %v", trial)
243 objname := prefix + trial.path
245 _, err := bucket.GetReader(objname)
246 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
247 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
248 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
250 buf := make([]byte, trial.size)
253 err = bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
254 c.Check(err, check.IsNil)
256 rdr, err := bucket.GetReader(objname)
257 if strings.HasSuffix(trial.path, "/") && !s.testServer.Config.cluster.Collections.S3FolderObjects {
258 c.Check(err, check.NotNil)
260 } else if !c.Check(err, check.IsNil) {
263 buf2, err := ioutil.ReadAll(rdr)
264 c.Check(err, check.IsNil)
265 c.Check(buf2, check.HasLen, len(buf))
266 c.Check(bytes.Equal(buf, buf2), check.Equals, true)
270 func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
271 stage := s.s3setup(c)
272 defer stage.teardown(c)
273 bucket := stage.projbucket
275 for _, trial := range []struct {
283 contentType: "application/octet-stream",
285 path: "newdir/newfile",
287 contentType: "application/octet-stream",
291 contentType: "application/x-directory",
294 c.Logf("=== %v", trial)
296 _, err := bucket.GetReader(trial.path)
297 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
298 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
299 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
301 buf := make([]byte, trial.size)
304 err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
305 c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
306 c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
307 c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
309 _, err = bucket.GetReader(trial.path)
310 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
311 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
312 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
316 func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) {
317 stage := s.s3setup(c)
318 defer stage.teardown(c)
319 s.testS3DeleteObject(c, stage.collbucket, "")
321 func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) {
322 stage := s.s3setup(c)
323 defer stage.teardown(c)
324 s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
326 func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
327 s.testServer.Config.cluster.Collections.S3FolderObjects = true
328 for _, trial := range []struct {
339 objname := prefix + trial.path
340 comment := check.Commentf("objname %q", objname)
342 err := bucket.Del(objname)
343 if trial.path == "/" {
344 c.Check(err, check.NotNil)
347 c.Check(err, check.IsNil, comment)
348 _, err = bucket.GetReader(objname)
349 c.Check(err, check.NotNil, comment)
353 func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) {
354 stage := s.s3setup(c)
355 defer stage.teardown(c)
356 s.testS3PutObjectFailure(c, stage.collbucket, "")
358 func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
359 stage := s.s3setup(c)
360 defer stage.teardown(c)
361 s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
363 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
364 s.testServer.Config.cluster.Collections.S3FolderObjects = false
366 // Can't use V4 signature for these tests, because
367 // double-slash is incorrectly cleaned by the aws.V4Signature,
368 // resulting in a "bad signature" error. (Cleaning the path is
369 // appropriate for other services, but not in S3 where object
370 // names "foo//bar" and "foo/bar" are semantically different.)
371 bucket.S3.Auth = *(aws.NewAuth(arvadostest.ActiveToken, "none", "", time.Now().Add(time.Hour)))
372 bucket.S3.Signature = aws.V2Signature
374 var wg sync.WaitGroup
375 for _, trial := range []struct {
379 path: "emptyfile/newname", // emptyfile exists, see s3setup()
381 path: "emptyfile/", // emptyfile exists, see s3setup()
383 path: "emptydir", // dir already exists, see s3setup()
406 c.Logf("=== %v", trial)
408 objname := prefix + trial.path
410 buf := make([]byte, 1234)
413 err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
414 if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
418 if objname != "" && objname != "/" {
419 _, err = bucket.GetReader(objname)
420 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
421 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
422 c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
429 func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
430 fs, err := stage.coll.FileSystem(stage.arv, stage.kc)
431 c.Assert(err, check.IsNil)
432 for d := 0; d < dirs; d++ {
433 dir := fmt.Sprintf("dir%d", d)
434 c.Assert(fs.Mkdir(dir, 0755), check.IsNil)
435 for i := 0; i < filesPerDir; i++ {
436 f, err := fs.OpenFile(fmt.Sprintf("%s/file%d.txt", dir, i), os.O_CREATE|os.O_WRONLY, 0644)
437 c.Assert(err, check.IsNil)
438 c.Assert(f.Close(), check.IsNil)
441 c.Assert(fs.Sync(), check.IsNil)
444 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
445 stage := s.s3setup(c)
446 defer stage.teardown(c)
447 for _, trial := range []struct {
452 responseRegexp []string
455 url: "https://" + stage.collbucket.Name + ".example.com/",
457 responseCode: http.StatusOK,
458 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
461 url: "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
463 responseCode: http.StatusOK,
464 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
467 url: "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
469 responseCode: http.StatusOK,
470 responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
473 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
475 responseCode: http.StatusOK,
476 responseRegexp: []string{`⛵\n`},
479 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
482 responseCode: http.StatusOK,
485 url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
487 responseCode: http.StatusOK,
488 responseRegexp: []string{`boop`},
491 url, err := url.Parse(trial.url)
492 c.Assert(err, check.IsNil)
493 req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
494 c.Assert(err, check.IsNil)
495 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
496 rr := httptest.NewRecorder()
497 s.testServer.Server.Handler.ServeHTTP(rr, req)
499 c.Check(resp.StatusCode, check.Equals, trial.responseCode)
500 body, err := ioutil.ReadAll(resp.Body)
501 c.Assert(err, check.IsNil)
502 for _, re := range trial.responseRegexp {
503 c.Check(string(body), check.Matches, re)
508 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
509 stage := s.s3setup(c)
510 defer stage.teardown(c)
511 for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
512 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
513 c.Check(err, check.IsNil)
514 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
515 req.URL.RawQuery = "versioning"
516 resp, err := http.DefaultClient.Do(req)
517 c.Assert(err, check.IsNil)
518 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
519 buf, err := ioutil.ReadAll(resp.Body)
520 c.Assert(err, check.IsNil)
521 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>\n")
525 // If there are no CommonPrefixes entries, the CommonPrefixes XML tag
526 // should not appear at all.
527 func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
528 stage := s.s3setup(c)
529 defer stage.teardown(c)
531 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
532 c.Assert(err, check.IsNil)
533 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
534 req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
535 resp, err := http.DefaultClient.Do(req)
536 c.Assert(err, check.IsNil)
537 buf, err := ioutil.ReadAll(resp.Body)
538 c.Assert(err, check.IsNil)
539 c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
542 // If there is no delimiter in the request, or the results are not
543 // truncated, the NextMarker XML tag should not appear in the response
545 func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
546 stage := s.s3setup(c)
547 defer stage.teardown(c)
549 for _, query := range []string{"prefix=e&delimiter=/", ""} {
550 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
551 c.Assert(err, check.IsNil)
552 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
553 req.URL.RawQuery = query
554 resp, err := http.DefaultClient.Do(req)
555 c.Assert(err, check.IsNil)
556 buf, err := ioutil.ReadAll(resp.Body)
557 c.Assert(err, check.IsNil)
558 c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`)
562 // List response should include KeyCount field.
563 func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
564 stage := s.s3setup(c)
565 defer stage.teardown(c)
567 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
568 c.Assert(err, check.IsNil)
569 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
570 req.URL.RawQuery = "prefix=&delimiter=/"
571 resp, err := http.DefaultClient.Do(req)
572 c.Assert(err, check.IsNil)
573 buf, err := ioutil.ReadAll(resp.Body)
574 c.Assert(err, check.IsNil)
575 c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
578 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
579 stage := s.s3setup(c)
580 defer stage.teardown(c)
583 for markers, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
586 stage.writeBigDirs(c, dirs, filesPerDir)
587 // Total # objects is:
588 // 2 file entries from s3setup (emptyfile and sailboat.txt)
589 // +1 fake "directory" marker from s3setup (emptydir) (if enabled)
590 // +dirs fake "directory" marker from writeBigDirs (dir0/, dir1/) (if enabled)
591 // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
592 s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
593 s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
594 s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
597 func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
598 c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.testServer.Config.cluster.Collections.S3FolderObjects)
599 expectPageSize := pageSize
600 if expectPageSize > 1000 {
601 expectPageSize = 1000
603 gotKeys := map[string]s3.Key{}
607 resp, err := bucket.List(prefix, "", nextMarker, pageSize)
608 if !c.Check(err, check.IsNil) {
611 c.Check(len(resp.Contents) <= expectPageSize, check.Equals, true)
612 if pages++; !c.Check(pages <= (expectFiles/expectPageSize)+1, check.Equals, true) {
615 for _, key := range resp.Contents {
616 gotKeys[key.Key] = key
617 if strings.Contains(key.Key, "sailboat.txt") {
618 c.Check(key.Size, check.Equals, int64(4))
621 if !resp.IsTruncated {
622 c.Check(resp.NextMarker, check.Equals, "")
625 if !c.Check(resp.NextMarker, check.Not(check.Equals), "") {
628 nextMarker = resp.NextMarker
630 c.Check(len(gotKeys), check.Equals, expectFiles)
633 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
634 for _, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
635 s.testS3CollectionListRollup(c)
639 func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
640 stage := s.s3setup(c)
641 defer stage.teardown(c)
645 stage.writeBigDirs(c, dirs, filesPerDir)
646 err := stage.collbucket.PutReader("dingbats", &bytes.Buffer{}, 0, "application/octet-stream", s3.Private, s3.Options{})
647 c.Assert(err, check.IsNil)
648 var allfiles []string
649 for marker := ""; ; {
650 resp, err := stage.collbucket.List("", "", marker, 20000)
651 c.Check(err, check.IsNil)
652 for _, key := range resp.Contents {
653 if len(allfiles) == 0 || allfiles[len(allfiles)-1] != key.Key {
654 allfiles = append(allfiles, key.Key)
657 marker = resp.NextMarker
663 if s.testServer.Config.cluster.Collections.S3FolderObjects {
666 c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
668 gotDirMarker := map[string]bool{}
669 for _, name := range allfiles {
670 isDirMarker := strings.HasSuffix(name, "/")
672 c.Check(isDirMarker, check.Equals, false, check.Commentf("name %q", name))
673 } else if isDirMarker {
674 gotDirMarker[name] = true
675 } else if i := strings.LastIndex(name, "/"); i >= 0 {
676 c.Check(gotDirMarker[name[:i+1]], check.Equals, true, check.Commentf("name %q", name))
677 gotDirMarker[name[:i+1]] = true // skip redundant complaints about this dir marker
681 for _, trial := range []struct {
696 {"dir0", "/", "dir0/file14.txt"}, // no commonprefixes
697 {"", "", "dir0/file14.txt"}, // middle page, skip walking dir1
698 {"", "", "dir1/file14.txt"}, // middle page, skip walking dir0
699 {"", "", "dir1/file498.txt"}, // last page of results
700 {"dir1/file", "", "dir1/file498.txt"}, // last page of results, with prefix
701 {"dir1/file", "/", "dir1/file498.txt"}, // last page of results, with prefix + delimiter
702 {"dir1", "Z", "dir1/file498.txt"}, // delimiter "Z" never appears
703 {"dir2", "/", ""}, // prefix "dir2" does not exist
706 c.Logf("\n\n=== trial %+v markers=%d", trial, markers)
709 resp, err := stage.collbucket.List(trial.prefix, trial.delimiter, trial.marker, maxKeys)
710 c.Check(err, check.IsNil)
711 if resp.IsTruncated && trial.delimiter == "" {
712 // goamz List method fills in the missing
713 // NextMarker field if resp.IsTruncated, so
714 // now we can't really tell whether it was
715 // sent by the server or by goamz. In cases
716 // where it should be empty but isn't, assume
717 // it's goamz's fault.
721 var expectKeys []string
722 var expectPrefixes []string
723 var expectNextMarker string
724 var expectTruncated bool
725 for _, key := range allfiles {
726 full := len(expectKeys)+len(expectPrefixes) >= maxKeys
727 if !strings.HasPrefix(key, trial.prefix) || key < trial.marker {
729 } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
730 prefix := key[:len(trial.prefix)+idx+1]
731 if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
732 // same prefix as previous key
734 expectNextMarker = key
735 expectTruncated = true
737 expectPrefixes = append(expectPrefixes, prefix)
740 if trial.delimiter != "" {
741 expectNextMarker = key
743 expectTruncated = true
746 expectKeys = append(expectKeys, key)
751 for _, key := range resp.Contents {
752 gotKeys = append(gotKeys, key.Key)
754 var gotPrefixes []string
755 for _, prefix := range resp.CommonPrefixes {
756 gotPrefixes = append(gotPrefixes, prefix)
758 commentf := check.Commentf("trial %+v markers=%d", trial, markers)
759 c.Check(gotKeys, check.DeepEquals, expectKeys, commentf)
760 c.Check(gotPrefixes, check.DeepEquals, expectPrefixes, commentf)
761 c.Check(resp.NextMarker, check.Equals, expectNextMarker, commentf)
762 c.Check(resp.IsTruncated, check.Equals, expectTruncated, commentf)
763 c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
767 // TestS3cmd checks compatibility with the s3cmd command line tool, if
768 // it's installed. As of Debian buster, s3cmd is only in backports, so
769 // `arvados-server install` don't install it, and this test skips if
770 // it's not installed.
771 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
772 if _, err := exec.LookPath("s3cmd"); err != nil {
773 c.Skip("s3cmd not found")
777 stage := s.s3setup(c)
778 defer stage.teardown(c)
780 cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.Addr, "--host-bucket="+s.testServer.Addr, "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
781 buf, err := cmd.CombinedOutput()
782 c.Check(err, check.IsNil)
783 c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
786 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
787 stage := s.s3setup(c)
788 defer stage.teardown(c)
790 hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
791 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
792 c.Check(body, check.Equals, "⛵\n")