X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a9be3117466506dffc39617be1c58406c5914e4b..8e57b2534bbf5ece2518c3c24ce79c5bb07adf3a:/services/keep-web/s3_test.go

diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index aa91d82ae3..556f8ba2c2 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -17,6 +17,7 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -817,8 +818,8 @@ func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
 
 	var markers int
 	for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
-		dirs := 2
-		filesPerDir := 1001
+		dirs := 2000
+		filesPerDir := 2
 		stage.writeBigDirs(c, dirs, filesPerDir)
 		// Total # objects is:
 		//                 2 file entries from s3setup (emptyfile and sailboat.txt)
@@ -827,6 +828,7 @@ func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
 		// +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
 		s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
 		s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
+		s.testS3List(c, stage.collbucket, "", 51, markers+2+(filesPerDir+markers)*dirs)
 		s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
 	}
 }
@@ -849,6 +851,9 @@ func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix stri
 			break
 		}
 		for _, key := range resp.Contents {
+			if _, dup := gotKeys[key.Key]; dup {
+				c.Errorf("got duplicate key %q on page %d", key.Key, pages)
+			}
 			gotKeys[key.Key] = key
 			if strings.Contains(key.Key, "sailboat.txt") {
 				c.Check(key.Size, check.Equals, int64(4))
@@ -863,7 +868,16 @@ func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix stri
 		}
 		nextMarker = resp.NextMarker
 	}
-	c.Check(len(gotKeys), check.Equals, expectFiles)
+	if !c.Check(len(gotKeys), check.Equals, expectFiles) {
+		var sorted []string
+		for k := range gotKeys {
+			sorted = append(sorted, k)
+		}
+		sort.Strings(sorted)
+		for _, k := range sorted {
+			c.Logf("got %s", k)
+		}
+	}
 }
 
 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
@@ -960,28 +974,31 @@ func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
 		var expectTruncated bool
 		for _, key := range allfiles {
 			full := len(expectKeys)+len(expectPrefixes) >= maxKeys
-			if !strings.HasPrefix(key, trial.prefix) || key < trial.marker {
+			if !strings.HasPrefix(key, trial.prefix) || key <= trial.marker {
 				continue
 			} else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
 				prefix := key[:len(trial.prefix)+idx+1]
 				if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
 					// same prefix as previous key
 				} else if full {
-					expectNextMarker = key
 					expectTruncated = true
 				} else {
 					expectPrefixes = append(expectPrefixes, prefix)
+					expectNextMarker = prefix
 				}
 			} else if full {
-				if trial.delimiter != "" {
-					expectNextMarker = key
-				}
 				expectTruncated = true
 				break
 			} else {
 				expectKeys = append(expectKeys, key)
+				if trial.delimiter != "" {
+					expectNextMarker = key
+				}
 			}
 		}
+		if !expectTruncated {
+			expectNextMarker = ""
+		}
 
 		var gotKeys []string
 		for _, key := range resp.Contents {