20726: Fix ListObjects[V2] pages duplicating last item on next page.

[arvados.git] / services / keep-web / s3_test.go

diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index aa91d82ae36ab6c01cb50ef7b7dd944af16b4236..556f8ba2c2661b8c06902ed20de29456062ee0bc 100644 (file)
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -17,6 +17,7 @@ import (
        "net/url"
        "os"
        "os/exec"
+       "sort"
        "strings"
        "sync"
        "time"
@@ -817,8 +818,8 @@ func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
 
        var markers int
        for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
-               dirs := 2
-               filesPerDir := 1001
+               dirs := 2000
+               filesPerDir := 2
                stage.writeBigDirs(c, dirs, filesPerDir)
                // Total # objects is:
                //                 2 file entries from s3setup (emptyfile and sailboat.txt)
@@ -827,6 +828,7 @@ func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
                // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
                s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
                s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
+               s.testS3List(c, stage.collbucket, "", 51, markers+2+(filesPerDir+markers)*dirs)
                s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
        }
 }
@@ -849,6 +851,9 @@ func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix stri
                        break
                }
                for _, key := range resp.Contents {
+                       if _, dup := gotKeys[key.Key]; dup {
+                               c.Errorf("got duplicate key %q on page %d", key.Key, pages)
+                       }
                        gotKeys[key.Key] = key
                        if strings.Contains(key.Key, "sailboat.txt") {
                                c.Check(key.Size, check.Equals, int64(4))
@@ -863,7 +868,16 @@ func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix stri
                }
                nextMarker = resp.NextMarker
        }
-       c.Check(len(gotKeys), check.Equals, expectFiles)
+       if !c.Check(len(gotKeys), check.Equals, expectFiles) {
+               var sorted []string
+               for k := range gotKeys {
+                       sorted = append(sorted, k)
+               }
+               sort.Strings(sorted)
+               for _, k := range sorted {
+                       c.Logf("got %s", k)
+               }
+       }
 }
 
 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
@@ -960,28 +974,31 @@ func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
                var expectTruncated bool
                for _, key := range allfiles {
                        full := len(expectKeys)+len(expectPrefixes) >= maxKeys
-                       if !strings.HasPrefix(key, trial.prefix) || key < trial.marker {
+                       if !strings.HasPrefix(key, trial.prefix) || key <= trial.marker {
                                continue
                        } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
                                prefix := key[:len(trial.prefix)+idx+1]
                                if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
                                        // same prefix as previous key
                                } else if full {
-                                       expectNextMarker = key
                                        expectTruncated = true
                                } else {
                                        expectPrefixes = append(expectPrefixes, prefix)
+                                       expectNextMarker = prefix
                                }
                        } else if full {
-                               if trial.delimiter != "" {
-                                       expectNextMarker = key
-                               }
                                expectTruncated = true
                                break
                        } else {
                                expectKeys = append(expectKeys, key)
+                               if trial.delimiter != "" {
+                                       expectNextMarker = key
+                               }
                        }
                }
+               if !expectTruncated {
+                       expectNextMarker = ""
+               }
 
                var gotKeys []string
                for _, key := range resp.Contents {