X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/61d3aed1cd0286c7928167a84267c225938a37ec..8e57b2534bbf5ece2518c3c24ce79c5bb07adf3a:/services/keep-web/s3_test.go diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go index 851bee4b72..556f8ba2c2 100644 --- a/services/keep-web/s3_test.go +++ b/services/keep-web/s3_test.go @@ -17,6 +17,7 @@ import ( "net/url" "os" "os/exec" + "sort" "strings" "sync" "time" @@ -316,14 +317,14 @@ func (s *IntegrationSuite) TestS3PropertiesAsMetadata(c *check.C) { func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) { stage := s.s3setup(c) defer stage.teardown(c) - s.testS3PutObjectSuccess(c, stage.collbucket, "") + s.testS3PutObjectSuccess(c, stage.collbucket, "", stage.coll.UUID) } func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) { stage := s.s3setup(c) defer stage.teardown(c) - s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/") + s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/", stage.coll.UUID) } -func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string) { +func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string, collUUID string) { for _, trial := range []struct { path string size int @@ -367,7 +368,7 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, if !c.Check(err, check.NotNil) { continue } - c.Check(err.(*s3.Error).StatusCode, check.Equals, 404) + c.Check(err.(*s3.Error).StatusCode, check.Equals, http.StatusNotFound) c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`) if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) { continue @@ -390,6 +391,14 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, c.Check(err, check.IsNil) c.Check(buf2, check.HasLen, len(buf)) c.Check(bytes.Equal(buf, buf2), check.Equals, true) + + // Check that the change is immediately visible via + // (non-S3) webdav request. + _, resp := s.do("GET", "http://"+collUUID+".keep-web.example/"+trial.path, arvadostest.ActiveTokenV2, nil) + c.Check(resp.Code, check.Equals, http.StatusOK) + if !strings.HasSuffix(trial.path, "/") { + c.Check(resp.Body.Len(), check.Equals, trial.size) + } } } @@ -809,8 +818,8 @@ func (s *IntegrationSuite) TestS3CollectionList(c *check.C) { var markers int for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} { - dirs := 2 - filesPerDir := 1001 + dirs := 2000 + filesPerDir := 2 stage.writeBigDirs(c, dirs, filesPerDir) // Total # objects is: // 2 file entries from s3setup (emptyfile and sailboat.txt) @@ -819,6 +828,7 @@ func (s *IntegrationSuite) TestS3CollectionList(c *check.C) { // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.) s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs) s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs) + s.testS3List(c, stage.collbucket, "", 51, markers+2+(filesPerDir+markers)*dirs) s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers) } } @@ -841,6 +851,9 @@ func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix stri break } for _, key := range resp.Contents { + if _, dup := gotKeys[key.Key]; dup { + c.Errorf("got duplicate key %q on page %d", key.Key, pages) + } gotKeys[key.Key] = key if strings.Contains(key.Key, "sailboat.txt") { c.Check(key.Size, check.Equals, int64(4)) @@ -855,7 +868,16 @@ func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix stri } nextMarker = resp.NextMarker } - c.Check(len(gotKeys), check.Equals, expectFiles) + if !c.Check(len(gotKeys), check.Equals, expectFiles) { + var sorted []string + for k := range gotKeys { + sorted = append(sorted, k) + } + sort.Strings(sorted) + for _, k := range sorted { + c.Logf("got %s", k) + } + } } func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) { @@ -952,28 +974,31 @@ func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) { var expectTruncated bool for _, key := range allfiles { full := len(expectKeys)+len(expectPrefixes) >= maxKeys - if !strings.HasPrefix(key, trial.prefix) || key < trial.marker { + if !strings.HasPrefix(key, trial.prefix) || key <= trial.marker { continue } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 { prefix := key[:len(trial.prefix)+idx+1] if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix { // same prefix as previous key } else if full { - expectNextMarker = key expectTruncated = true } else { expectPrefixes = append(expectPrefixes, prefix) + expectNextMarker = prefix } } else if full { - if trial.delimiter != "" { - expectNextMarker = key - } expectTruncated = true break } else { expectKeys = append(expectKeys, key) + if trial.delimiter != "" { + expectNextMarker = key + } } } + if !expectTruncated { + expectNextMarker = "" + } var gotKeys []string for _, key := range resp.Contents {