X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/fd26f5508b755061046727fb652aa4141029e8b9..28f250ee0a43e873760d43b39119e5710de75fe8:/services/keepstore/s3_volume_test.go diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go index 63b186220c..5c639d6295 100644 --- a/services/keepstore/s3_volume_test.go +++ b/services/keepstore/s3_volume_test.go @@ -1,18 +1,28 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( "bytes" "context" "crypto/md5" + "encoding/json" "fmt" - "io/ioutil" "log" + "net/http" + "net/http/httptest" "os" + "strings" "time" "git.curoverse.com/arvados.git/sdk/go/arvados" + "git.curoverse.com/arvados.git/sdk/go/ctxlog" "github.com/AdRoll/goamz/s3" "github.com/AdRoll/goamz/s3/s3test" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" check "gopkg.in/check.v1" ) @@ -31,34 +41,41 @@ func (c *fakeClock) Now() time.Time { return *c.now } -func init() { - // Deleting isn't safe from races, but if it's turned on - // anyway we do expect it to pass the generic volume tests. - s3UnsafeDelete = true -} - var _ = check.Suite(&StubbedS3Suite{}) type StubbedS3Suite struct { - volumes []*TestableS3Volume + s3server *httptest.Server + cluster *arvados.Cluster + handler *handler + volumes []*TestableS3Volume +} + +func (s *StubbedS3Suite) SetUpTest(c *check.C) { + s.s3server = nil + s.cluster = testCluster(c) + s.cluster.Volumes = map[string]arvados.Volume{ + "zzzzz-nyw5e-000000000000000": {Driver: "S3"}, + "zzzzz-nyw5e-111111111111111": {Driver: "S3"}, + } + s.handler = &handler{} } func (s *StubbedS3Suite) TestGeneric(c *check.C) { - DoGenericVolumeTests(c, func(t TB) TestableVolume { + DoGenericVolumeTests(c, false, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume { // Use a negative raceWindow so s3test's 1-second // timestamp precision doesn't confuse fixRace. - return s.newTestableVolume(c, -2*time.Second, false, 2) + return s.newTestableVolume(c, cluster, volume, metrics, -2*time.Second) }) } func (s *StubbedS3Suite) TestGenericReadOnly(c *check.C) { - DoGenericVolumeTests(c, func(t TB) TestableVolume { - return s.newTestableVolume(c, -2*time.Second, true, 2) + DoGenericVolumeTests(c, true, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume { + return s.newTestableVolume(c, cluster, volume, metrics, -2*time.Second) }) } func (s *StubbedS3Suite) TestIndex(c *check.C) { - v := s.newTestableVolume(c, 0, false, 2) + v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 0) v.IndexPageSize = 3 for i := 0; i < 256; i++ { v.PutRaw(fmt.Sprintf("%02x%030x", i, i), []byte{102, 111, 111}) @@ -82,15 +99,129 @@ func (s *StubbedS3Suite) TestIndex(c *check.C) { } } +func (s *StubbedS3Suite) TestStats(c *check.C) { + v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 5*time.Minute) + stats := func() string { + buf, err := json.Marshal(v.InternalStats()) + c.Check(err, check.IsNil) + return string(buf) + } + + c.Check(stats(), check.Matches, `.*"Ops":0,.*`) + + loc := "acbd18db4cc2f85cedef654fccc4a4d8" + _, err := v.Get(context.Background(), loc, make([]byte, 3)) + c.Check(err, check.NotNil) + c.Check(stats(), check.Matches, `.*"Ops":[^0],.*`) + c.Check(stats(), check.Matches, `.*"\*s3.Error 404 [^"]*":[^0].*`) + c.Check(stats(), check.Matches, `.*"InBytes":0,.*`) + + err = v.Put(context.Background(), loc, []byte("foo")) + c.Check(err, check.IsNil) + c.Check(stats(), check.Matches, `.*"OutBytes":3,.*`) + c.Check(stats(), check.Matches, `.*"PutOps":2,.*`) + + _, err = v.Get(context.Background(), loc, make([]byte, 3)) + c.Check(err, check.IsNil) + _, err = v.Get(context.Background(), loc, make([]byte, 3)) + c.Check(err, check.IsNil) + c.Check(stats(), check.Matches, `.*"InBytes":6,.*`) +} + +type blockingHandler struct { + requested chan *http.Request + unblock chan struct{} +} + +func (h *blockingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == "PUT" && !strings.Contains(strings.Trim(r.URL.Path, "/"), "/") { + // Accept PutBucket ("PUT /bucketname/"), called by + // newTestableVolume + return + } + if h.requested != nil { + h.requested <- r + } + if h.unblock != nil { + <-h.unblock + } + http.Error(w, "nothing here", http.StatusNotFound) +} + +func (s *StubbedS3Suite) TestGetContextCancel(c *check.C) { + loc := "acbd18db4cc2f85cedef654fccc4a4d8" + buf := make([]byte, 3) + + s.testContextCancel(c, func(ctx context.Context, v *TestableS3Volume) error { + _, err := v.Get(ctx, loc, buf) + return err + }) +} + +func (s *StubbedS3Suite) TestCompareContextCancel(c *check.C) { + loc := "acbd18db4cc2f85cedef654fccc4a4d8" + buf := []byte("bar") + + s.testContextCancel(c, func(ctx context.Context, v *TestableS3Volume) error { + return v.Compare(ctx, loc, buf) + }) +} + +func (s *StubbedS3Suite) TestPutContextCancel(c *check.C) { + loc := "acbd18db4cc2f85cedef654fccc4a4d8" + buf := []byte("foo") + + s.testContextCancel(c, func(ctx context.Context, v *TestableS3Volume) error { + return v.Put(ctx, loc, buf) + }) +} + +func (s *StubbedS3Suite) testContextCancel(c *check.C, testFunc func(context.Context, *TestableS3Volume) error) { + handler := &blockingHandler{} + s.s3server = httptest.NewServer(handler) + defer s.s3server.Close() + + v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 5*time.Minute) + + ctx, cancel := context.WithCancel(context.Background()) + + handler.requested = make(chan *http.Request) + handler.unblock = make(chan struct{}) + defer close(handler.unblock) + + doneFunc := make(chan struct{}) + go func() { + err := testFunc(ctx, v) + c.Check(err, check.Equals, context.Canceled) + close(doneFunc) + }() + + timeout := time.After(10 * time.Second) + + // Wait for the stub server to receive a request, meaning + // Get() is waiting for an s3 operation. + select { + case <-timeout: + c.Fatal("timed out waiting for test func to call our handler") + case <-doneFunc: + c.Fatal("test func finished without even calling our handler!") + case <-handler.requested: + } + + cancel() + + select { + case <-timeout: + c.Fatal("timed out") + case <-doneFunc: + } +} + func (s *StubbedS3Suite) TestBackendStates(c *check.C) { - defer func(tl, bs arvados.Duration) { - theConfig.TrashLifetime = tl - theConfig.BlobSignatureTTL = bs - }(theConfig.TrashLifetime, theConfig.BlobSignatureTTL) - theConfig.TrashLifetime.Set("1h") - theConfig.BlobSignatureTTL.Set("1h") - - v := s.newTestableVolume(c, 5*time.Minute, false, 2) + s.cluster.Collections.BlobTrashLifetime.Set("1h") + s.cluster.Collections.BlobSigningTTL.Set("1h") + + v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 5*time.Minute) var none time.Time putS3Obj := func(t time.Time, key string, data []byte) { @@ -185,12 +316,12 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) { false, false, false, true, false, false, }, { - "Erroneously trashed during a race, detected before TrashLifetime", + "Erroneously trashed during a race, detected before BlobTrashLifetime", none, t0.Add(-30 * time.Minute), t0.Add(-29 * time.Minute), true, false, true, true, true, false, }, { - "Erroneously trashed during a race, rescue during EmptyTrash despite reaching TrashLifetime", + "Erroneously trashed during a race, rescue during EmptyTrash despite reaching BlobTrashLifetime", none, t0.Add(-90 * time.Minute), t0.Add(-89 * time.Minute), true, false, true, true, true, false, }, @@ -231,7 +362,7 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) { } // Call Trash, then check canTrash and canGetAfterTrash - loc, blk = setupScenario() + loc, _ = setupScenario() err = v.Trash(loc) c.Check(err == nil, check.Equals, scenario.canTrash) _, err = v.Get(context.Background(), loc, buf) @@ -241,7 +372,7 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) { } // Call Untrash, then check canUntrash - loc, blk = setupScenario() + loc, _ = setupScenario() err = v.Untrash(loc) c.Check(err == nil, check.Equals, scenario.canUntrash) if scenario.dataT != none || scenario.trashT != none { @@ -255,7 +386,7 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) { // Call EmptyTrash, then check haveTrashAfterEmpty and // freshAfterEmpty - loc, blk = setupScenario() + loc, _ = setupScenario() v.EmptyTrash() _, err = v.bucket.Head("trash/"+loc, nil) c.Check(err == nil, check.Equals, scenario.haveTrashAfterEmpty) @@ -285,38 +416,39 @@ type TestableS3Volume struct { serverClock *fakeClock } -func (s *StubbedS3Suite) newTestableVolume(c *check.C, raceWindow time.Duration, readonly bool, replication int) *TestableS3Volume { +func (s *StubbedS3Suite) newTestableVolume(c *check.C, cluster *arvados.Cluster, volume arvados.Volume, metrics *volumeMetricsVecs, raceWindow time.Duration) *TestableS3Volume { clock := &fakeClock{} srv, err := s3test.NewServer(&s3test.Config{Clock: clock}) c.Assert(err, check.IsNil) - - tmp, err := ioutil.TempFile("", "keepstore") - c.Assert(err, check.IsNil) - defer os.Remove(tmp.Name()) - _, err = tmp.Write([]byte("xxx\n")) - c.Assert(err, check.IsNil) - c.Assert(tmp.Close(), check.IsNil) + endpoint := srv.URL() + if s.s3server != nil { + endpoint = s.s3server.URL + } v := &TestableS3Volume{ S3Volume: &S3Volume{ + AccessKey: "xxx", + SecretKey: "xxx", Bucket: TestBucketName, - AccessKeyFile: tmp.Name(), - SecretKeyFile: tmp.Name(), - Endpoint: srv.URL(), + Endpoint: endpoint, Region: "test-region-1", LocationConstraint: true, - RaceWindow: arvados.Duration(raceWindow), - S3Replication: replication, - UnsafeDelete: s3UnsafeDelete, - ReadOnly: readonly, + UnsafeDelete: true, IndexPageSize: 1000, + cluster: cluster, + volume: volume, + logger: ctxlog.TestLogger(c), + metrics: metrics, }, + c: c, server: srv, serverClock: clock, } - c.Assert(v.Start(), check.IsNil) - err = v.bucket.PutBucket(s3.ACL("private")) - c.Assert(err, check.IsNil) + c.Assert(v.S3Volume.check(), check.IsNil) + c.Assert(v.bucket.PutBucket(s3.ACL("private")), check.IsNil) + // We couldn't set RaceWindow until now because check() + // rejects negative values. + v.S3Volume.RaceWindow = arvados.Duration(raceWindow) return v } @@ -324,7 +456,11 @@ func (s *StubbedS3Suite) newTestableVolume(c *check.C, raceWindow time.Duration, func (v *TestableS3Volume) PutRaw(loc string, block []byte) { err := v.bucket.Put(loc, block, "application/octet-stream", s3ACL, s3.Options{}) if err != nil { - log.Printf("PutRaw: %+v", err) + log.Printf("PutRaw: %s: %+v", loc, err) + } + err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) + if err != nil { + log.Printf("PutRaw: recent/%s: %+v", loc, err) } } @@ -343,3 +479,7 @@ func (v *TestableS3Volume) TouchWithDate(locator string, lastPut time.Time) { func (v *TestableS3Volume) Teardown() { v.server.Quit() } + +func (v *TestableS3Volume) ReadWriteOperationLabelValues() (r, w string) { + return "get", "put" +}