From 35ea47144857dc16ab8b6b8a272a87af6d50cb88 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Wed, 20 Jul 2016 11:40:41 -0400 Subject: [PATCH] 8555: Test various backend states. Update recent/X timestamp during Untrash. --- services/keepstore/s3_volume.go | 7 +- services/keepstore/s3_volume_test.go | 143 +++++++++++++++++++++- services/keepstore/volume_generic_test.go | 3 + 3 files changed, 150 insertions(+), 3 deletions(-) diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go index a0a6860e6d..5eb99414a0 100644 --- a/services/keepstore/s3_volume.go +++ b/services/keepstore/s3_volume.go @@ -427,7 +427,12 @@ func (v *S3Volume) lastModified(resp *http.Response) (t time.Time, err error) { } func (v *S3Volume) Untrash(loc string) error { - return v.safeCopy(loc, "trash/"+loc) + err := v.safeCopy(loc, "trash/"+loc) + if err != nil { + return err + } + err = v.Bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) + return v.translateError(err) } func (v *S3Volume) Status() *VolumeStatus { diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go index 73c6b76841..a40b84f88d 100644 --- a/services/keepstore/s3_volume_test.go +++ b/services/keepstore/s3_volume_test.go @@ -2,8 +2,10 @@ package main import ( "bytes" + "crypto/md5" "fmt" "log" + "os" "time" "github.com/AdRoll/goamz/aws" @@ -74,13 +76,13 @@ func (s *StubbedS3Suite) TestGeneric(c *check.C) { DoGenericVolumeTests(c, func(t TB) TestableVolume { // Use a negative raceWindow so s3test's 1-second // timestamp precision doesn't confuse fixRace. - return NewTestableS3Volume(c, -time.Second, false, 2) + return NewTestableS3Volume(c, -2*time.Second, false, 2) }) } func (s *StubbedS3Suite) TestGenericReadOnly(c *check.C) { DoGenericVolumeTests(c, func(t TB) TestableVolume { - return NewTestableS3Volume(c, -time.Second, true, 2) + return NewTestableS3Volume(c, -2*time.Second, true, 2) }) } @@ -109,6 +111,143 @@ func (s *StubbedS3Suite) TestIndex(c *check.C) { } } +func (s *StubbedS3Suite) TestBackendStates(c *check.C) { + defer func(tl, bs time.Duration) { + trashLifetime = tl + blobSignatureTTL = bs + }(trashLifetime, blobSignatureTTL) + trashLifetime = time.Hour + blobSignatureTTL = time.Hour + + v := NewTestableS3Volume(c, 5*time.Minute, false, 2) + var none time.Time + + stubKey := func(t time.Time, key string, data []byte) { + if t == none { + return + } + v.serverClock.now = &t + v.Bucket.Put(key, data, "application/octet-stream", s3ACL, s3.Options{}) + } + + t0 := time.Now() + nextKey := 0 + for _, test := range []struct { + label string + data time.Time + recent time.Time + trash time.Time + canGet bool + canTrash bool + canGetAfterTrash bool + canUntrash bool + haveTrashAfterEmpty bool + }{ + { + "No related objects", + none, none, none, + false, false, false, false, false}, + { + // Stored by older version, or there was a + // race between EmptyTrash and Put: Trash is a + // no-op even though the data object is very + // old + "No recent/X", + t0.Add(-48 * time.Hour), none, none, + true, true, true, false, false}, + { + "Not trash; old enough to trash", + t0.Add(-24 * time.Hour), t0.Add(-2 * time.Hour), none, + true, true, false, false, false}, + { + "Not trash; not old enough to trash", + t0.Add(-24 * time.Hour), t0.Add(-30 * time.Minute), none, + true, true, true, false, false}, + { + "Trash + not-trash: recent race between Trash and Put", + t0.Add(-24 * time.Hour), t0.Add(-3 * time.Minute), t0.Add(-2 * time.Minute), + true, true, true, true, true}, + { + "Trash + not-trash, nearly eligible for deletion, prone to Trash race", + t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute), + true, false, true, true, true}, + { + "Trash + not-trash, eligible for deletion, prone to Trash race", + t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-61 * time.Minute), + true, false, true, true, false}, + // FIXME: old trash never gets deleted! + // { + // "Not trash; old race between Trash and Put, or incomplete Trash", + // t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-12 * time.Hour), + // true, false, true, true, false}, + { + "Trash operation was interrupted", + t0.Add(-24 * time.Hour), t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), + true, false, true, true, false}, + { + "Trash, not yet eligible for deletion", + none, t0.Add(-12 * time.Hour), t0.Add(-time.Minute), + false, false, false, true, true}, + { + "Trash, not yet eligible for deletion, prone to races", + none, t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute), + false, false, false, true, true}, + { + "Trash, eligible for deletion", + none, t0.Add(-12 * time.Hour), t0.Add(-2 * time.Hour), + false, false, false, true, false}, + { + "Erroneously trashed during a race, detected before trashLifetime", + none, t0.Add(-30 * time.Minute), t0.Add(-29 * time.Minute), + true, false, true, true, true}, + { + "Erroneously trashed during a race, rescue during EmptyTrash despite reaching trashLifetime", + none, t0.Add(-90 * time.Minute), t0.Add(-89 * time.Minute), + true, false, true, true, true}, + } { + c.Log("Scenario: ", test.label) + var loc string + var blk []byte + + setup := func() { + nextKey++ + blk = []byte(fmt.Sprintf("%d", nextKey)) + loc = fmt.Sprintf("%x", md5.Sum(blk)) + c.Log("\t", loc) + stubKey(test.data, loc, blk) + stubKey(test.recent, "recent/"+loc, nil) + stubKey(test.trash, "trash/"+loc, blk) + v.serverClock.now = &t0 + } + + setup() + buf := make([]byte, len(blk)) + _, err := v.Get(loc, buf) + c.Check(err == nil, check.Equals, test.canGet) + if err != nil { + c.Check(os.IsNotExist(err), check.Equals, true) + } + + setup() + err = v.Trash(loc) + c.Check(err == nil, check.Equals, test.canTrash) + _, err = v.Get(loc, buf) + c.Check(err == nil, check.Equals, test.canGetAfterTrash) + if err != nil { + c.Check(os.IsNotExist(err), check.Equals, true) + } + + setup() + err = v.Untrash(loc) + c.Check(err == nil, check.Equals, test.canUntrash) + + setup() + v.EmptyTrash() + _, err = v.Bucket.Head("trash/"+loc, nil) + c.Check(err == nil, check.Equals, test.haveTrashAfterEmpty) + } +} + // PutRaw skips the ContentMD5 test func (v *TestableS3Volume) PutRaw(loc string, block []byte) { err := v.Bucket.Put(loc, block, "application/octet-stream", s3ACL, s3.Options{}) diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go index c17c837820..bc3e537a89 100644 --- a/services/keepstore/volume_generic_test.go +++ b/services/keepstore/volume_generic_test.go @@ -895,6 +895,9 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { t.Fatal(err) } + // Untrash might have updated the timestamp, so backdate again + v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + // Second set: EmptyTrash after the trash deadline has passed. trashLifetime = time.Nanosecond -- 2.30.2