X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/19ae770973482257117fe8ded5619c3018c4b60f..094024f2476d84a5b9a453ede79b243e1d282bab:/services/keepstore/s3_volume_test.go diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go index 0c2cd4942e..59402dc6b0 100644 --- a/services/keepstore/s3_volume_test.go +++ b/services/keepstore/s3_volume_test.go @@ -2,9 +2,10 @@ package main import ( "bytes" + "crypto/md5" "fmt" "log" - "strings" + "os" "time" "github.com/AdRoll/goamz/aws" @@ -41,7 +42,7 @@ func init() { s3UnsafeDelete = true } -func NewTestableS3Volume(c *check.C, readonly bool, replication int) *TestableS3Volume { +func NewTestableS3Volume(c *check.C, raceWindow time.Duration, readonly bool, replication int) *TestableS3Volume { clock := &fakeClock{} srv, err := s3test.NewServer(&s3test.Config{Clock: clock}) c.Assert(err, check.IsNil) @@ -59,7 +60,7 @@ func NewTestableS3Volume(c *check.C, readonly bool, replication int) *TestableS3 c.Assert(err, check.IsNil) return &TestableS3Volume{ - S3Volume: NewS3Volume(auth, region, TestBucketName, readonly, replication), + S3Volume: NewS3Volume(auth, region, TestBucketName, raceWindow, readonly, replication), server: srv, serverClock: clock, } @@ -73,18 +74,20 @@ type StubbedS3Suite struct { func (s *StubbedS3Suite) TestGeneric(c *check.C) { DoGenericVolumeTests(c, func(t TB) TestableVolume { - return NewTestableS3Volume(c, false, 2) + // Use a negative raceWindow so s3test's 1-second + // timestamp precision doesn't confuse fixRace. + return NewTestableS3Volume(c, -2*time.Second, false, 2) }) } func (s *StubbedS3Suite) TestGenericReadOnly(c *check.C) { DoGenericVolumeTests(c, func(t TB) TestableVolume { - return NewTestableS3Volume(c, true, 2) + return NewTestableS3Volume(c, -2*time.Second, true, 2) }) } func (s *StubbedS3Suite) TestIndex(c *check.C) { - v := NewTestableS3Volume(c, false, 2) + v := NewTestableS3Volume(c, 0, false, 2) v.indexPageSize = 3 for i := 0; i < 256; i++ { v.PutRaw(fmt.Sprintf("%02x%030x", i, i), []byte{102, 111, 111}) @@ -108,6 +111,154 @@ func (s *StubbedS3Suite) TestIndex(c *check.C) { } } +func (s *StubbedS3Suite) TestBackendStates(c *check.C) { + defer func(tl, bs time.Duration) { + trashLifetime = tl + blobSignatureTTL = bs + }(trashLifetime, blobSignatureTTL) + trashLifetime = time.Hour + blobSignatureTTL = time.Hour + + v := NewTestableS3Volume(c, 5*time.Minute, false, 2) + var none time.Time + + stubKey := func(t time.Time, key string, data []byte) { + if t == none { + return + } + v.serverClock.now = &t + v.Bucket.Put(key, data, "application/octet-stream", s3ACL, s3.Options{}) + } + + t0 := time.Now() + nextKey := 0 + for _, test := range []struct { + label string + data time.Time + recent time.Time + trash time.Time + canGet bool + canTrash bool + canGetAfterTrash bool + canUntrash bool + haveTrashAfterEmpty bool + freshAfterEmpty bool + }{ + { + "No related objects", + none, none, none, + false, false, false, false, false, false}, + { + // Stored by older version, or there was a + // race between EmptyTrash and Put: Trash is a + // no-op even though the data object is very + // old + "No recent/X", + t0.Add(-48 * time.Hour), none, none, + true, true, true, false, false, false}, + { + "Not trash; old enough to trash", + t0.Add(-24 * time.Hour), t0.Add(-2 * time.Hour), none, + true, true, false, false, false, false}, + { + "Not trash; not old enough to trash", + t0.Add(-24 * time.Hour), t0.Add(-30 * time.Minute), none, + true, true, true, false, false, false}, + { + "Trash + not-trash: recent race between Trash and Put", + t0.Add(-24 * time.Hour), t0.Add(-3 * time.Minute), t0.Add(-2 * time.Minute), + true, true, true, true, true, false}, + { + "Trash + not-trash, nearly eligible for deletion, prone to Trash race", + t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute), + true, false, true, true, true, false}, + { + "Trash + not-trash, eligible for deletion, prone to Trash race", + t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-61 * time.Minute), + true, false, true, true, false, false}, + { + "Trash + not-trash, unsafe to empty; old race between Put and unfinished Trash", + t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-12 * time.Hour), + true, false, true, true, true, true}, + { + "Trash + not-trash, was unsafe to empty, but since made safe by fixRace+Touch", + t0.Add(-time.Second), t0.Add(-time.Second), t0.Add(-12 * time.Hour), + true, true, true, true, false, false}, + { + "Trash operation was interrupted", + t0.Add(-24 * time.Hour), t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), + true, false, true, true, false, false}, + { + "Trash, not yet eligible for deletion", + none, t0.Add(-12 * time.Hour), t0.Add(-time.Minute), + false, false, false, true, true, false}, + { + "Trash, not yet eligible for deletion, prone to races", + none, t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute), + false, false, false, true, true, false}, + { + "Trash, eligible for deletion", + none, t0.Add(-12 * time.Hour), t0.Add(-2 * time.Hour), + false, false, false, true, false, false}, + { + "Erroneously trashed during a race, detected before trashLifetime", + none, t0.Add(-30 * time.Minute), t0.Add(-29 * time.Minute), + true, false, true, true, true, false}, + { + "Erroneously trashed during a race, rescue during EmptyTrash despite reaching trashLifetime", + none, t0.Add(-90 * time.Minute), t0.Add(-89 * time.Minute), + true, false, true, true, true, false}, + } { + c.Log("Scenario: ", test.label) + var loc string + var blk []byte + + setup := func() { + nextKey++ + blk = []byte(fmt.Sprintf("%d", nextKey)) + loc = fmt.Sprintf("%x", md5.Sum(blk)) + c.Log("\t", loc) + stubKey(test.data, loc, blk) + stubKey(test.recent, "recent/"+loc, nil) + stubKey(test.trash, "trash/"+loc, blk) + v.serverClock.now = &t0 + } + + setup() + buf := make([]byte, len(blk)) + _, err := v.Get(loc, buf) + c.Check(err == nil, check.Equals, test.canGet) + if err != nil { + c.Check(os.IsNotExist(err), check.Equals, true) + } + + setup() + err = v.Trash(loc) + c.Check(err == nil, check.Equals, test.canTrash) + _, err = v.Get(loc, buf) + c.Check(err == nil, check.Equals, test.canGetAfterTrash) + if err != nil { + c.Check(os.IsNotExist(err), check.Equals, true) + } + + setup() + err = v.Untrash(loc) + c.Check(err == nil, check.Equals, test.canUntrash) + + setup() + v.EmptyTrash() + _, err = v.Bucket.Head("trash/"+loc, nil) + c.Check(err == nil, check.Equals, test.haveTrashAfterEmpty) + if test.freshAfterEmpty { + t, err := v.Mtime(loc) + c.Check(err, check.IsNil) + // new mtime must be current (with an + // allowance for 1s timestamp precision) + c.Check(t.After(t0.Add(-time.Second)), check.Equals, true) + } + } +} + // PutRaw skips the ContentMD5 test func (v *TestableS3Volume) PutRaw(loc string, block []byte) { err := v.Bucket.Put(loc, block, "application/octet-stream", s3ACL, s3.Options{}) @@ -121,9 +272,9 @@ func (v *TestableS3Volume) PutRaw(loc string, block []byte) { // while we do this. func (v *TestableS3Volume) TouchWithDate(locator string, lastPut time.Time) { v.serverClock.now = &lastPut - err := v.Touch(locator) - if err != nil && !strings.Contains(err.Error(), "PutCopy returned old LastModified") { - log.Printf("Touch: %+v", err) + err := v.Bucket.Put("recent/"+locator, nil, "application/octet-stream", s3ACL, s3.Options{}) + if err != nil { + panic(err) } v.serverClock.now = nil }