8555: Test various backend states. Update recent/X timestamp during Untrash.

diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index a0a6860e6d5661f6763d23c454cdfe0c3f17639e..5eb99414a0eddea7edbd8bcae738575a424ea66c 100644 (file)
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -427,7 +427,12 @@ func (v *S3Volume) lastModified(resp *http.Response) (t time.Time, err error) {
 }
 
 func (v *S3Volume) Untrash(loc string) error {
-       return v.safeCopy(loc, "trash/"+loc)
+       err := v.safeCopy(loc, "trash/"+loc)
+       if err != nil {
+               return err
+       }
+       err = v.Bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
+       return v.translateError(err)
 }
 
 func (v *S3Volume) Status() *VolumeStatus {

diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
index 73c6b7684144e503e964cc252f7541d36b3e96d2..a40b84f88d23a970017f038df20429a02390a6de 100644 (file)
--- a/services/keepstore/s3_volume_test.go
+++ b/services/keepstore/s3_volume_test.go
@@ -2,8 +2,10 @@ package main
 
 import (
        "bytes"
+       "crypto/md5"
        "fmt"
        "log"
+       "os"
        "time"
 
        "github.com/AdRoll/goamz/aws"
@@ -74,13 +76,13 @@ func (s *StubbedS3Suite) TestGeneric(c *check.C) {
        DoGenericVolumeTests(c, func(t TB) TestableVolume {
                // Use a negative raceWindow so s3test's 1-second
                // timestamp precision doesn't confuse fixRace.
-               return NewTestableS3Volume(c, -time.Second, false, 2)
+               return NewTestableS3Volume(c, -2*time.Second, false, 2)
        })
 }
 
 func (s *StubbedS3Suite) TestGenericReadOnly(c *check.C) {
        DoGenericVolumeTests(c, func(t TB) TestableVolume {
-               return NewTestableS3Volume(c, -time.Second, true, 2)
+               return NewTestableS3Volume(c, -2*time.Second, true, 2)
        })
 }
 
@@ -109,6 +111,143 @@ func (s *StubbedS3Suite) TestIndex(c *check.C) {
        }
 }
 
+func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
+       defer func(tl, bs time.Duration) {
+               trashLifetime = tl
+               blobSignatureTTL = bs
+       }(trashLifetime, blobSignatureTTL)
+       trashLifetime = time.Hour
+       blobSignatureTTL = time.Hour
+
+       v := NewTestableS3Volume(c, 5*time.Minute, false, 2)
+       var none time.Time
+
+       stubKey := func(t time.Time, key string, data []byte) {
+               if t == none {
+                       return
+               }
+               v.serverClock.now = &t
+               v.Bucket.Put(key, data, "application/octet-stream", s3ACL, s3.Options{})
+       }
+
+       t0 := time.Now()
+       nextKey := 0
+       for _, test := range []struct {
+               label               string
+               data                time.Time
+               recent              time.Time
+               trash               time.Time
+               canGet              bool
+               canTrash            bool
+               canGetAfterTrash    bool
+               canUntrash          bool
+               haveTrashAfterEmpty bool
+       }{
+               {
+                       "No related objects",
+                       none, none, none,
+                       false, false, false, false, false},
+               {
+                       // Stored by older version, or there was a
+                       // race between EmptyTrash and Put: Trash is a
+                       // no-op even though the data object is very
+                       // old
+                       "No recent/X",
+                       t0.Add(-48 * time.Hour), none, none,
+                       true, true, true, false, false},
+               {
+                       "Not trash; old enough to trash",
+                       t0.Add(-24 * time.Hour), t0.Add(-2 * time.Hour), none,
+                       true, true, false, false, false},
+               {
+                       "Not trash; not old enough to trash",
+                       t0.Add(-24 * time.Hour), t0.Add(-30 * time.Minute), none,
+                       true, true, true, false, false},
+               {
+                       "Trash + not-trash: recent race between Trash and Put",
+                       t0.Add(-24 * time.Hour), t0.Add(-3 * time.Minute), t0.Add(-2 * time.Minute),
+                       true, true, true, true, true},
+               {
+                       "Trash + not-trash, nearly eligible for deletion, prone to Trash race",
+                       t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute),
+                       true, false, true, true, true},
+               {
+                       "Trash + not-trash, eligible for deletion, prone to Trash race",
+                       t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-61 * time.Minute),
+                       true, false, true, true, false},
+               // FIXME: old trash never gets deleted!
+               // {
+               //      "Not trash; old race between Trash and Put, or incomplete Trash",
+               //      t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-12 * time.Hour),
+               //      true, false, true, true, false},
+               {
+                       "Trash operation was interrupted",
+                       t0.Add(-24 * time.Hour), t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour),
+                       true, false, true, true, false},
+               {
+                       "Trash, not yet eligible for deletion",
+                       none, t0.Add(-12 * time.Hour), t0.Add(-time.Minute),
+                       false, false, false, true, true},
+               {
+                       "Trash, not yet eligible for deletion, prone to races",
+                       none, t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute),
+                       false, false, false, true, true},
+               {
+                       "Trash, eligible for deletion",
+                       none, t0.Add(-12 * time.Hour), t0.Add(-2 * time.Hour),
+                       false, false, false, true, false},
+               {
+                       "Erroneously trashed during a race, detected before trashLifetime",
+                       none, t0.Add(-30 * time.Minute), t0.Add(-29 * time.Minute),
+                       true, false, true, true, true},
+               {
+                       "Erroneously trashed during a race, rescue during EmptyTrash despite reaching trashLifetime",
+                       none, t0.Add(-90 * time.Minute), t0.Add(-89 * time.Minute),
+                       true, false, true, true, true},
+       } {
+               c.Log("Scenario: ", test.label)
+               var loc string
+               var blk []byte
+
+               setup := func() {
+                       nextKey++
+                       blk = []byte(fmt.Sprintf("%d", nextKey))
+                       loc = fmt.Sprintf("%x", md5.Sum(blk))
+                       c.Log("\t", loc)
+                       stubKey(test.data, loc, blk)
+                       stubKey(test.recent, "recent/"+loc, nil)
+                       stubKey(test.trash, "trash/"+loc, blk)
+                       v.serverClock.now = &t0
+               }
+
+               setup()
+               buf := make([]byte, len(blk))
+               _, err := v.Get(loc, buf)
+               c.Check(err == nil, check.Equals, test.canGet)
+               if err != nil {
+                       c.Check(os.IsNotExist(err), check.Equals, true)
+               }
+
+               setup()
+               err = v.Trash(loc)
+               c.Check(err == nil, check.Equals, test.canTrash)
+               _, err = v.Get(loc, buf)
+               c.Check(err == nil, check.Equals, test.canGetAfterTrash)
+               if err != nil {
+                       c.Check(os.IsNotExist(err), check.Equals, true)
+               }
+
+               setup()
+               err = v.Untrash(loc)
+               c.Check(err == nil, check.Equals, test.canUntrash)
+
+               setup()
+               v.EmptyTrash()
+               _, err = v.Bucket.Head("trash/"+loc, nil)
+               c.Check(err == nil, check.Equals, test.haveTrashAfterEmpty)
+       }
+}
+
 // PutRaw skips the ContentMD5 test
 func (v *TestableS3Volume) PutRaw(loc string, block []byte) {
        err := v.Bucket.Put(loc, block, "application/octet-stream", s3ACL, s3.Options{})

diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go
index c17c837820db53902f022d3c2b75936f0ff88623..bc3e537a89a815037102af7fb920e8b9d2b84f61 100644 (file)
--- a/services/keepstore/volume_generic_test.go
+++ b/services/keepstore/volume_generic_test.go
@@ -895,6 +895,9 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
                t.Fatal(err)
        }
 
+       // Untrash might have updated the timestamp, so backdate again
+       v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
+
        // Second set: EmptyTrash after the trash deadline has passed.
 
        trashLifetime = time.Nanosecond