8555: Log statistics in EmptyTrash.
[arvados.git] / services / keepstore / s3_volume_test.go
index 59402dc6b07acbc5f38baab97c4cc816aaa33bd9..8a06f19c32b64f78686b06a0873946ef3b4b6f5a 100644 (file)
@@ -122,7 +122,7 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
        v := NewTestableS3Volume(c, 5*time.Minute, false, 2)
        var none time.Time
 
-       stubKey := func(t time.Time, key string, data []byte) {
+       putS3Obj := func(t time.Time, key string, data []byte) {
                if t == none {
                        return
                }
@@ -132,11 +132,11 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
 
        t0 := time.Now()
        nextKey := 0
-       for _, test := range []struct {
+       for _, scenario := range []struct {
                label               string
-               data                time.Time
-               recent              time.Time
-               trash               time.Time
+               dataT               time.Time
+               recentT             time.Time
+               trashT              time.Time
                canGet              bool
                canTrash            bool
                canGetAfterTrash    bool
@@ -147,7 +147,8 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
                {
                        "No related objects",
                        none, none, none,
-                       false, false, false, false, false, false},
+                       false, false, false, false, false, false,
+               },
                {
                        // Stored by older version, or there was a
                        // race between EmptyTrash and Put: Trash is a
@@ -155,101 +156,121 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
                        // old
                        "No recent/X",
                        t0.Add(-48 * time.Hour), none, none,
-                       true, true, true, false, false, false},
+                       true, true, true, false, false, false,
+               },
                {
-                       "Not trash; old enough to trash",
+                       "Not trash, but old enough to be eligible for trash",
                        t0.Add(-24 * time.Hour), t0.Add(-2 * time.Hour), none,
-                       true, true, false, false, false, false},
+                       true, true, false, false, false, false,
+               },
                {
-                       "Not trash; not old enough to trash",
+                       "Not trash, and not old enough to be eligible for trash",
                        t0.Add(-24 * time.Hour), t0.Add(-30 * time.Minute), none,
-                       true, true, true, false, false, false},
+                       true, true, true, false, false, false,
+               },
                {
-                       "Trash + not-trash: recent race between Trash and Put",
+                       "Trashed + untrashed copies exist, due to recent race between Trash and Put",
                        t0.Add(-24 * time.Hour), t0.Add(-3 * time.Minute), t0.Add(-2 * time.Minute),
-                       true, true, true, true, true, false},
+                       true, true, true, true, true, false,
+               },
                {
-                       "Trash + not-trash, nearly eligible for deletion, prone to Trash race",
+                       "Trashed + untrashed copies exist, trash nearly eligible for deletion: prone to Trash race",
                        t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute),
-                       true, false, true, true, true, false},
+                       true, false, true, true, true, false,
+               },
                {
-                       "Trash + not-trash, eligible for deletion, prone to Trash race",
+                       "Trashed + untrashed copies exist, trash is eligible for deletion: prone to Trash race",
                        t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-61 * time.Minute),
-                       true, false, true, true, false, false},
+                       true, false, true, true, false, false,
+               },
                {
-                       "Trash + not-trash, unsafe to empty; old race between Put and unfinished Trash",
+                       "Trashed + untrashed copies exist, due to old race between Put and unfinished Trash: emptying trash is unsafe",
                        t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-12 * time.Hour),
-                       true, false, true, true, true, true},
+                       true, false, true, true, true, true,
+               },
                {
-                       "Trash + not-trash, was unsafe to empty, but since made safe by fixRace+Touch",
+                       "Trashed + untrashed copies exist, used to be unsafe to empty, but since made safe by fixRace+Touch",
                        t0.Add(-time.Second), t0.Add(-time.Second), t0.Add(-12 * time.Hour),
-                       true, true, true, true, false, false},
+                       true, true, true, true, false, false,
+               },
                {
-                       "Trash operation was interrupted",
+                       "Trashed + untrashed copies exist because Trash operation was interrupted (no race)",
                        t0.Add(-24 * time.Hour), t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour),
-                       true, false, true, true, false, false},
+                       true, false, true, true, false, false,
+               },
                {
                        "Trash, not yet eligible for deletion",
                        none, t0.Add(-12 * time.Hour), t0.Add(-time.Minute),
-                       false, false, false, true, true, false},
+                       false, false, false, true, true, false,
+               },
                {
                        "Trash, not yet eligible for deletion, prone to races",
                        none, t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute),
-                       false, false, false, true, true, false},
+                       false, false, false, true, true, false,
+               },
                {
                        "Trash, eligible for deletion",
                        none, t0.Add(-12 * time.Hour), t0.Add(-2 * time.Hour),
-                       false, false, false, true, false, false},
+                       false, false, false, true, false, false,
+               },
                {
                        "Erroneously trashed during a race, detected before trashLifetime",
                        none, t0.Add(-30 * time.Minute), t0.Add(-29 * time.Minute),
-                       true, false, true, true, true, false},
+                       true, false, true, true, true, false,
+               },
                {
                        "Erroneously trashed during a race, rescue during EmptyTrash despite reaching trashLifetime",
                        none, t0.Add(-90 * time.Minute), t0.Add(-89 * time.Minute),
-                       true, false, true, true, true, false},
+                       true, false, true, true, true, false,
+               },
        } {
-               c.Log("Scenario: ", test.label)
-               var loc string
-               var blk []byte
+               c.Log("Scenario: ", scenario.label)
 
-               setup := func() {
+               // We have a few tests to run for each scenario, and
+               // the tests are expected to change state. By calling
+               // this setup func between tests, we (re)create the
+               // scenario as specified, using a new unique block
+               // locator to prevent interference from previous
+               // tests.
+
+               setupScenario := func() (string, []byte) {
                        nextKey++
-                       blk = []byte(fmt.Sprintf("%d", nextKey))
-                       loc = fmt.Sprintf("%x", md5.Sum(blk))
+                       blk := []byte(fmt.Sprintf("%d", nextKey))
+                       loc := fmt.Sprintf("%x", md5.Sum(blk))
                        c.Log("\t", loc)
-                       stubKey(test.data, loc, blk)
-                       stubKey(test.recent, "recent/"+loc, nil)
-                       stubKey(test.trash, "trash/"+loc, blk)
+                       putS3Obj(scenario.dataT, loc, blk)
+                       putS3Obj(scenario.recentT, "recent/"+loc, nil)
+                       putS3Obj(scenario.trashT, "trash/"+loc, blk)
                        v.serverClock.now = &t0
+                       return loc, blk
                }
 
-               setup()
+               loc, blk := setupScenario()
                buf := make([]byte, len(blk))
                _, err := v.Get(loc, buf)
-               c.Check(err == nil, check.Equals, test.canGet)
+               c.Check(err == nil, check.Equals, scenario.canGet)
                if err != nil {
                        c.Check(os.IsNotExist(err), check.Equals, true)
                }
 
-               setup()
+               loc, blk = setupScenario()
                err = v.Trash(loc)
-               c.Check(err == nil, check.Equals, test.canTrash)
+               c.Check(err == nil, check.Equals, scenario.canTrash)
                _, err = v.Get(loc, buf)
-               c.Check(err == nil, check.Equals, test.canGetAfterTrash)
+               c.Check(err == nil, check.Equals, scenario.canGetAfterTrash)
                if err != nil {
                        c.Check(os.IsNotExist(err), check.Equals, true)
                }
 
-               setup()
+               loc, blk = setupScenario()
                err = v.Untrash(loc)
-               c.Check(err == nil, check.Equals, test.canUntrash)
+               c.Check(err == nil, check.Equals, scenario.canUntrash)
 
-               setup()
+               loc, blk = setupScenario()
                v.EmptyTrash()
                _, err = v.Bucket.Head("trash/"+loc, nil)
-               c.Check(err == nil, check.Equals, test.haveTrashAfterEmpty)
-               if test.freshAfterEmpty {
+               c.Check(err == nil, check.Equals, scenario.haveTrashAfterEmpty)
+               if scenario.freshAfterEmpty {
                        t, err := v.Mtime(loc)
                        c.Check(err, check.IsNil)
                        // new mtime must be current (with an