7121: Replace Get(loc,true) with CompareAndTouch(). Add Compare method to Volume...

[arvados.git] / services / keepstore / trash_worker_test.go

diff --git a/services/keepstore/trash_worker_test.go b/services/keepstore/trash_worker_test.go
index a11756b5f79e5c451563e92021c2cd9b1ecfe59c..a626d9be9b67aff2369d6e4399f90f81b806d99b 100644 (file)
--- a/services/keepstore/trash_worker_test.go
+++ b/services/keepstore/trash_worker_test.go
@@ -15,8 +15,8 @@ type TrashWorkerTestData struct {
        Block2      []byte
        BlockMtime2 int64
 
-       CreateData       bool
-       CreateInVolume1  bool
+       CreateData      bool
+       CreateInVolume1 bool
 
        UseTrashLifeTime bool
        DifferentMtimes  bool
@@ -31,6 +31,7 @@ type TrashWorkerTestData struct {
    Expect no errors.
 */
 func TestTrashWorkerIntegration_GetNonExistingLocator(t *testing.T) {
+       never_delete = false
        testData := TrashWorkerTestData{
                Locator1: "5d41402abc4b2a76b9719d911017c592",
                Block1:   []byte("hello"),
@@ -52,6 +53,7 @@ func TestTrashWorkerIntegration_GetNonExistingLocator(t *testing.T) {
    Expect the second locator in volume 2 to be unaffected.
 */
 func TestTrashWorkerIntegration_LocatorInVolume1(t *testing.T) {
+       never_delete = false
        testData := TrashWorkerTestData{
                Locator1: TEST_HASH,
                Block1:   TEST_BLOCK,
@@ -73,6 +75,7 @@ func TestTrashWorkerIntegration_LocatorInVolume1(t *testing.T) {
    Expect the first locator in volume 1 to be unaffected.
 */
 func TestTrashWorkerIntegration_LocatorInVolume2(t *testing.T) {
+       never_delete = false
        testData := TrashWorkerTestData{
                Locator1: TEST_HASH,
                Block1:   TEST_BLOCK,
@@ -94,6 +97,7 @@ func TestTrashWorkerIntegration_LocatorInVolume2(t *testing.T) {
    Expect locator to be deleted from both volumes.
 */
 func TestTrashWorkerIntegration_LocatorInBothVolumes(t *testing.T) {
+       never_delete = false
        testData := TrashWorkerTestData{
                Locator1: TEST_HASH,
                Block1:   TEST_BLOCK,
@@ -115,6 +119,7 @@ func TestTrashWorkerIntegration_LocatorInBothVolumes(t *testing.T) {
    Delete the second and expect the first to be still around.
 */
 func TestTrashWorkerIntegration_MtimeMatchesForLocator1ButNotForLocator2(t *testing.T) {
+       never_delete = false
        testData := TrashWorkerTestData{
                Locator1: TEST_HASH,
                Block1:   TEST_BLOCK,
@@ -138,6 +143,7 @@ func TestTrashWorkerIntegration_MtimeMatchesForLocator1ButNotForLocator2(t *test
    Expect the other unaffected.
 */
 func TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(t *testing.T) {
+       never_delete = false
        testData := TrashWorkerTestData{
                Locator1: TEST_HASH,
                Block1:   TEST_BLOCK,
@@ -160,6 +166,7 @@ func TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(t *testing.T) {
    will not be deleted becuase its Mtime is within the trash life time.
 */
 func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(t *testing.T) {
+       never_delete = false
        testData := TrashWorkerTestData{
                Locator1: TEST_HASH,
                Block1:   TEST_BLOCK,
@@ -181,6 +188,28 @@ func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(
        performTrashWorkerTest(testData, t)
 }
 
+/* Delete a block with matching mtime for locator in both volumes, but never_delete is true,
+   so block won't be deleted.
+*/
+func TestTrashWorkerIntegration_NeverDelete(t *testing.T) {
+       never_delete = true
+       testData := TrashWorkerTestData{
+               Locator1: TEST_HASH,
+               Block1:   TEST_BLOCK,
+
+               Locator2: TEST_HASH,
+               Block2:   TEST_BLOCK,
+
+               CreateData: true,
+
+               DeleteLocator: TEST_HASH,
+
+               ExpectLocator1: true,
+               ExpectLocator2: true,
+       }
+       performTrashWorkerTest(testData, t)
+}
+
 /* Perform the test */
 func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
        // Create Keep Volumes
@@ -202,7 +231,7 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
                }
        }
 
-       oldBlockTime := time.Now().Add(-blob_signature_ttl-time.Minute)
+       oldBlockTime := time.Now().Add(-blob_signature_ttl - time.Minute)
 
        // Create TrashRequest for the test
        trashRequest := TrashRequest{
@@ -229,11 +258,39 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
        }
        go RunTrashWorker(trashq)
 
+       // Install gate so all local operations block until we say go
+       gate := make(chan struct{})
+       for _, v := range vols {
+               v.(*MockVolume).Gate = gate
+       }
+
+       assertStatusItem := func(k string, expect float64) {
+               if v := getStatusItem("TrashQueue", k); v != expect {
+                       t.Errorf("Got %s %v, expected %v", k, v, expect)
+               }
+       }
+
+       assertStatusItem("InProgress", 0)
+       assertStatusItem("Queued", 0)
+
+       listLen := trashList.Len()
        trashq.ReplaceQueue(trashList)
-       time.Sleep(10 * time.Millisecond) // give a moment to finish processing the list
+
+       // Wait for worker to take request(s)
+       expectEqualWithin(t, time.Second, listLen, func() interface{} { return trashq.Status().InProgress })
+
+       // Ensure status.json also reports work is happening
+       assertStatusItem("InProgress", float64(1))
+       assertStatusItem("Queued", float64(listLen-1))
+
+       // Let worker proceed
+       close(gate)
+
+       // Wait for worker to finish
+       expectEqualWithin(t, time.Second, 0, func() interface{} { return trashq.Status().InProgress })
 
        // Verify Locator1 to be un/deleted as expected
-       data, _ := GetBlock(testData.Locator1, false)
+       data, _ := GetBlock(testData.Locator1)
        if testData.ExpectLocator1 {
                if len(data) == 0 {
                        t.Errorf("Expected Locator1 to be still present: %s", testData.Locator1)
@@ -246,7 +303,7 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
 
        // Verify Locator2 to be un/deleted as expected
        if testData.Locator1 != testData.Locator2 {
-               data, _ = GetBlock(testData.Locator2, false)
+               data, _ = GetBlock(testData.Locator2)
                if testData.ExpectLocator2 {
                        if len(data) == 0 {
                                t.Errorf("Expected Locator2 to be still present: %s", testData.Locator2)