X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/682dd5b6cc23a455766a7651e3e841257660b31c..0821f5481edd016a3744bb50d97a9e5b99cd1a0f:/services/keepstore/trash_worker_test.go

diff --git a/services/keepstore/trash_worker_test.go b/services/keepstore/trash_worker_test.go
index 0511b48d37..94798d95ac 100644
--- a/services/keepstore/trash_worker_test.go
+++ b/services/keepstore/trash_worker_test.go
@@ -31,6 +31,7 @@ type TrashWorkerTestData struct {
    Expect no errors.
 */
 func TestTrashWorkerIntegration_GetNonExistingLocator(t *testing.T) {
+	neverDelete = false
 	testData := TrashWorkerTestData{
 		Locator1: "5d41402abc4b2a76b9719d911017c592",
 		Block1:   []byte("hello"),
@@ -52,16 +53,17 @@ func TestTrashWorkerIntegration_GetNonExistingLocator(t *testing.T) {
    Expect the second locator in volume 2 to be unaffected.
 */
 func TestTrashWorkerIntegration_LocatorInVolume1(t *testing.T) {
+	neverDelete = false
 	testData := TrashWorkerTestData{
-		Locator1: TEST_HASH,
-		Block1:   TEST_BLOCK,
+		Locator1: TestHash,
+		Block1:   TestBlock,
 
-		Locator2: TEST_HASH_2,
-		Block2:   TEST_BLOCK_2,
+		Locator2: TestHash2,
+		Block2:   TestBlock2,
 
 		CreateData: true,
 
-		DeleteLocator: TEST_HASH, // first locator
+		DeleteLocator: TestHash, // first locator
 
 		ExpectLocator1: false,
 		ExpectLocator2: true,
@@ -73,16 +75,17 @@ func TestTrashWorkerIntegration_LocatorInVolume1(t *testing.T) {
    Expect the first locator in volume 1 to be unaffected.
 */
 func TestTrashWorkerIntegration_LocatorInVolume2(t *testing.T) {
+	neverDelete = false
 	testData := TrashWorkerTestData{
-		Locator1: TEST_HASH,
-		Block1:   TEST_BLOCK,
+		Locator1: TestHash,
+		Block1:   TestBlock,
 
-		Locator2: TEST_HASH_2,
-		Block2:   TEST_BLOCK_2,
+		Locator2: TestHash2,
+		Block2:   TestBlock2,
 
 		CreateData: true,
 
-		DeleteLocator: TEST_HASH_2, // locator 2
+		DeleteLocator: TestHash2, // locator 2
 
 		ExpectLocator1: true,
 		ExpectLocator2: false,
@@ -94,16 +97,17 @@ func TestTrashWorkerIntegration_LocatorInVolume2(t *testing.T) {
    Expect locator to be deleted from both volumes.
 */
 func TestTrashWorkerIntegration_LocatorInBothVolumes(t *testing.T) {
+	neverDelete = false
 	testData := TrashWorkerTestData{
-		Locator1: TEST_HASH,
-		Block1:   TEST_BLOCK,
+		Locator1: TestHash,
+		Block1:   TestBlock,
 
-		Locator2: TEST_HASH,
-		Block2:   TEST_BLOCK,
+		Locator2: TestHash,
+		Block2:   TestBlock,
 
 		CreateData: true,
 
-		DeleteLocator: TEST_HASH,
+		DeleteLocator: TestHash,
 
 		ExpectLocator1: false,
 		ExpectLocator2: false,
@@ -115,17 +119,18 @@ func TestTrashWorkerIntegration_LocatorInBothVolumes(t *testing.T) {
    Delete the second and expect the first to be still around.
 */
 func TestTrashWorkerIntegration_MtimeMatchesForLocator1ButNotForLocator2(t *testing.T) {
+	neverDelete = false
 	testData := TrashWorkerTestData{
-		Locator1: TEST_HASH,
-		Block1:   TEST_BLOCK,
+		Locator1: TestHash,
+		Block1:   TestBlock,
 
-		Locator2: TEST_HASH,
-		Block2:   TEST_BLOCK,
+		Locator2: TestHash,
+		Block2:   TestBlock,
 
 		CreateData:      true,
 		DifferentMtimes: true,
 
-		DeleteLocator: TEST_HASH,
+		DeleteLocator: TestHash,
 
 		ExpectLocator1: true,
 		ExpectLocator2: false,
@@ -138,17 +143,18 @@ func TestTrashWorkerIntegration_MtimeMatchesForLocator1ButNotForLocator2(t *test
    Expect the other unaffected.
 */
 func TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(t *testing.T) {
+	neverDelete = false
 	testData := TrashWorkerTestData{
-		Locator1: TEST_HASH,
-		Block1:   TEST_BLOCK,
+		Locator1: TestHash,
+		Block1:   TestBlock,
 
-		Locator2: TEST_HASH_2,
-		Block2:   TEST_BLOCK_2,
+		Locator2: TestHash2,
+		Block2:   TestBlock2,
 
 		CreateData:      true,
 		CreateInVolume1: true,
 
-		DeleteLocator: TEST_HASH, // locator 1
+		DeleteLocator: TestHash, // locator 1
 
 		ExpectLocator1: false,
 		ExpectLocator2: true,
@@ -157,22 +163,23 @@ func TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(t *testing.T) {
 }
 
 /* Allow default Trash Life time to be used. Thus, the newly created block
-   will not be deleted becuase its Mtime is within the trash life time.
+   will not be deleted because its Mtime is within the trash life time.
 */
 func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(t *testing.T) {
+	neverDelete = false
 	testData := TrashWorkerTestData{
-		Locator1: TEST_HASH,
-		Block1:   TEST_BLOCK,
+		Locator1: TestHash,
+		Block1:   TestBlock,
 
-		Locator2: TEST_HASH_2,
-		Block2:   TEST_BLOCK_2,
+		Locator2: TestHash2,
+		Block2:   TestBlock2,
 
 		CreateData:      true,
 		CreateInVolume1: true,
 
 		UseTrashLifeTime: true,
 
-		DeleteLocator: TEST_HASH, // locator 1
+		DeleteLocator: TestHash, // locator 1
 
 		// Since trash life time is in effect, block won't be deleted.
 		ExpectLocator1: true,
@@ -181,6 +188,28 @@ func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(
 	performTrashWorkerTest(testData, t)
 }
 
+/* Delete a block with matching mtime for locator in both volumes, but neverDelete is true,
+   so block won't be deleted.
+*/
+func TestTrashWorkerIntegration_NeverDelete(t *testing.T) {
+	neverDelete = true
+	testData := TrashWorkerTestData{
+		Locator1: TestHash,
+		Block1:   TestBlock,
+
+		Locator2: TestHash,
+		Block2:   TestBlock,
+
+		CreateData: true,
+
+		DeleteLocator: TestHash,
+
+		ExpectLocator1: true,
+		ExpectLocator2: true,
+	}
+	performTrashWorkerTest(testData, t)
+}
+
 /* Perform the test */
 func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
 	// Create Keep Volumes
@@ -202,12 +231,12 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
 		}
 	}
 
-	oldBlockTime := time.Now().Add(-blob_signature_ttl - time.Minute)
+	oldBlockTime := time.Now().Add(-blobSignatureTTL - time.Minute)
 
 	// Create TrashRequest for the test
 	trashRequest := TrashRequest{
 		Locator:    testData.DeleteLocator,
-		BlockMtime: oldBlockTime.Unix(),
+		BlockMtime: oldBlockTime.UnixNano(),
 	}
 
 	// Run trash worker and put the trashRequest on trashq
@@ -229,30 +258,59 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
 	}
 	go RunTrashWorker(trashq)
 
+	// Install gate so all local operations block until we say go
+	gate := make(chan struct{})
+	for _, v := range vols {
+		v.(*MockVolume).Gate = gate
+	}
+
+	assertStatusItem := func(k string, expect float64) {
+		if v := getStatusItem("TrashQueue", k); v != expect {
+			t.Errorf("Got %s %v, expected %v", k, v, expect)
+		}
+	}
+
+	assertStatusItem("InProgress", 0)
+	assertStatusItem("Queued", 0)
+
+	listLen := trashList.Len()
 	trashq.ReplaceQueue(trashList)
-	time.Sleep(10 * time.Millisecond) // give a moment to finish processing the list
+
+	// Wait for worker to take request(s)
+	expectEqualWithin(t, time.Second, listLen, func() interface{} { return trashq.Status().InProgress })
+
+	// Ensure status.json also reports work is happening
+	assertStatusItem("InProgress", float64(1))
+	assertStatusItem("Queued", float64(listLen-1))
+
+	// Let worker proceed
+	close(gate)
+
+	// Wait for worker to finish
+	expectEqualWithin(t, time.Second, 0, func() interface{} { return trashq.Status().InProgress })
 
 	// Verify Locator1 to be un/deleted as expected
-	data, _ := GetBlock(testData.Locator1, false)
+	buf := make([]byte, BlockSize)
+	size, err := GetBlock(testData.Locator1, buf, nil)
 	if testData.ExpectLocator1 {
-		if len(data) == 0 {
+		if size == 0 || err != nil {
 			t.Errorf("Expected Locator1 to be still present: %s", testData.Locator1)
 		}
 	} else {
-		if len(data) > 0 {
+		if size > 0 || err == nil {
 			t.Errorf("Expected Locator1 to be deleted: %s", testData.Locator1)
 		}
 	}
 
 	// Verify Locator2 to be un/deleted as expected
 	if testData.Locator1 != testData.Locator2 {
-		data, _ = GetBlock(testData.Locator2, false)
+		size, err = GetBlock(testData.Locator2, buf, nil)
 		if testData.ExpectLocator2 {
-			if len(data) == 0 {
+			if size == 0 || err != nil {
 				t.Errorf("Expected Locator2 to be still present: %s", testData.Locator2)
 			}
 		} else {
-			if len(data) > 0 {
+			if size > 0 || err == nil {
 				t.Errorf("Expected Locator2 to be deleted: %s", testData.Locator2)
 			}
 		}
@@ -264,7 +322,8 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
 	if testData.DifferentMtimes {
 		locatorFoundIn := 0
 		for _, volume := range KeepVM.AllReadable() {
-			if _, err := volume.Get(testData.Locator1); err == nil {
+			buf := make([]byte, BlockSize)
+			if _, err := volume.Get(testData.Locator1, buf); err == nil {
 				locatorFoundIn = locatorFoundIn + 1
 			}
 		}