X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/32f7abdccc3ad293266e8df3e50344614ecd0dac..0eb72b526bf8bbb011551ecf019f604e17a534f1:/services/keepstore/volume_generic_test.go diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go index c614a08521..a9b96a4b21 100644 --- a/services/keepstore/volume_generic_test.go +++ b/services/keepstore/volume_generic_test.go @@ -1,15 +1,22 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( "bytes" + "context" "crypto/md5" "fmt" "os" "regexp" "sort" + "strconv" "strings" "time" + "git.curoverse.com/arvados.git/sdk/go/arvados" "git.curoverse.com/arvados.git/sdk/go/arvadostest" ) @@ -78,9 +85,7 @@ func DoGenericVolumeTests(t TB, factory TestableVolumeFactory) { testPutFullBlock(t, factory) testTrashUntrash(t, factory) - testEmptyTrashTrashLifetime0s(t, factory) - testEmptyTrashTrashLifetime3600s(t, factory) - testEmptyTrashTrashLifetime1s(t, factory) + testTrashEmptyTrashUntrash(t, factory) } // Put a test block, get it and verify content @@ -91,14 +96,13 @@ func testGet(t TB, factory TestableVolumeFactory) { v.PutRaw(TestHash, TestBlock) - buf, err := v.Get(TestHash) + buf := make([]byte, BlockSize) + n, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Fatal(err) } - bufs.Put(buf) - - if bytes.Compare(buf, TestBlock) != 0 { + if bytes.Compare(buf[:n], TestBlock) != 0 { t.Errorf("expected %s, got %s", string(TestBlock), string(buf)) } } @@ -109,7 +113,8 @@ func testGetNoSuchBlock(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() - if _, err := v.Get(TestHash2); err == nil { + buf := make([]byte, BlockSize) + if _, err := v.Get(context.Background(), TestHash2, buf); err == nil { t.Errorf("Expected error while getting non-existing block %v", TestHash2) } } @@ -121,7 +126,7 @@ func testCompareNonexistent(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() - err := v.Compare(TestHash, TestBlock) + err := v.Compare(context.Background(), TestHash, TestBlock) if err != os.ErrNotExist { t.Errorf("Got err %T %q, expected os.ErrNotExist", err, err) } @@ -136,7 +141,7 @@ func testCompareSameContent(t TB, factory TestableVolumeFactory, testHash string v.PutRaw(testHash, testData) // Compare the block locator with same content - err := v.Compare(testHash, testData) + err := v.Compare(context.Background(), testHash, testData) if err != nil { t.Errorf("Got err %q, expected nil", err) } @@ -154,7 +159,7 @@ func testCompareWithCollision(t TB, factory TestableVolumeFactory, testHash stri v.PutRaw(testHash, testDataA) // Compare the block locator with different content; collision - err := v.Compare(TestHash, testDataB) + err := v.Compare(context.Background(), TestHash, testDataB) if err == nil { t.Errorf("Got err nil, expected error due to collision") } @@ -170,7 +175,7 @@ func testCompareWithCorruptStoredData(t TB, factory TestableVolumeFactory, testH v.PutRaw(TestHash, testDataB) - err := v.Compare(testHash, testDataA) + err := v.Compare(context.Background(), testHash, testDataA) if err == nil || err == CollisionError { t.Errorf("Got err %+v, expected non-collision error", err) } @@ -186,12 +191,12 @@ func testPutBlockWithSameContent(t TB, factory TestableVolumeFactory, testHash s return } - err := v.Put(testHash, testData) + err := v.Put(context.Background(), testHash, testData) if err != nil { t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err) } - err = v.Put(testHash, testData) + err = v.Put(context.Background(), testHash, testData) if err != nil { t.Errorf("Got err putting block second time %q: %q, expected nil", TestBlock, err) } @@ -209,25 +214,23 @@ func testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testH v.PutRaw(testHash, testDataA) - putErr := v.Put(testHash, testDataB) - buf, getErr := v.Get(testHash) + putErr := v.Put(context.Background(), testHash, testDataB) + buf := make([]byte, BlockSize) + n, getErr := v.Get(context.Background(), testHash, buf) if putErr == nil { // Put must not return a nil error unless it has // overwritten the existing data. - if bytes.Compare(buf, testDataB) != 0 { - t.Errorf("Put succeeded but Get returned %+q, expected %+q", buf, testDataB) + if bytes.Compare(buf[:n], testDataB) != 0 { + t.Errorf("Put succeeded but Get returned %+q, expected %+q", buf[:n], testDataB) } } else { // It is permissible for Put to fail, but it must // leave us with either the original data, the new // data, or nothing at all. - if getErr == nil && bytes.Compare(buf, testDataA) != 0 && bytes.Compare(buf, testDataB) != 0 { - t.Errorf("Put failed but Get returned %+q, which is neither %+q nor %+q", buf, testDataA, testDataB) + if getErr == nil && bytes.Compare(buf[:n], testDataA) != 0 && bytes.Compare(buf[:n], testDataB) != 0 { + t.Errorf("Put failed but Get returned %+q, which is neither %+q nor %+q", buf[:n], testDataA, testDataB) } } - if getErr == nil { - bufs.Put(buf) - } } // Put and get multiple blocks @@ -240,49 +243,47 @@ func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) { return } - err := v.Put(TestHash, TestBlock) + err := v.Put(context.Background(), TestHash, TestBlock) if err != nil { t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err) } - err = v.Put(TestHash2, TestBlock2) + err = v.Put(context.Background(), TestHash2, TestBlock2) if err != nil { t.Errorf("Got err putting block %q: %q, expected nil", TestBlock2, err) } - err = v.Put(TestHash3, TestBlock3) + err = v.Put(context.Background(), TestHash3, TestBlock3) if err != nil { t.Errorf("Got err putting block %q: %q, expected nil", TestBlock3, err) } - data, err := v.Get(TestHash) + data := make([]byte, BlockSize) + n, err := v.Get(context.Background(), TestHash, data) if err != nil { t.Error(err) } else { - if bytes.Compare(data, TestBlock) != 0 { - t.Errorf("Block present, but got %+q, expected %+q", data, TestBlock) + if bytes.Compare(data[:n], TestBlock) != 0 { + t.Errorf("Block present, but got %+q, expected %+q", data[:n], TestBlock) } - bufs.Put(data) } - data, err = v.Get(TestHash2) + n, err = v.Get(context.Background(), TestHash2, data) if err != nil { t.Error(err) } else { - if bytes.Compare(data, TestBlock2) != 0 { - t.Errorf("Block present, but got %+q, expected %+q", data, TestBlock2) + if bytes.Compare(data[:n], TestBlock2) != 0 { + t.Errorf("Block present, but got %+q, expected %+q", data[:n], TestBlock2) } - bufs.Put(data) } - data, err = v.Get(TestHash3) + n, err = v.Get(context.Background(), TestHash3, data) if err != nil { t.Error(err) } else { - if bytes.Compare(data, TestBlock3) != 0 { - t.Errorf("Block present, but to %+q, expected %+q", data, TestBlock3) + if bytes.Compare(data[:n], TestBlock3) != 0 { + t.Errorf("Block present, but to %+q, expected %+q", data[:n], TestBlock3) } - bufs.Put(data) } } @@ -298,7 +299,7 @@ func testPutAndTouch(t TB, factory TestableVolumeFactory) { return } - if err := v.Put(TestHash, TestBlock); err != nil { + if err := v.Put(context.Background(), TestHash, TestBlock); err != nil { t.Error(err) } @@ -318,7 +319,7 @@ func testPutAndTouch(t TB, factory TestableVolumeFactory) { } // Write the same block again. - if err := v.Put(TestHash, TestBlock); err != nil { + if err := v.Put(context.Background(), TestHash, TestBlock); err != nil { t.Error(err) } @@ -361,10 +362,22 @@ func testIndexTo(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() + // minMtime and maxMtime are the minimum and maximum + // acceptable values the index can report for our test + // blocks. 1-second precision is acceptable. + minMtime := time.Now().UTC().UnixNano() + minMtime -= minMtime % 1e9 + v.PutRaw(TestHash, TestBlock) v.PutRaw(TestHash2, TestBlock2) v.PutRaw(TestHash3, TestBlock3) + maxMtime := time.Now().UTC().UnixNano() + if maxMtime%1e9 > 0 { + maxMtime -= maxMtime % 1e9 + maxMtime += 1e9 + } + // Blocks whose names aren't Keep hashes should be omitted from // index v.PutRaw("fffffffffnotreallyahashfffffffff", nil) @@ -377,15 +390,21 @@ func testIndexTo(t TB, factory TestableVolumeFactory) { indexRows := strings.Split(string(buf.Bytes()), "\n") sort.Strings(indexRows) sortedIndex := strings.Join(indexRows, "\n") - m, err := regexp.MatchString( - `^\n`+TestHash+`\+\d+ \d+\n`+ - TestHash3+`\+\d+ \d+\n`+ - TestHash2+`\+\d+ \d+$`, - sortedIndex) - if err != nil { - t.Error(err) - } else if !m { + m := regexp.MustCompile( + `^\n` + TestHash + `\+\d+ (\d+)\n` + + TestHash3 + `\+\d+ \d+\n` + + TestHash2 + `\+\d+ \d+$`, + ).FindStringSubmatch(sortedIndex) + if m == nil { t.Errorf("Got index %q for empty prefix", sortedIndex) + } else { + mtime, err := strconv.ParseInt(m[1], 10, 64) + if err != nil { + t.Error(err) + } else if mtime < minMtime || mtime > maxMtime { + t.Errorf("got %d for TestHash timestamp, expected %d <= t <= %d", + mtime, minMtime, maxMtime) + } } for _, prefix := range []string{"f", "f15", "f15ac"} { @@ -402,7 +421,7 @@ func testIndexTo(t TB, factory TestableVolumeFactory) { for _, prefix := range []string{"zero", "zip", "zilch"} { buf = new(bytes.Buffer) - v.IndexTo(prefix, buf) + err := v.IndexTo(prefix, buf) if err != nil { t.Errorf("Got error on IndexTo with no such prefix %v", err.Error()) } else if buf.Len() != 0 { @@ -417,49 +436,69 @@ func testIndexTo(t TB, factory TestableVolumeFactory) { func testDeleteNewBlock(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() - blobSignatureTTL = 300 * time.Second + theConfig.BlobSignatureTTL.Set("5m") if v.Writable() == false { return } - v.Put(TestHash, TestBlock) + v.Put(context.Background(), TestHash, TestBlock) if err := v.Trash(TestHash); err != nil { t.Error(err) } - data, err := v.Get(TestHash) + data := make([]byte, BlockSize) + n, err := v.Get(context.Background(), TestHash, data) if err != nil { t.Error(err) - } else { - if bytes.Compare(data, TestBlock) != 0 { - t.Errorf("Got data %+q, expected %+q", data, TestBlock) - } - bufs.Put(data) + } else if bytes.Compare(data[:n], TestBlock) != 0 { + t.Errorf("Got data %+q, expected %+q", data[:n], TestBlock) } } // Calling Delete() for a block with a timestamp older than -// blobSignatureTTL seconds in the past should delete the data. +// BlobSignatureTTL seconds in the past should delete the data. // Test is intended for only writable volumes func testDeleteOldBlock(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() - blobSignatureTTL = 300 * time.Second + theConfig.BlobSignatureTTL.Set("5m") if v.Writable() == false { return } - v.Put(TestHash, TestBlock) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.Put(context.Background(), TestHash, TestBlock) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) if err := v.Trash(TestHash); err != nil { t.Error(err) } - if _, err := v.Get(TestHash); err == nil || !os.IsNotExist(err) { + data := make([]byte, BlockSize) + if _, err := v.Get(context.Background(), TestHash, data); err == nil || !os.IsNotExist(err) { t.Errorf("os.IsNotExist(%v) should have been true", err) } + + _, err := v.Mtime(TestHash) + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } + + err = v.Compare(context.Background(), TestHash, TestBlock) + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } + + indexBuf := new(bytes.Buffer) + v.IndexTo("", indexBuf) + if strings.Contains(string(indexBuf.Bytes()), TestHash) { + t.Fatalf("Found trashed block in IndexTo") + } + + err = v.Touch(TestHash) + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } } // Calling Delete() for a block that does not exist should result in error. @@ -516,19 +555,20 @@ func testUpdateReadOnly(t TB, factory TestableVolumeFactory) { } v.PutRaw(TestHash, TestBlock) + buf := make([]byte, BlockSize) // Get from read-only volume should succeed - _, err := v.Get(TestHash) + _, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Errorf("got err %v, expected nil", err) } // Put a new block to read-only volume should result in error - err = v.Put(TestHash2, TestBlock2) + err = v.Put(context.Background(), TestHash2, TestBlock2) if err == nil { t.Errorf("Expected error when putting block in a read-only volume") } - _, err = v.Get(TestHash2) + _, err = v.Get(context.Background(), TestHash2, buf) if err == nil { t.Errorf("Expected error when getting block whose put in read-only volume failed") } @@ -546,7 +586,7 @@ func testUpdateReadOnly(t TB, factory TestableVolumeFactory) { } // Overwriting an existing block in read-only volume should result in error - err = v.Put(TestHash, TestBlock) + err = v.Put(context.Background(), TestHash, TestBlock) if err == nil { t.Errorf("Expected error when putting block in a read-only volume") } @@ -563,45 +603,45 @@ func testGetConcurrent(t TB, factory TestableVolumeFactory) { v.PutRaw(TestHash3, TestBlock3) sem := make(chan int) - go func(sem chan int) { - buf, err := v.Get(TestHash) + go func() { + buf := make([]byte, BlockSize) + n, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Errorf("err1: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("buf should be %s, is %s", string(TestBlock), string(buf)) + if bytes.Compare(buf[:n], TestBlock) != 0 { + t.Errorf("buf should be %s, is %s", string(TestBlock), string(buf[:n])) } sem <- 1 - }(sem) + }() - go func(sem chan int) { - buf, err := v.Get(TestHash2) + go func() { + buf := make([]byte, BlockSize) + n, err := v.Get(context.Background(), TestHash2, buf) if err != nil { t.Errorf("err2: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock2) != 0 { - t.Errorf("buf should be %s, is %s", string(TestBlock2), string(buf)) + if bytes.Compare(buf[:n], TestBlock2) != 0 { + t.Errorf("buf should be %s, is %s", string(TestBlock2), string(buf[:n])) } sem <- 1 - }(sem) + }() - go func(sem chan int) { - buf, err := v.Get(TestHash3) + go func() { + buf := make([]byte, BlockSize) + n, err := v.Get(context.Background(), TestHash3, buf) if err != nil { t.Errorf("err3: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock3) != 0 { - t.Errorf("buf should be %s, is %s", string(TestBlock3), string(buf)) + if bytes.Compare(buf[:n], TestBlock3) != 0 { + t.Errorf("buf should be %s, is %s", string(TestBlock3), string(buf[:n])) } sem <- 1 - }(sem) + }() // Wait for all goroutines to finish - for done := 0; done < 3; { - done += <-sem + for done := 0; done < 3; done++ { + <-sem } } @@ -617,7 +657,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) { sem := make(chan int) go func(sem chan int) { - err := v.Put(TestHash, TestBlock) + err := v.Put(context.Background(), TestHash, TestBlock) if err != nil { t.Errorf("err1: %v", err) } @@ -625,7 +665,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) { }(sem) go func(sem chan int) { - err := v.Put(TestHash2, TestBlock2) + err := v.Put(context.Background(), TestHash2, TestBlock2) if err != nil { t.Errorf("err2: %v", err) } @@ -633,7 +673,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) { }(sem) go func(sem chan int) { - err := v.Put(TestHash3, TestBlock3) + err := v.Put(context.Background(), TestHash3, TestBlock3) if err != nil { t.Errorf("err3: %v", err) } @@ -641,36 +681,34 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) { }(sem) // Wait for all goroutines to finish - for done := 0; done < 3; { - done += <-sem + for done := 0; done < 3; done++ { + <-sem } // Double check that we actually wrote the blocks we expected to write. - buf, err := v.Get(TestHash) + buf := make([]byte, BlockSize) + n, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Errorf("Get #1: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("Get #1: expected %s, got %s", string(TestBlock), string(buf)) + if bytes.Compare(buf[:n], TestBlock) != 0 { + t.Errorf("Get #1: expected %s, got %s", string(TestBlock), string(buf[:n])) } - buf, err = v.Get(TestHash2) + n, err = v.Get(context.Background(), TestHash2, buf) if err != nil { t.Errorf("Get #2: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock2) != 0 { - t.Errorf("Get #2: expected %s, got %s", string(TestBlock2), string(buf)) + if bytes.Compare(buf[:n], TestBlock2) != 0 { + t.Errorf("Get #2: expected %s, got %s", string(TestBlock2), string(buf[:n])) } - buf, err = v.Get(TestHash3) + n, err = v.Get(context.Background(), TestHash3, buf) if err != nil { t.Errorf("Get #3: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock3) != 0 { - t.Errorf("Get #3: expected %s, got %s", string(TestBlock3), string(buf)) + if bytes.Compare(buf[:n], TestBlock3) != 0 { + t.Errorf("Get #3: expected %s, got %s", string(TestBlock3), string(buf[:n])) } } @@ -687,22 +725,21 @@ func testPutFullBlock(t TB, factory TestableVolumeFactory) { wdata[0] = 'a' wdata[BlockSize-1] = 'z' hash := fmt.Sprintf("%x", md5.Sum(wdata)) - err := v.Put(hash, wdata) + err := v.Put(context.Background(), hash, wdata) if err != nil { t.Fatal(err) } - rdata, err := v.Get(hash) + buf := make([]byte, BlockSize) + n, err := v.Get(context.Background(), hash, buf) if err != nil { t.Error(err) - } else { - defer bufs.Put(rdata) } - if bytes.Compare(rdata, wdata) != 0 { - t.Error("rdata != wdata") + if bytes.Compare(buf[:n], wdata) != 0 { + t.Error("buf %+q != wdata %+q", buf[:n], wdata) } } -// With trashLifetime != 0, perform: +// With TrashLifetime != 0, perform: // Trash an old block - which either raises ErrNotImplemented or succeeds // Untrash - which either raises ErrNotImplemented or succeeds // Get - which must succeed @@ -710,36 +747,36 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() defer func() { - trashLifetime = 0 + theConfig.TrashLifetime = 0 }() - trashLifetime = 3600 + theConfig.TrashLifetime.Set("1h") // put block and backdate it v.PutRaw(TestHash, TestBlock) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) - buf, err := v.Get(TestHash) + buf := make([]byte, BlockSize) + n, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Fatal(err) } - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("Got data %+q, expected %+q", buf, TestBlock) + if bytes.Compare(buf[:n], TestBlock) != 0 { + t.Errorf("Got data %+q, expected %+q", buf[:n], TestBlock) } - bufs.Put(buf) // Trash err = v.Trash(TestHash) if v.Writable() == false { if err != MethodDisabledError { - t.Error(err) + t.Fatal(err) } } else if err != nil { if err != ErrNotImplemented { - t.Error(err) + t.Fatal(err) } } else { - _, err = v.Get(TestHash) + _, err = v.Get(context.Background(), TestHash, buf) if err == nil || !os.IsNotExist(err) { t.Errorf("os.IsNotExist(%v) should have been true", err) } @@ -752,166 +789,227 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) { } // Get the block - after trash and untrash sequence - buf, err = v.Get(TestHash) + n, err = v.Get(context.Background(), TestHash, buf) if err != nil { t.Fatal(err) } - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("Got data %+q, expected %+q", buf, TestBlock) + if bytes.Compare(buf[:n], TestBlock) != 0 { + t.Errorf("Got data %+q, expected %+q", buf[:n], TestBlock) } - bufs.Put(buf) } -// With trashLifetime == 0, perform: -// Trash an old block - which either raises ErrNotImplemented or succeeds to delete it -// Untrash - which either raises ErrNotImplemented or is a no-op for the deleted block -// Get - which must fail to find the block, since it was deleted and hence not untrashed -func testEmptyTrashTrashLifetime0s(t TB, factory TestableVolumeFactory) { +func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() - defer func() { - trashLifetime = 0 - doneEmptyingTrash <- true - }() - - trashLifetime = 0 - trashCheckInterval = 1 + defer func(orig arvados.Duration) { + theConfig.TrashLifetime = orig + }(theConfig.TrashLifetime) - go emptyTrash(trashCheckInterval) + checkGet := func() error { + buf := make([]byte, BlockSize) + n, err := v.Get(context.Background(), TestHash, buf) + if err != nil { + return err + } + if bytes.Compare(buf[:n], TestBlock) != 0 { + t.Fatalf("Got data %+q, expected %+q", buf[:n], TestBlock) + } - // Trash old block; since trashLifetime = 0, Trash actually deletes the block - err := trashUntrashOldBlock(t, v, 0) + _, err = v.Mtime(TestHash) + if err != nil { + return err + } - // Get it; for writable volumes, this should not find the block since it was deleted - buf, err := v.Get(TestHash) - if err != nil { - if !os.IsNotExist(err) { - t.Errorf("os.IsNotExist(%v) should have been true", err) + err = v.Compare(context.Background(), TestHash, TestBlock) + if err != nil { + return err } - } else { - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("Got data %+q, expected %+q", buf, TestBlock) + + indexBuf := new(bytes.Buffer) + v.IndexTo("", indexBuf) + if !strings.Contains(string(indexBuf.Bytes()), TestHash) { + return os.ErrNotExist } - bufs.Put(buf) + + return nil } -} -// With large trashLifetime, perform: -// Run emptyTrash goroutine with much smaller trashCheckInterval -// Trash an old block - which either raises ErrNotImplemented or succeeds -// Untrash - which either raises ErrNotImplemented or succeeds -// Get - which must find the block -func testEmptyTrashTrashLifetime3600s(t TB, factory TestableVolumeFactory) { - v := factory(t) - defer v.Teardown() - defer func() { - trashLifetime = 0 - doneEmptyingTrash <- true - }() + // First set: EmptyTrash before reaching the trash deadline. + + theConfig.TrashLifetime.Set("1h") + + v.PutRaw(TestHash, TestBlock) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) + + err := checkGet() + if err != nil { + t.Fatal(err) + } + + // Trash the block + err = v.Trash(TestHash) + if err == MethodDisabledError || err == ErrNotImplemented { + // Skip the trash tests for read-only volumes, and + // volume types that don't support TrashLifetime>0. + return + } - trashLifetime = 3600 - trashCheckInterval = 1 + err = checkGet() + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } - go emptyTrash(trashCheckInterval) + err = v.Touch(TestHash) + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } - // Trash old block - err := trashUntrashOldBlock(t, v, 2) + v.EmptyTrash() - // Get is expected to succeed after untrash before EmptyTrash - // It is still found on readonly volumes - buf, err := v.Get(TestHash) + // Even after emptying the trash, we can untrash our block + // because the deadline hasn't been reached. + err = v.Untrash(TestHash) if err != nil { - if !os.IsNotExist(err) { - t.Errorf("os.IsNotExist(%v) should have been true", err) - } - } else { - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("Got data %+q, expected %+q", buf, TestBlock) - } - bufs.Put(buf) + t.Fatal(err) } -} -// With trashLifetime = 1, perform: -// Run emptyTrash goroutine -// Trash an old block - which either raises ErrNotImplemented or succeeds -// Untrash - after emptyTrash goroutine ticks, and hence does not actually untrash -// Get - which must fail to find the block -func testEmptyTrashTrashLifetime1s(t TB, factory TestableVolumeFactory) { - v := factory(t) - defer v.Teardown() - defer func() { - trashLifetime = 0 - doneEmptyingTrash <- true - }() + err = checkGet() + if err != nil { + t.Fatal(err) + } + + err = v.Touch(TestHash) + if err != nil { + t.Fatal(err) + } + + // Because we Touch'ed, need to backdate again for next set of tests + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) + + // If the only block in the trash has already been untrashed, + // most volumes will fail a subsequent Untrash with a 404, but + // it's also acceptable for Untrash to succeed. + err = v.Untrash(TestHash) + if err != nil && !os.IsNotExist(err) { + t.Fatalf("Expected success or os.IsNotExist(), but got: %v", err) + } - volumes = append(volumes, v) + // The additional Untrash should not interfere with our + // already-untrashed copy. + err = checkGet() + if err != nil { + t.Fatal(err) + } - trashLifetime = 1 - trashCheckInterval = 1 + // Untrash might have updated the timestamp, so backdate again + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) - go emptyTrash(trashCheckInterval) + // Second set: EmptyTrash after the trash deadline has passed. - // Trash old block and untrash a little after first trashCheckInterval - err := trashUntrashOldBlock(t, v, 3) + theConfig.TrashLifetime.Set("1ns") - // Get is expected to fail due to EmptyTrash before Untrash - // It is still found on readonly volumes - buf, err := v.Get(TestHash) + err = v.Trash(TestHash) if err != nil { - if !os.IsNotExist(err) { - t.Errorf("os.IsNotExist(%v) should have been true", err) - } - } else { - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("Got data %+q, expected %+q", buf, TestBlock) - } - bufs.Put(buf) + t.Fatal(err) + } + err = checkGet() + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) } -} -// Put a block, backdate it, trash it, untrash it after the untrashAfter seconds -func trashUntrashOldBlock(t TB, v TestableVolume, untrashAfter int) error { - // put block and backdate it + // Even though 1ns has passed, we can untrash because we + // haven't called EmptyTrash yet. + err = v.Untrash(TestHash) + if err != nil { + t.Fatal(err) + } + err = checkGet() + if err != nil { + t.Fatal(err) + } + + // Trash it again, and this time call EmptyTrash so it really + // goes away. + // (In Azure volumes, un/trash changes Mtime, so first backdate again) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) + err = v.Trash(TestHash) + err = checkGet() + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } + v.EmptyTrash() + + // Untrash won't find it + err = v.Untrash(TestHash) + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } + + // Get block won't find it + err = checkGet() + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } + + // Third set: If the same data block gets written again after + // being trashed, and then the trash gets emptied, the newer + // un-trashed copy doesn't get deleted along with it. + v.PutRaw(TestHash, TestBlock) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) - buf, err := v.Get(TestHash) + theConfig.TrashLifetime.Set("1ns") + err = v.Trash(TestHash) if err != nil { t.Fatal(err) } - if bytes.Compare(buf, TestBlock) != 0 { - t.Fatalf("Got data %+q, expected %+q", buf, TestBlock) + err = checkGet() + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) } - bufs.Put(buf) - // Trash + v.PutRaw(TestHash, TestBlock) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) + + // EmptyTrash should not delete the untrashed copy. + v.EmptyTrash() + err = checkGet() + if err != nil { + t.Fatal(err) + } + + // Fourth set: If the same data block gets trashed twice with + // different deadlines A and C, and then the trash is emptied + // at intermediate time B (A < B < C), it is still possible to + // untrash the block whose deadline is "C". + + v.PutRaw(TestHash, TestBlock) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) + + theConfig.TrashLifetime.Set("1ns") err = v.Trash(TestHash) if err != nil { - if err != ErrNotImplemented && err != MethodDisabledError { - t.Fatal(err) - } else { - // To test emptyTrash goroutine effectively, we need to give the - // ticker a couple rounds, adding some sleep time to the test. - // This delay is unnecessary for volumes that are currently - // not yet supporting trashLifetime > 0 (this case is already - // covered in the testTrashUntrash already) - return err - } - } else { - _, err = v.Get(TestHash) - if err == nil || !os.IsNotExist(err) { - t.Fatalf("os.IsNotExist(%v) should have been true", err) - } + t.Fatal(err) + } + + v.PutRaw(TestHash, TestBlock) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) + + theConfig.TrashLifetime.Set("1h") + err = v.Trash(TestHash) + if err != nil { + t.Fatal(err) } - // Untrash after give wait time - time.Sleep(time.Duration(untrashAfter) * time.Second) + // EmptyTrash should not prevent us from recovering the + // time.Hour ("C") trash + v.EmptyTrash() err = v.Untrash(TestHash) if err != nil { - if err != ErrNotImplemented && err != MethodDisabledError { - t.Fatal(err) - } + t.Fatal(err) + } + err = checkGet() + if err != nil { + t.Fatal(err) } - return err }