X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/3b1598bb557a30f9a896ed36988702c5ae9f2ba9..1c859533008748afa273aa7c379ef5aec88d4ce6:/services/keepstore/volume_generic_test.go diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go index 193d9d2c2c..61088f10fa 100644 --- a/services/keepstore/volume_generic_test.go +++ b/services/keepstore/volume_generic_test.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "crypto/md5" + "fmt" "os" "regexp" "sort" @@ -22,12 +24,22 @@ func DoGenericVolumeTests(t *testing.T, factory TestableVolumeFactory) { testGet(t, factory) testGetNoSuchBlock(t, factory) - testCompareSameContent(t, factory) - testCompareWithDifferentContent(t, factory) - testCompareWithBadData(t, factory) - - testPutBlockWithSameContent(t, factory) - testPutBlockWithDifferentContent(t, factory) + testCompareNonexistent(t, factory) + testCompareSameContent(t, factory, TestHash, TestBlock) + testCompareSameContent(t, factory, EmptyHash, EmptyBlock) + testCompareWithCollision(t, factory, TestHash, TestBlock, []byte("baddata")) + testCompareWithCollision(t, factory, TestHash, TestBlock, EmptyBlock) + testCompareWithCollision(t, factory, EmptyHash, EmptyBlock, TestBlock) + testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, []byte("baddata")) + testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, EmptyBlock) + testCompareWithCorruptStoredData(t, factory, EmptyHash, EmptyBlock, []byte("baddata")) + + testPutBlockWithSameContent(t, factory, TestHash, TestBlock) + testPutBlockWithSameContent(t, factory, EmptyHash, EmptyBlock) + testPutBlockWithDifferentContent(t, factory, TestHash, TestBlock, TestBlock2) + testPutBlockWithDifferentContent(t, factory, TestHash, EmptyBlock, TestBlock) + testPutBlockWithDifferentContent(t, factory, TestHash, TestBlock, EmptyBlock) + testPutBlockWithDifferentContent(t, factory, EmptyHash, EmptyBlock, TestBlock) testPutMultipleBlocks(t, factory) testPutAndTouch(t, factory) @@ -49,6 +61,8 @@ func DoGenericVolumeTests(t *testing.T, factory TestableVolumeFactory) { testGetConcurrent(t, factory) testPutConcurrent(t, factory) + + testPutFullBlock(t, factory) } // Put a test block, get it and verify content @@ -61,7 +75,7 @@ func testGet(t *testing.T, factory TestableVolumeFactory) { buf, err := v.Get(TestHash) if err != nil { - t.Error(err) + t.Fatal(err) } bufs.Put(buf) @@ -82,56 +96,71 @@ func testGetNoSuchBlock(t *testing.T, factory TestableVolumeFactory) { } } +// Compare() should return os.ErrNotExist if the block does not exist. +// Otherwise, writing new data causes CompareAndTouch() to generate +// error logs even though everything is working fine. +func testCompareNonexistent(t *testing.T, factory TestableVolumeFactory) { + v := factory(t) + defer v.Teardown() + + err := v.Compare(TestHash, TestBlock) + if err != os.ErrNotExist { + t.Errorf("Got err %T %q, expected os.ErrNotExist", err, err) + } +} + // Put a test block and compare the locator with same content // Test should pass for both writable and read-only volumes -func testCompareSameContent(t *testing.T, factory TestableVolumeFactory) { +func testCompareSameContent(t *testing.T, factory TestableVolumeFactory, testHash string, testData []byte) { v := factory(t) defer v.Teardown() - v.PutRaw(TestHash, TestBlock) + v.PutRaw(testHash, testData) // Compare the block locator with same content - err := v.Compare(TestHash, TestBlock) + err := v.Compare(testHash, testData) if err != nil { t.Errorf("Got err %q, expected nil", err) } } -// Put a test block and compare the locator with a different content -// Expect error due to collision +// Test behavior of Compare() when stored data matches expected +// checksum but differs from new data we need to store. Requires +// testHash = md5(testDataA). +// // Test should pass for both writable and read-only volumes -func testCompareWithDifferentContent(t *testing.T, factory TestableVolumeFactory) { +func testCompareWithCollision(t *testing.T, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) { v := factory(t) defer v.Teardown() - v.PutRaw(TestHash, TestBlock) + v.PutRaw(testHash, testDataA) // Compare the block locator with different content; collision - err := v.Compare(TestHash, []byte("baddata")) + err := v.Compare(TestHash, testDataB) if err == nil { - t.Errorf("Expected error due to collision") + t.Errorf("Got err nil, expected error due to collision") } } -// Put a test block with bad data (hash does not match, but Put does not verify) -// Compare the locator with good data whose hash matches with locator -// Expect error due to corruption. +// Test behavior of Compare() when stored data has become +// corrupted. Requires testHash = md5(testDataA) != md5(testDataB). +// // Test should pass for both writable and read-only volumes -func testCompareWithBadData(t *testing.T, factory TestableVolumeFactory) { +func testCompareWithCorruptStoredData(t *testing.T, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) { v := factory(t) defer v.Teardown() - v.PutRaw(TestHash, []byte("baddata")) + v.PutRaw(TestHash, testDataB) - err := v.Compare(TestHash, TestBlock) - if err == nil { - t.Errorf("Expected error due to corruption") + err := v.Compare(testHash, testDataA) + if err == nil || err == CollisionError { + t.Errorf("Got err %+v, expected non-collision error", err) } } // Put a block and put again with same content // Test is intended for only writable volumes -func testPutBlockWithSameContent(t *testing.T, factory TestableVolumeFactory) { +func testPutBlockWithSameContent(t *testing.T, factory TestableVolumeFactory, testHash string, testData []byte) { v := factory(t) defer v.Teardown() @@ -139,12 +168,12 @@ func testPutBlockWithSameContent(t *testing.T, factory TestableVolumeFactory) { return } - err := v.Put(TestHash, TestBlock) + err := v.Put(testHash, testData) if err != nil { t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err) } - err = v.Put(TestHash, TestBlock) + err = v.Put(testHash, testData) if err != nil { t.Errorf("Got err putting block second time %q: %q, expected nil", TestBlock, err) } @@ -152,7 +181,7 @@ func testPutBlockWithSameContent(t *testing.T, factory TestableVolumeFactory) { // Put a block and put again with different content // Test is intended for only writable volumes -func testPutBlockWithDifferentContent(t *testing.T, factory TestableVolumeFactory) { +func testPutBlockWithDifferentContent(t *testing.T, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) { v := factory(t) defer v.Teardown() @@ -160,25 +189,25 @@ func testPutBlockWithDifferentContent(t *testing.T, factory TestableVolumeFactor return } - err := v.Put(TestHash, TestBlock) + err := v.Put(testHash, testDataA) if err != nil { - t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err) + t.Errorf("Got err putting block %q: %q, expected nil", testDataA, err) } - putErr := v.Put(TestHash, TestBlock2) - buf, getErr := v.Get(TestHash) + putErr := v.Put(testHash, testDataB) + buf, getErr := v.Get(testHash) if putErr == nil { // Put must not return a nil error unless it has // overwritten the existing data. - if bytes.Compare(buf, TestBlock2) != 0 { - t.Errorf("Put succeeded but Get returned %+v, expected %+v", buf, TestBlock2) + if bytes.Compare(buf, testDataB) != 0 { + t.Errorf("Put succeeded but Get returned %+q, expected %+q", buf, testDataB) } } else { // It is permissible for Put to fail, but it must // leave us with either the original data, the new // data, or nothing at all. - if getErr == nil && bytes.Compare(buf, TestBlock) != 0 && bytes.Compare(buf, TestBlock2) != 0 { - t.Errorf("Put failed but Get returned %+v, which is neither %+v nor %+v", buf, TestBlock, TestBlock2) + if getErr == nil && bytes.Compare(buf, testDataA) != 0 && bytes.Compare(buf, testDataB) != 0 { + t.Errorf("Put failed but Get returned %+q, which is neither %+q nor %+q", buf, testDataA, testDataB) } } if getErr == nil { @@ -214,26 +243,32 @@ func testPutMultipleBlocks(t *testing.T, factory TestableVolumeFactory) { data, err := v.Get(TestHash) if err != nil { t.Error(err) - } else if bytes.Compare(data, TestBlock) != 0 { - t.Errorf("Block present, but content is incorrect: Expected: %v Found: %v", data, TestBlock) + } else { + if bytes.Compare(data, TestBlock) != 0 { + t.Errorf("Block present, but got %+q, expected %+q", data, TestBlock) + } + bufs.Put(data) } - bufs.Put(data) data, err = v.Get(TestHash2) if err != nil { t.Error(err) - } else if bytes.Compare(data, TestBlock2) != 0 { - t.Errorf("Block present, but content is incorrect: Expected: %v Found: %v", data, TestBlock2) + } else { + if bytes.Compare(data, TestBlock2) != 0 { + t.Errorf("Block present, but got %+q, expected %+q", data, TestBlock2) + } + bufs.Put(data) } - bufs.Put(data) data, err = v.Get(TestHash3) if err != nil { t.Error(err) - } else if bytes.Compare(data, TestBlock3) != 0 { - t.Errorf("Block present, but content is incorrect: Expected: %v Found: %v", data, TestBlock3) + } else { + if bytes.Compare(data, TestBlock3) != 0 { + t.Errorf("Block present, but to %+q, expected %+q", data, TestBlock3) + } + bufs.Put(data) } - bufs.Put(data) } // testPutAndTouch @@ -315,6 +350,13 @@ func testIndexTo(t *testing.T, factory TestableVolumeFactory) { v.PutRaw(TestHash2, TestBlock2) v.PutRaw(TestHash3, TestBlock3) + // Blocks whose names aren't Keep hashes should be omitted from + // index + v.PutRaw("fffffffffnotreallyahashfffffffff", nil) + v.PutRaw("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", nil) + v.PutRaw("f0000000000000000000000000000000f", nil) + v.PutRaw("f00", nil) + buf := new(bytes.Buffer) v.IndexTo("", buf) indexRows := strings.Split(string(buf.Bytes()), "\n") @@ -360,6 +402,7 @@ func testIndexTo(t *testing.T, factory TestableVolumeFactory) { func testDeleteNewBlock(t *testing.T, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() + blobSignatureTTL = 300 * time.Second if v.Writable() == false { return @@ -373,10 +416,12 @@ func testDeleteNewBlock(t *testing.T, factory TestableVolumeFactory) { data, err := v.Get(TestHash) if err != nil { t.Error(err) - } else if bytes.Compare(data, TestBlock) != 0 { - t.Error("Block still present, but content is incorrect: %+v != %+v", data, TestBlock) + } else { + if bytes.Compare(data, TestBlock) != 0 { + t.Errorf("Got data %+q, expected %+q", data, TestBlock) + } + bufs.Put(data) } - bufs.Put(data) } // Calling Delete() for a block with a timestamp older than @@ -385,19 +430,20 @@ func testDeleteNewBlock(t *testing.T, factory TestableVolumeFactory) { func testDeleteOldBlock(t *testing.T, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() + blobSignatureTTL = 300 * time.Second if v.Writable() == false { return } v.Put(TestHash, TestBlock) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL*time.Second)) + v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) if err := v.Delete(TestHash); err != nil { t.Error(err) } if _, err := v.Get(TestHash); err == nil || !os.IsNotExist(err) { - t.Errorf("os.IsNotExist(%v) should have been true", err.Error()) + t.Errorf("os.IsNotExist(%v) should have been true", err) } } @@ -612,3 +658,31 @@ func testPutConcurrent(t *testing.T, factory TestableVolumeFactory) { t.Errorf("Get #3: expected %s, got %s", string(TestBlock3), string(buf)) } } + +// Write and read back a full size block +func testPutFullBlock(t *testing.T, factory TestableVolumeFactory) { + v := factory(t) + defer v.Teardown() + + if !v.Writable() { + return + } + + wdata := make([]byte, BlockSize) + wdata[0] = 'a' + wdata[BlockSize-1] = 'z' + hash := fmt.Sprintf("%x", md5.Sum(wdata)) + err := v.Put(hash, wdata) + if err != nil { + t.Fatal(err) + } + rdata, err := v.Get(hash) + if err != nil { + t.Error(err) + } else { + defer bufs.Put(rdata) + } + if bytes.Compare(rdata, wdata) != 0 { + t.Error("rdata != wdata") + } +}