X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b1233abce0628266a1805e52d9f9fc651c2a5c59..a74c81b035c67d299e2a7298f8db3d368a578510:/services/datamanager/summary/summary_test.go diff --git a/services/datamanager/summary/summary_test.go b/services/datamanager/summary/summary_test.go index fb9e18bac1..8268404127 100644 --- a/services/datamanager/summary/summary_test.go +++ b/services/datamanager/summary/summary_test.go @@ -5,13 +5,14 @@ import ( "git.curoverse.com/arvados.git/services/datamanager/collection" "git.curoverse.com/arvados.git/services/datamanager/keep" "reflect" + "sort" "testing" ) func BlockSetFromSlice(digests []int) (bs BlockSet) { bs = make(BlockSet) for _, digest := range digests { - bs.Insert(blockdigest.MakeTestBlockDigest(digest)) + bs.Insert(blockdigest.MakeTestDigestWithSize(digest)) } return } @@ -31,9 +32,31 @@ func (cis CollectionIndexSet) ToSlice() (ints []int) { ints[i] = collectionIndex i++ } + sort.Ints(ints) return } +// Helper method to meet interface expected by older tests. +func SummarizeReplication(readCollections collection.ReadCollections, + keepServerInfo keep.ReadServers) (rs ReplicationSummary) { + return BucketReplication(readCollections, keepServerInfo). + SummarizeBuckets(readCollections) +} + +// Takes a map from block digest to replication level and represents +// it in a keep.ReadServers structure. +func SpecifyReplication(digestToReplication map[int]int) (rs keep.ReadServers) { + rs.BlockToServers = make(map[blockdigest.DigestWithSize][]keep.BlockServerInfo) + for digest, replication := range digestToReplication { + rs.BlockToServers[blockdigest.MakeTestDigestWithSize(digest)] = + make([]keep.BlockServerInfo, replication) + } + return +} + +// Verifies that +// blocks.ToCollectionIndexSet(rc.BlockToCollectionIndices) returns +// expectedCollections. func VerifyToCollectionIndexSet( t *testing.T, blocks []int, @@ -43,10 +66,10 @@ func VerifyToCollectionIndexSet( expected := CollectionIndexSetFromSlice(expectedCollections) rc := collection.ReadCollections{ - BlockToCollectionIndices: map[blockdigest.BlockDigest][]int{}, + BlockToCollectionIndices: map[blockdigest.DigestWithSize][]int{}, } for digest, indices := range blockToCollectionIndices { - rc.BlockToCollectionIndices[blockdigest.MakeTestBlockDigest(digest)] = indices + rc.BlockToCollectionIndices[blockdigest.MakeTestDigestWithSize(digest)] = indices } returned := make(CollectionIndexSet) @@ -62,39 +85,26 @@ func VerifyToCollectionIndexSet( } func TestToCollectionIndexSet(t *testing.T) { - VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1}}, []int{1}) - VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1, 9}}, []int{1, 9}) + VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{6: {0}}, []int{0}) + VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: {1}}, []int{1}) + VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: {1, 9}}, []int{1, 9}) VerifyToCollectionIndexSet(t, []int{5, 6}, - map[int][]int{5: []int{2, 3}, 6: []int{3, 4}}, + map[int][]int{5: {2, 3}, 6: {3, 4}}, []int{2, 3, 4}) + VerifyToCollectionIndexSet(t, []int{5, 6}, + map[int][]int{5: {8}, 6: {4}}, + []int{4, 8}) + VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{5: {0}}, []int{}) } func TestSimpleSummary(t *testing.T) { rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{ - collection.TestCollectionSpec{ - ReplicationLevel: 1, - Blocks: []int{1, 2}, - }, + {ReplicationLevel: 1, Blocks: []int{1, 2}}, }) + rc.Summarize(nil) + cIndex := rc.CollectionIndicesForTesting() - rc.Summarize() - - // The internals aren't actually examined, so we can reuse the same one. - dummyBlockServerInfo := keep.BlockServerInfo{} - - blockDigest1 := blockdigest.MakeTestBlockDigest(1) - blockDigest2 := blockdigest.MakeTestBlockDigest(2) - - keepInfo := keep.ReadServers{ - BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{ - blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo}, - blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo}, - }, - } - - returnedSummary := SummarizeReplication(rc, keepInfo) - - c := rc.UuidToCollection["col0"] + keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1}) expectedSummary := ReplicationSummary{ CollectionBlocksNotInKeep: BlockSet{}, @@ -103,13 +113,14 @@ func TestSimpleSummary(t *testing.T) { CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1, 2}), KeepBlocksNotInCollections: BlockSet{}, - CollectionsNotFullyInKeep: CollectionIndexSet{}, - UnderReplicatedCollections: CollectionIndexSet{}, - OverReplicatedCollections: CollectionIndexSet{}, - CorrectlyReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c.Uuid]}), + CollectionsNotFullyInKeep: CollectionIndexSet{}, + UnderReplicatedCollections: CollectionIndexSet{}, + OverReplicatedCollections: CollectionIndexSet{}, + CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}), } + returnedSummary := SummarizeReplication(rc, keepInfo) + if !reflect.DeepEqual(returnedSummary, expectedSummary) { t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary) } @@ -117,28 +128,12 @@ func TestSimpleSummary(t *testing.T) { func TestMissingBlock(t *testing.T) { rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{ - collection.TestCollectionSpec{ - ReplicationLevel: 1, - Blocks: []int{1, 2}, - }, + {ReplicationLevel: 1, Blocks: []int{1, 2}}, }) + rc.Summarize(nil) + cIndex := rc.CollectionIndicesForTesting() - rc.Summarize() - - // The internals aren't actually examined, so we can reuse the same one. - dummyBlockServerInfo := keep.BlockServerInfo{} - - blockDigest1 := blockdigest.MakeTestBlockDigest(1) - - keepInfo := keep.ReadServers{ - BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{ - blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo}, - }, - } - - returnedSummary := SummarizeReplication(rc, keepInfo) - - c := rc.UuidToCollection["col0"] + keepInfo := SpecifyReplication(map[int]int{1: 1}) expectedSummary := ReplicationSummary{ CollectionBlocksNotInKeep: BlockSetFromSlice([]int{2}), @@ -147,44 +142,29 @@ func TestMissingBlock(t *testing.T) { CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1}), KeepBlocksNotInCollections: BlockSet{}, - CollectionsNotFullyInKeep: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c.Uuid]}), + CollectionsNotFullyInKeep: CollectionIndexSetFromSlice([]int{cIndex[0]}), UnderReplicatedCollections: CollectionIndexSet{}, OverReplicatedCollections: CollectionIndexSet{}, CorrectlyReplicatedCollections: CollectionIndexSet{}, } + returnedSummary := SummarizeReplication(rc, keepInfo) + if !reflect.DeepEqual(returnedSummary, expectedSummary) { - t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary) + t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", + expectedSummary, + returnedSummary) } } func TestUnderAndOverReplicatedBlocks(t *testing.T) { rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{ - collection.TestCollectionSpec{ - ReplicationLevel: 2, - Blocks: []int{1, 2}, - }, + {ReplicationLevel: 2, Blocks: []int{1, 2}}, }) + rc.Summarize(nil) + cIndex := rc.CollectionIndicesForTesting() - rc.Summarize() - - // The internals aren't actually examined, so we can reuse the same one. - dummyBlockServerInfo := keep.BlockServerInfo{} - - blockDigest1 := blockdigest.MakeTestBlockDigest(1) - blockDigest2 := blockdigest.MakeTestBlockDigest(2) - - keepInfo := keep.ReadServers{ - BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{ - blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo}, - blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo}, - }, - } - - returnedSummary := SummarizeReplication(rc, keepInfo) - - c := rc.UuidToCollection["col0"] + keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 3}) expectedSummary := ReplicationSummary{ CollectionBlocksNotInKeep: BlockSet{}, @@ -193,56 +173,31 @@ func TestUnderAndOverReplicatedBlocks(t *testing.T) { CorrectlyReplicatedBlocks: BlockSet{}, KeepBlocksNotInCollections: BlockSet{}, - CollectionsNotFullyInKeep: CollectionIndexSet{}, - UnderReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c.Uuid]}), - OverReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c.Uuid]}), + CollectionsNotFullyInKeep: CollectionIndexSet{}, + UnderReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}), + OverReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}), CorrectlyReplicatedCollections: CollectionIndexSet{}, } + returnedSummary := SummarizeReplication(rc, keepInfo) + if !reflect.DeepEqual(returnedSummary, expectedSummary) { - t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary) + t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", + expectedSummary, + returnedSummary) } } func TestMixedReplication(t *testing.T) { rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{ - collection.TestCollectionSpec{ - ReplicationLevel: 1, - Blocks: []int{1, 2}, - }, - collection.TestCollectionSpec{ - ReplicationLevel: 1, - Blocks: []int{3, 4}, - }, - collection.TestCollectionSpec{ - ReplicationLevel: 2, - Blocks: []int{5, 6}, - }, + {ReplicationLevel: 1, Blocks: []int{1, 2}}, + {ReplicationLevel: 1, Blocks: []int{3, 4}}, + {ReplicationLevel: 2, Blocks: []int{5, 6}}, }) + rc.Summarize(nil) + cIndex := rc.CollectionIndicesForTesting() - rc.Summarize() - - // The internals aren't actually examined, so we can reuse the same one. - dummyBlockServerInfo := keep.BlockServerInfo{} - - keepInfo := keep.ReadServers{ - BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{ - blockdigest.MakeTestBlockDigest(1): []keep.BlockServerInfo{dummyBlockServerInfo}, - blockdigest.MakeTestBlockDigest(2): []keep.BlockServerInfo{dummyBlockServerInfo}, - blockdigest.MakeTestBlockDigest(3): []keep.BlockServerInfo{dummyBlockServerInfo}, - blockdigest.MakeTestBlockDigest(5): []keep.BlockServerInfo{dummyBlockServerInfo}, - blockdigest.MakeTestBlockDigest(6): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo}, - blockdigest.MakeTestBlockDigest(7): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo}, - }, - } - - returnedSummary := SummarizeReplication(rc, keepInfo) - - c0 := rc.UuidToCollection["col0"] - c1 := rc.UuidToCollection["col1"] - c2 := rc.UuidToCollection["col2"] + keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1, 3: 1, 5: 1, 6: 3, 7: 2}) expectedSummary := ReplicationSummary{ CollectionBlocksNotInKeep: BlockSetFromSlice([]int{4}), @@ -251,17 +206,15 @@ func TestMixedReplication(t *testing.T) { CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1, 2, 3}), KeepBlocksNotInCollections: BlockSetFromSlice([]int{7}), - CollectionsNotFullyInKeep: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c1.Uuid]}), - UnderReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c2.Uuid]}), - OverReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c2.Uuid]}), - CorrectlyReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c0.Uuid]}), + CollectionsNotFullyInKeep: CollectionIndexSetFromSlice([]int{cIndex[1]}), + UnderReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[2]}), + OverReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[2]}), + CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}), } + returnedSummary := SummarizeReplication(rc, keepInfo) + if !reflect.DeepEqual(returnedSummary, expectedSummary) { - t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUuid, rc.BlockToCollectionIndices) + t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUUID, rc.BlockToCollectionIndices) } }