X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f09da35ed284ebc1ed1e941f3e3cb63b06a35d51..f51b562b9def7f6bc17e0cc52b60ffc2641d40b5:/services/datamanager/summary/summary_test.go diff --git a/services/datamanager/summary/summary_test.go b/services/datamanager/summary/summary_test.go index ff73b876df..ea76df4d34 100644 --- a/services/datamanager/summary/summary_test.go +++ b/services/datamanager/summary/summary_test.go @@ -5,13 +5,14 @@ import ( "git.curoverse.com/arvados.git/services/datamanager/collection" "git.curoverse.com/arvados.git/services/datamanager/keep" "reflect" + "sort" "testing" ) func BlockSetFromSlice(digests []int) (bs BlockSet) { bs = make(BlockSet) for _, digest := range digests { - bs.Insert(blockdigest.MakeTestBlockDigest(digest)) + bs.Insert(blockdigest.MakeTestDigestWithSize(digest)) } return } @@ -31,9 +32,31 @@ func (cis CollectionIndexSet) ToSlice() (ints []int) { ints[i] = collectionIndex i++ } + sort.Ints(ints) return } +// Helper method to meet interface expected by older tests. +func SummarizeReplication(readCollections collection.ReadCollections, + keepServerInfo keep.ReadServers) (rs ReplicationSummary) { + return BucketReplication(readCollections, keepServerInfo). + SummarizeBuckets(readCollections) +} + +// Takes a map from block digest to replication level and represents +// it in a keep.ReadServers structure. +func SpecifyReplication(digestToReplication map[int]int) (rs keep.ReadServers) { + rs.BlockToServers = make(map[blockdigest.DigestWithSize][]keep.BlockServerInfo) + for digest, replication := range digestToReplication { + rs.BlockToServers[blockdigest.MakeTestDigestWithSize(digest)] = + make([]keep.BlockServerInfo, replication) + } + return +} + +// Verifies that +// blocks.ToCollectionIndexSet(rc.BlockToCollectionIndices) returns +// expectedCollections. func VerifyToCollectionIndexSet( t *testing.T, blocks []int, @@ -43,10 +66,10 @@ func VerifyToCollectionIndexSet( expected := CollectionIndexSetFromSlice(expectedCollections) rc := collection.ReadCollections{ - BlockToCollectionIndices: map[blockdigest.BlockDigest][]int{}, + BlockToCollectionIndices: map[blockdigest.DigestWithSize][]int{}, } for digest, indices := range blockToCollectionIndices { - rc.BlockToCollectionIndices[blockdigest.MakeTestBlockDigest(digest)] = indices + rc.BlockToCollectionIndices[blockdigest.MakeTestDigestWithSize(digest)] = indices } returned := make(CollectionIndexSet) @@ -62,55 +85,42 @@ func VerifyToCollectionIndexSet( } func TestToCollectionIndexSet(t *testing.T) { - VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4:[]int{1}}, []int{1}) - VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4:[]int{1,9}}, []int{1,9}) - VerifyToCollectionIndexSet(t, []int{5,6}, - map[int][]int{5:[]int{2,3}, 6:[]int{3,4}}, - []int{2,3,4}) + VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{6: []int{0}}, []int{0}) + VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1}}, []int{1}) + VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1, 9}}, []int{1, 9}) + VerifyToCollectionIndexSet(t, []int{5, 6}, + map[int][]int{5: []int{2, 3}, 6: []int{3, 4}}, + []int{2, 3, 4}) + VerifyToCollectionIndexSet(t, []int{5, 6}, + map[int][]int{5: []int{8}, 6: []int{4}}, + []int{4, 8}) + VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{5: []int{0}}, []int{}) } - func TestSimpleSummary(t *testing.T) { rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{ - collection.TestCollectionSpec{ - ReplicationLevel: 1, - Blocks: []int{1, 2}, - }, + collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}}, }) + rc.Summarize(nil) + cIndex := rc.CollectionIndicesForTesting() - rc.Summarize() - - // The internals aren't actually examined, so we can reuse the same one. - dummyBlockServerInfo := keep.BlockServerInfo{} - - blockDigest1 := blockdigest.MakeTestBlockDigest(1) - blockDigest2 := blockdigest.MakeTestBlockDigest(2) - - keepInfo := keep.ReadServers{ - BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{ - blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo}, - blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo}, - }, - } - - returnedSummary := SummarizeReplication(rc, keepInfo) - - c := rc.UuidToCollection["col0"] + keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1}) expectedSummary := ReplicationSummary{ - CollectionBlocksNotInKeep: BlockSet{}, - UnderReplicatedBlocks: BlockSet{}, - OverReplicatedBlocks: BlockSet{}, - CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1,2}), + CollectionBlocksNotInKeep: BlockSet{}, + UnderReplicatedBlocks: BlockSet{}, + OverReplicatedBlocks: BlockSet{}, + CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1, 2}), KeepBlocksNotInCollections: BlockSet{}, - CollectionsNotFullyInKeep: CollectionIndexSet{}, - UnderReplicatedCollections: CollectionIndexSet{}, - OverReplicatedCollections: CollectionIndexSet{}, - CorrectlyReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c.Uuid]}), + CollectionsNotFullyInKeep: CollectionIndexSet{}, + UnderReplicatedCollections: CollectionIndexSet{}, + OverReplicatedCollections: CollectionIndexSet{}, + CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}), } + returnedSummary := SummarizeReplication(rc, keepInfo) + if !reflect.DeepEqual(returnedSummary, expectedSummary) { t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary) } @@ -118,155 +128,91 @@ func TestSimpleSummary(t *testing.T) { func TestMissingBlock(t *testing.T) { rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{ - collection.TestCollectionSpec{ - ReplicationLevel: 1, - Blocks: []int{1, 2}, - }, + collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}}, }) + rc.Summarize(nil) + cIndex := rc.CollectionIndicesForTesting() - rc.Summarize() - - // The internals aren't actually examined, so we can reuse the same one. - dummyBlockServerInfo := keep.BlockServerInfo{} - - blockDigest1 := blockdigest.MakeTestBlockDigest(1) - - keepInfo := keep.ReadServers{ - BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{ - blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo}, - }, - } - - returnedSummary := SummarizeReplication(rc, keepInfo) - - c := rc.UuidToCollection["col0"] + keepInfo := SpecifyReplication(map[int]int{1: 1}) expectedSummary := ReplicationSummary{ - CollectionBlocksNotInKeep: BlockSetFromSlice([]int{2}), - UnderReplicatedBlocks: BlockSet{}, - OverReplicatedBlocks: BlockSet{}, - CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1}), + CollectionBlocksNotInKeep: BlockSetFromSlice([]int{2}), + UnderReplicatedBlocks: BlockSet{}, + OverReplicatedBlocks: BlockSet{}, + CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1}), KeepBlocksNotInCollections: BlockSet{}, - CollectionsNotFullyInKeep: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c.Uuid]}), + CollectionsNotFullyInKeep: CollectionIndexSetFromSlice([]int{cIndex[0]}), UnderReplicatedCollections: CollectionIndexSet{}, OverReplicatedCollections: CollectionIndexSet{}, CorrectlyReplicatedCollections: CollectionIndexSet{}, } + returnedSummary := SummarizeReplication(rc, keepInfo) + if !reflect.DeepEqual(returnedSummary, expectedSummary) { - t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary) + t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", + expectedSummary, + returnedSummary) } } func TestUnderAndOverReplicatedBlocks(t *testing.T) { rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{ - collection.TestCollectionSpec{ - ReplicationLevel: 2, - Blocks: []int{1, 2}, - }, + collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{1, 2}}, }) + rc.Summarize(nil) + cIndex := rc.CollectionIndicesForTesting() - rc.Summarize() - - // The internals aren't actually examined, so we can reuse the same one. - dummyBlockServerInfo := keep.BlockServerInfo{} - - blockDigest1 := blockdigest.MakeTestBlockDigest(1) - blockDigest2 := blockdigest.MakeTestBlockDigest(2) - - keepInfo := keep.ReadServers{ - BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{ - blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo}, - blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo}, - }, - } - - returnedSummary := SummarizeReplication(rc, keepInfo) - - c := rc.UuidToCollection["col0"] + keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 3}) expectedSummary := ReplicationSummary{ - CollectionBlocksNotInKeep: BlockSet{}, - UnderReplicatedBlocks: BlockSetFromSlice([]int{1}), - OverReplicatedBlocks: BlockSetFromSlice([]int{2}), + CollectionBlocksNotInKeep: BlockSet{}, + UnderReplicatedBlocks: BlockSetFromSlice([]int{1}), + OverReplicatedBlocks: BlockSetFromSlice([]int{2}), CorrectlyReplicatedBlocks: BlockSet{}, KeepBlocksNotInCollections: BlockSet{}, - CollectionsNotFullyInKeep: CollectionIndexSet{}, - UnderReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c.Uuid]}), - OverReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c.Uuid]}), + CollectionsNotFullyInKeep: CollectionIndexSet{}, + UnderReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}), + OverReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}), CorrectlyReplicatedCollections: CollectionIndexSet{}, } + returnedSummary := SummarizeReplication(rc, keepInfo) + if !reflect.DeepEqual(returnedSummary, expectedSummary) { - t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary) + t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", + expectedSummary, + returnedSummary) } } func TestMixedReplication(t *testing.T) { rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{ - collection.TestCollectionSpec{ - ReplicationLevel: 1, - Blocks: []int{1, 2}, - }, - collection.TestCollectionSpec{ - ReplicationLevel: 1, - Blocks: []int{3, 4}, - }, - collection.TestCollectionSpec{ - ReplicationLevel: 2, - Blocks: []int{5, 6}, - }, + collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}}, + collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{3, 4}}, + collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{5, 6}}, }) + rc.Summarize(nil) + cIndex := rc.CollectionIndicesForTesting() - rc.Summarize() - - // The internals aren't actually examined, so we can reuse the same one. - dummyBlockServerInfo := keep.BlockServerInfo{} - - keepInfo := keep.ReadServers{ - BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{ - blockdigest.MakeTestBlockDigest(1): []keep.BlockServerInfo{dummyBlockServerInfo}, - blockdigest.MakeTestBlockDigest(2): []keep.BlockServerInfo{dummyBlockServerInfo}, - blockdigest.MakeTestBlockDigest(3): []keep.BlockServerInfo{dummyBlockServerInfo}, - blockdigest.MakeTestBlockDigest(5): []keep.BlockServerInfo{dummyBlockServerInfo}, - blockdigest.MakeTestBlockDigest(6): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo}, - blockdigest.MakeTestBlockDigest(7): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo}, - }, - } - - returnedSummary := SummarizeReplication(rc, keepInfo) - - c0 := rc.UuidToCollection["col0"] - c1 := rc.UuidToCollection["col1"] - c2 := rc.UuidToCollection["col2"] + keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1, 3: 1, 5: 1, 6: 3, 7: 2}) expectedSummary := ReplicationSummary{ - CollectionBlocksNotInKeep: BlockSetFromSlice([]int{4}), - UnderReplicatedBlocks: BlockSetFromSlice([]int{5}), - OverReplicatedBlocks: BlockSetFromSlice([]int{6}), - CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1,2,3}), + CollectionBlocksNotInKeep: BlockSetFromSlice([]int{4}), + UnderReplicatedBlocks: BlockSetFromSlice([]int{5}), + OverReplicatedBlocks: BlockSetFromSlice([]int{6}), + CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1, 2, 3}), KeepBlocksNotInCollections: BlockSetFromSlice([]int{7}), - CollectionsNotFullyInKeep: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c1.Uuid]}), - UnderReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c2.Uuid]}), - OverReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c2.Uuid]}), - CorrectlyReplicatedCollections: CollectionIndexSetFromSlice( - []int{rc.CollectionUuidToIndex[c0.Uuid]}), + CollectionsNotFullyInKeep: CollectionIndexSetFromSlice([]int{cIndex[1]}), + UnderReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[2]}), + OverReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[2]}), + CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}), } - tempCis := make(CollectionIndexSet) - returnedSummary.CollectionBlocksNotInKeep.ToCollectionIndexSet(rc, &tempCis) - t.Logf("blocks not in keep: %v, collections not fully in keep: %v", - returnedSummary.CollectionBlocksNotInKeep, - tempCis) + returnedSummary := SummarizeReplication(rc, keepInfo) if !reflect.DeepEqual(returnedSummary, expectedSummary) { t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUuid, rc.BlockToCollectionIndices)