"git.curoverse.com/arvados.git/services/datamanager/collection"
"git.curoverse.com/arvados.git/services/datamanager/keep"
"reflect"
+ "sort"
"testing"
)
func BlockSetFromSlice(digests []int) (bs BlockSet) {
bs = make(BlockSet)
for _, digest := range digests {
- bs.Insert(blockdigest.MakeTestBlockDigest(digest))
+ bs.Insert(blockdigest.MakeTestDigestWithSize(digest))
}
return
}
ints[i] = collectionIndex
i++
}
+ sort.Ints(ints)
return
}
+// Helper method to meet interface expected by older tests.
+func SummarizeReplication(readCollections collection.ReadCollections,
+ keepServerInfo keep.ReadServers) (rs ReplicationSummary) {
+ return BucketReplication(readCollections, keepServerInfo).
+ SummarizeBuckets(readCollections)
+}
+
+// Takes a map from block digest to replication level and represents
+// it in a keep.ReadServers structure.
+func SpecifyReplication(digestToReplication map[int]int) (rs keep.ReadServers) {
+ rs.BlockToServers = make(map[blockdigest.DigestWithSize][]keep.BlockServerInfo)
+ for digest, replication := range digestToReplication {
+ rs.BlockToServers[blockdigest.MakeTestDigestWithSize(digest)] =
+ make([]keep.BlockServerInfo, replication)
+ }
+ return
+}
+
+// Verifies that
+// blocks.ToCollectionIndexSet(rc.BlockToCollectionIndices) returns
+// expectedCollections.
func VerifyToCollectionIndexSet(
t *testing.T,
blocks []int,
expected := CollectionIndexSetFromSlice(expectedCollections)
rc := collection.ReadCollections{
- BlockToCollectionIndices: map[blockdigest.BlockDigest][]int{},
+ BlockToCollectionIndices: map[blockdigest.DigestWithSize][]int{},
}
for digest, indices := range blockToCollectionIndices {
- rc.BlockToCollectionIndices[blockdigest.MakeTestBlockDigest(digest)] = indices
+ rc.BlockToCollectionIndices[blockdigest.MakeTestDigestWithSize(digest)] = indices
}
returned := make(CollectionIndexSet)
}
func TestToCollectionIndexSet(t *testing.T) {
- VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4:[]int{1}}, []int{1})
- VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4:[]int{1,9}}, []int{1,9})
- VerifyToCollectionIndexSet(t, []int{5,6},
- map[int][]int{5:[]int{2,3}, 6:[]int{3,4}},
- []int{2,3,4})
+ VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{6: {0}}, []int{0})
+ VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: {1}}, []int{1})
+ VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: {1, 9}}, []int{1, 9})
+ VerifyToCollectionIndexSet(t, []int{5, 6},
+ map[int][]int{5: {2, 3}, 6: {3, 4}},
+ []int{2, 3, 4})
+ VerifyToCollectionIndexSet(t, []int{5, 6},
+ map[int][]int{5: {8}, 6: {4}},
+ []int{4, 8})
+ VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{5: {0}}, []int{})
}
-
func TestSimpleSummary(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{
- ReplicationLevel: 1,
- Blocks: []int{1, 2},
- },
+ {ReplicationLevel: 1, Blocks: []int{1, 2}},
})
+ rc.Summarize(nil)
+ cIndex := rc.CollectionIndicesForTesting()
- rc.Summarize()
-
- // The internals aren't actually examined, so we can reuse the same one.
- dummyBlockServerInfo := keep.BlockServerInfo{}
-
- blockDigest1 := blockdigest.MakeTestBlockDigest(1)
- blockDigest2 := blockdigest.MakeTestBlockDigest(2)
-
- keepInfo := keep.ReadServers{
- BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
- blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
- blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo},
- },
- }
-
- returnedSummary := SummarizeReplication(rc, keepInfo)
-
- c := rc.UuidToCollection["col0"]
+ keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1})
expectedSummary := ReplicationSummary{
- CollectionBlocksNotInKeep: BlockSet{},
- UnderReplicatedBlocks: BlockSet{},
- OverReplicatedBlocks: BlockSet{},
- CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1,2}),
+ CollectionBlocksNotInKeep: BlockSet{},
+ UnderReplicatedBlocks: BlockSet{},
+ OverReplicatedBlocks: BlockSet{},
+ CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1, 2}),
KeepBlocksNotInCollections: BlockSet{},
- CollectionsNotFullyInKeep: CollectionIndexSet{},
- UnderReplicatedCollections: CollectionIndexSet{},
- OverReplicatedCollections: CollectionIndexSet{},
- CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
- []int{rc.CollectionUuidToIndex[c.Uuid]}),
+ CollectionsNotFullyInKeep: CollectionIndexSet{},
+ UnderReplicatedCollections: CollectionIndexSet{},
+ OverReplicatedCollections: CollectionIndexSet{},
+ CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}),
}
+ returnedSummary := SummarizeReplication(rc, keepInfo)
+
if !reflect.DeepEqual(returnedSummary, expectedSummary) {
t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
}
func TestMissingBlock(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{
- ReplicationLevel: 1,
- Blocks: []int{1, 2},
- },
+ {ReplicationLevel: 1, Blocks: []int{1, 2}},
})
+ rc.Summarize(nil)
+ cIndex := rc.CollectionIndicesForTesting()
- rc.Summarize()
-
- // The internals aren't actually examined, so we can reuse the same one.
- dummyBlockServerInfo := keep.BlockServerInfo{}
-
- blockDigest1 := blockdigest.MakeTestBlockDigest(1)
-
- keepInfo := keep.ReadServers{
- BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
- blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
- },
- }
-
- returnedSummary := SummarizeReplication(rc, keepInfo)
-
- c := rc.UuidToCollection["col0"]
+ keepInfo := SpecifyReplication(map[int]int{1: 1})
expectedSummary := ReplicationSummary{
- CollectionBlocksNotInKeep: BlockSetFromSlice([]int{2}),
- UnderReplicatedBlocks: BlockSet{},
- OverReplicatedBlocks: BlockSet{},
- CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1}),
+ CollectionBlocksNotInKeep: BlockSetFromSlice([]int{2}),
+ UnderReplicatedBlocks: BlockSet{},
+ OverReplicatedBlocks: BlockSet{},
+ CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1}),
KeepBlocksNotInCollections: BlockSet{},
- CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
- []int{rc.CollectionUuidToIndex[c.Uuid]}),
+ CollectionsNotFullyInKeep: CollectionIndexSetFromSlice([]int{cIndex[0]}),
UnderReplicatedCollections: CollectionIndexSet{},
OverReplicatedCollections: CollectionIndexSet{},
CorrectlyReplicatedCollections: CollectionIndexSet{},
}
+ returnedSummary := SummarizeReplication(rc, keepInfo)
+
if !reflect.DeepEqual(returnedSummary, expectedSummary) {
- t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
+ t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v",
+ expectedSummary,
+ returnedSummary)
}
}
func TestUnderAndOverReplicatedBlocks(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{
- ReplicationLevel: 2,
- Blocks: []int{1, 2},
- },
+ {ReplicationLevel: 2, Blocks: []int{1, 2}},
})
+ rc.Summarize(nil)
+ cIndex := rc.CollectionIndicesForTesting()
- rc.Summarize()
-
- // The internals aren't actually examined, so we can reuse the same one.
- dummyBlockServerInfo := keep.BlockServerInfo{}
-
- blockDigest1 := blockdigest.MakeTestBlockDigest(1)
- blockDigest2 := blockdigest.MakeTestBlockDigest(2)
-
- keepInfo := keep.ReadServers{
- BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
- blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
- blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
- },
- }
-
- returnedSummary := SummarizeReplication(rc, keepInfo)
-
- c := rc.UuidToCollection["col0"]
+ keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 3})
expectedSummary := ReplicationSummary{
- CollectionBlocksNotInKeep: BlockSet{},
- UnderReplicatedBlocks: BlockSetFromSlice([]int{1}),
- OverReplicatedBlocks: BlockSetFromSlice([]int{2}),
+ CollectionBlocksNotInKeep: BlockSet{},
+ UnderReplicatedBlocks: BlockSetFromSlice([]int{1}),
+ OverReplicatedBlocks: BlockSetFromSlice([]int{2}),
CorrectlyReplicatedBlocks: BlockSet{},
KeepBlocksNotInCollections: BlockSet{},
- CollectionsNotFullyInKeep: CollectionIndexSet{},
- UnderReplicatedCollections: CollectionIndexSetFromSlice(
- []int{rc.CollectionUuidToIndex[c.Uuid]}),
- OverReplicatedCollections: CollectionIndexSetFromSlice(
- []int{rc.CollectionUuidToIndex[c.Uuid]}),
+ CollectionsNotFullyInKeep: CollectionIndexSet{},
+ UnderReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}),
+ OverReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}),
CorrectlyReplicatedCollections: CollectionIndexSet{},
}
+ returnedSummary := SummarizeReplication(rc, keepInfo)
+
if !reflect.DeepEqual(returnedSummary, expectedSummary) {
- t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
+ t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v",
+ expectedSummary,
+ returnedSummary)
}
}
func TestMixedReplication(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{
- ReplicationLevel: 1,
- Blocks: []int{1, 2},
- },
- collection.TestCollectionSpec{
- ReplicationLevel: 1,
- Blocks: []int{3, 4},
- },
- collection.TestCollectionSpec{
- ReplicationLevel: 2,
- Blocks: []int{5, 6},
- },
+ {ReplicationLevel: 1, Blocks: []int{1, 2}},
+ {ReplicationLevel: 1, Blocks: []int{3, 4}},
+ {ReplicationLevel: 2, Blocks: []int{5, 6}},
})
+ rc.Summarize(nil)
+ cIndex := rc.CollectionIndicesForTesting()
- rc.Summarize()
-
- // The internals aren't actually examined, so we can reuse the same one.
- dummyBlockServerInfo := keep.BlockServerInfo{}
-
- keepInfo := keep.ReadServers{
- BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
- blockdigest.MakeTestBlockDigest(1): []keep.BlockServerInfo{dummyBlockServerInfo},
- blockdigest.MakeTestBlockDigest(2): []keep.BlockServerInfo{dummyBlockServerInfo},
- blockdigest.MakeTestBlockDigest(3): []keep.BlockServerInfo{dummyBlockServerInfo},
- blockdigest.MakeTestBlockDigest(5): []keep.BlockServerInfo{dummyBlockServerInfo},
- blockdigest.MakeTestBlockDigest(6): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
- blockdigest.MakeTestBlockDigest(7): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo},
- },
- }
-
- returnedSummary := SummarizeReplication(rc, keepInfo)
-
- c0 := rc.UuidToCollection["col0"]
- c1 := rc.UuidToCollection["col1"]
- c2 := rc.UuidToCollection["col2"]
+ keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1, 3: 1, 5: 1, 6: 3, 7: 2})
expectedSummary := ReplicationSummary{
- CollectionBlocksNotInKeep: BlockSetFromSlice([]int{4}),
- UnderReplicatedBlocks: BlockSetFromSlice([]int{5}),
- OverReplicatedBlocks: BlockSetFromSlice([]int{6}),
- CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1,2,3}),
+ CollectionBlocksNotInKeep: BlockSetFromSlice([]int{4}),
+ UnderReplicatedBlocks: BlockSetFromSlice([]int{5}),
+ OverReplicatedBlocks: BlockSetFromSlice([]int{6}),
+ CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1, 2, 3}),
KeepBlocksNotInCollections: BlockSetFromSlice([]int{7}),
- CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
- []int{rc.CollectionUuidToIndex[c1.Uuid]}),
- UnderReplicatedCollections: CollectionIndexSetFromSlice(
- []int{rc.CollectionUuidToIndex[c2.Uuid]}),
- OverReplicatedCollections: CollectionIndexSetFromSlice(
- []int{rc.CollectionUuidToIndex[c2.Uuid]}),
- CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
- []int{rc.CollectionUuidToIndex[c0.Uuid]}),
+ CollectionsNotFullyInKeep: CollectionIndexSetFromSlice([]int{cIndex[1]}),
+ UnderReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[2]}),
+ OverReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[2]}),
+ CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}),
}
- tempCis := make(CollectionIndexSet)
- returnedSummary.CollectionBlocksNotInKeep.ToCollectionIndexSet(rc, &tempCis)
- t.Logf("blocks not in keep: %v, collections not fully in keep: %v",
- returnedSummary.CollectionBlocksNotInKeep,
- tempCis)
+ returnedSummary := SummarizeReplication(rc, keepInfo)
if !reflect.DeepEqual(returnedSummary, expectedSummary) {
- t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUuid, rc.BlockToCollectionIndices)
+ t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUUID, rc.BlockToCollectionIndices)
}
}
projects / arvados.git / blobdiff