--- /dev/null
+package summary
+
+import (
+ "git.curoverse.com/arvados.git/sdk/go/blockdigest"
+ "git.curoverse.com/arvados.git/services/datamanager/collection"
+ "git.curoverse.com/arvados.git/services/datamanager/keep"
+ "reflect"
+ "testing"
+)
+
+func BlockSetFromSlice(digests []int) (bs BlockSet) {
+ bs = make(BlockSet)
+ for _, digest := range digests {
+ bs.Insert(blockdigest.MakeTestBlockDigest(digest))
+ }
+ return
+}
+
+func CollectionIndexSetFromSlice(indices []int) (cis CollectionIndexSet) {
+ cis = make(CollectionIndexSet)
+ for _, index := range indices {
+ cis.Insert(index)
+ }
+ return
+}
+
+func (cis CollectionIndexSet) ToSlice() (ints []int) {
+ ints = make([]int, len(cis))
+ i := 0
+ for collectionIndex := range cis {
+ ints[i] = collectionIndex
+ i++
+ }
+ return
+}
+
+func VerifyToCollectionIndexSet(
+ t *testing.T,
+ blocks []int,
+ blockToCollectionIndices map[int][]int,
+ expectedCollections []int) {
+
+ expected := CollectionIndexSetFromSlice(expectedCollections)
+
+ rc := collection.ReadCollections{
+ BlockToCollectionIndices: map[blockdigest.BlockDigest][]int{},
+ }
+ for digest, indices := range blockToCollectionIndices {
+ rc.BlockToCollectionIndices[blockdigest.MakeTestBlockDigest(digest)] = indices
+ }
+
+ returned := make(CollectionIndexSet)
+ BlockSetFromSlice(blocks).ToCollectionIndexSet(rc, &returned)
+
+ if !reflect.DeepEqual(returned, expected) {
+ t.Errorf("Expected %v.ToCollectionIndexSet(%v) to return \n %v \n but instead received \n %v",
+ blocks,
+ blockToCollectionIndices,
+ expectedCollections,
+ returned.ToSlice())
+ }
+}
+
+func TestToCollectionIndexSet(t *testing.T) {
+ VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4:[]int{1}}, []int{1})
+ VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4:[]int{1,9}}, []int{1,9})
+ VerifyToCollectionIndexSet(t, []int{5,6},
+ map[int][]int{5:[]int{2,3}, 6:[]int{3,4}},
+ []int{2,3,4})
+}
+
+
+func TestSimpleSummary(t *testing.T) {
+ rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
+ collection.TestCollectionSpec{
+ ReplicationLevel: 1,
+ Blocks: []int{1, 2},
+ },
+ })
+
+ rc.Summarize()
+
+ // The internals aren't actually examined, so we can reuse the same one.
+ dummyBlockServerInfo := keep.BlockServerInfo{}
+
+ blockDigest1 := blockdigest.MakeTestBlockDigest(1)
+ blockDigest2 := blockdigest.MakeTestBlockDigest(2)
+
+ keepInfo := keep.ReadServers{
+ BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
+ blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
+ blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo},
+ },
+ }
+
+ returnedSummary := SummarizeReplication(rc, keepInfo)
+
+ c := rc.UuidToCollection["col0"]
+
+ expectedSummary := ReplicationSummary{
+ CollectionBlocksNotInKeep: BlockSet{},
+ UnderReplicatedBlocks: BlockSet{},
+ OverReplicatedBlocks: BlockSet{},
+ CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1,2}),
+ KeepBlocksNotInCollections: BlockSet{},
+
+ CollectionsNotFullyInKeep: CollectionIndexSet{},
+ UnderReplicatedCollections: CollectionIndexSet{},
+ OverReplicatedCollections: CollectionIndexSet{},
+ CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
+ []int{rc.CollectionUuidToIndex[c.Uuid]}),
+ }
+
+ if !reflect.DeepEqual(returnedSummary, expectedSummary) {
+ t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
+ }
+}
+
+func TestMissingBlock(t *testing.T) {
+ rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
+ collection.TestCollectionSpec{
+ ReplicationLevel: 1,
+ Blocks: []int{1, 2},
+ },
+ })
+
+ rc.Summarize()
+
+ // The internals aren't actually examined, so we can reuse the same one.
+ dummyBlockServerInfo := keep.BlockServerInfo{}
+
+ blockDigest1 := blockdigest.MakeTestBlockDigest(1)
+
+ keepInfo := keep.ReadServers{
+ BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
+ blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
+ },
+ }
+
+ returnedSummary := SummarizeReplication(rc, keepInfo)
+
+ c := rc.UuidToCollection["col0"]
+
+ expectedSummary := ReplicationSummary{
+ CollectionBlocksNotInKeep: BlockSetFromSlice([]int{2}),
+ UnderReplicatedBlocks: BlockSet{},
+ OverReplicatedBlocks: BlockSet{},
+ CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1}),
+ KeepBlocksNotInCollections: BlockSet{},
+
+ CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
+ []int{rc.CollectionUuidToIndex[c.Uuid]}),
+ UnderReplicatedCollections: CollectionIndexSet{},
+ OverReplicatedCollections: CollectionIndexSet{},
+ CorrectlyReplicatedCollections: CollectionIndexSet{},
+ }
+
+ if !reflect.DeepEqual(returnedSummary, expectedSummary) {
+ t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
+ }
+}
+
+func TestUnderAndOverReplicatedBlocks(t *testing.T) {
+ rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
+ collection.TestCollectionSpec{
+ ReplicationLevel: 2,
+ Blocks: []int{1, 2},
+ },
+ })
+
+ rc.Summarize()
+
+ // The internals aren't actually examined, so we can reuse the same one.
+ dummyBlockServerInfo := keep.BlockServerInfo{}
+
+ blockDigest1 := blockdigest.MakeTestBlockDigest(1)
+ blockDigest2 := blockdigest.MakeTestBlockDigest(2)
+
+ keepInfo := keep.ReadServers{
+ BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
+ blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
+ blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
+ },
+ }
+
+ returnedSummary := SummarizeReplication(rc, keepInfo)
+
+ c := rc.UuidToCollection["col0"]
+
+ expectedSummary := ReplicationSummary{
+ CollectionBlocksNotInKeep: BlockSet{},
+ UnderReplicatedBlocks: BlockSetFromSlice([]int{1}),
+ OverReplicatedBlocks: BlockSetFromSlice([]int{2}),
+ CorrectlyReplicatedBlocks: BlockSet{},
+ KeepBlocksNotInCollections: BlockSet{},
+
+ CollectionsNotFullyInKeep: CollectionIndexSet{},
+ UnderReplicatedCollections: CollectionIndexSetFromSlice(
+ []int{rc.CollectionUuidToIndex[c.Uuid]}),
+ OverReplicatedCollections: CollectionIndexSetFromSlice(
+ []int{rc.CollectionUuidToIndex[c.Uuid]}),
+ CorrectlyReplicatedCollections: CollectionIndexSet{},
+ }
+
+ if !reflect.DeepEqual(returnedSummary, expectedSummary) {
+ t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
+ }
+}
+
+func TestMixedReplication(t *testing.T) {
+ rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
+ collection.TestCollectionSpec{
+ ReplicationLevel: 1,
+ Blocks: []int{1, 2},
+ },
+ collection.TestCollectionSpec{
+ ReplicationLevel: 1,
+ Blocks: []int{3, 4},
+ },
+ collection.TestCollectionSpec{
+ ReplicationLevel: 2,
+ Blocks: []int{5, 6},
+ },
+ })
+
+ rc.Summarize()
+
+ // The internals aren't actually examined, so we can reuse the same one.
+ dummyBlockServerInfo := keep.BlockServerInfo{}
+
+ keepInfo := keep.ReadServers{
+ BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
+ blockdigest.MakeTestBlockDigest(1): []keep.BlockServerInfo{dummyBlockServerInfo},
+ blockdigest.MakeTestBlockDigest(2): []keep.BlockServerInfo{dummyBlockServerInfo},
+ blockdigest.MakeTestBlockDigest(3): []keep.BlockServerInfo{dummyBlockServerInfo},
+ blockdigest.MakeTestBlockDigest(5): []keep.BlockServerInfo{dummyBlockServerInfo},
+ blockdigest.MakeTestBlockDigest(6): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
+ blockdigest.MakeTestBlockDigest(7): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo},
+ },
+ }
+
+ returnedSummary := SummarizeReplication(rc, keepInfo)
+
+ c0 := rc.UuidToCollection["col0"]
+ c1 := rc.UuidToCollection["col1"]
+ c2 := rc.UuidToCollection["col2"]
+
+ expectedSummary := ReplicationSummary{
+ CollectionBlocksNotInKeep: BlockSetFromSlice([]int{4}),
+ UnderReplicatedBlocks: BlockSetFromSlice([]int{5}),
+ OverReplicatedBlocks: BlockSetFromSlice([]int{6}),
+ CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1,2,3}),
+ KeepBlocksNotInCollections: BlockSetFromSlice([]int{7}),
+
+ CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
+ []int{rc.CollectionUuidToIndex[c1.Uuid]}),
+ UnderReplicatedCollections: CollectionIndexSetFromSlice(
+ []int{rc.CollectionUuidToIndex[c2.Uuid]}),
+ OverReplicatedCollections: CollectionIndexSetFromSlice(
+ []int{rc.CollectionUuidToIndex[c2.Uuid]}),
+ CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
+ []int{rc.CollectionUuidToIndex[c0.Uuid]}),
+ }
+
+ tempCis := make(CollectionIndexSet)
+ returnedSummary.CollectionBlocksNotInKeep.ToCollectionIndexSet(rc, &tempCis)
+ t.Logf("blocks not in keep: %v, collections not fully in keep: %v",
+ returnedSummary.CollectionBlocksNotInKeep,
+ tempCis)
+
+ if !reflect.DeepEqual(returnedSummary, expectedSummary) {
+ t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUuid, rc.BlockToCollectionIndices)
+ }
+}
projects / arvados.git / blobdiff