services/datamanager/summary/summary_test.go

   1 package summary
   2
   3 import (
   4         "git.curoverse.com/arvados.git/sdk/go/blockdigest"
   5         "git.curoverse.com/arvados.git/services/datamanager/collection"
   6         "git.curoverse.com/arvados.git/services/datamanager/keep"
   7         "reflect"
   8         "sort"
   9         "testing"
  10 )
  11
  12 func BlockSetFromSlice(digests []int) (bs BlockSet) {
  13         bs = make(BlockSet)
  14         for _, digest := range digests {
  15                 bs.Insert(blockdigest.MakeTestBlockDigest(digest))
  16         }
  17         return
  18 }
  19
  20 func CollectionIndexSetFromSlice(indices []int) (cis CollectionIndexSet) {
  21         cis = make(CollectionIndexSet)
  22         for _, index := range indices {
  23                 cis.Insert(index)
  24         }
  25         return
  26 }
  27
  28 func (cis CollectionIndexSet) ToSlice() (ints []int) {
  29         ints = make([]int, len(cis))
  30         i := 0
  31         for collectionIndex := range cis {
  32                 ints[i] = collectionIndex
  33                 i++
  34         }
  35         sort.Ints(ints)
  36         return
  37 }
  38
  39 // Takes a map from block digest to replication level and represents
  40 // it in a keep.ReadServers structure.
  41 func SpecifyReplication(digestToReplication map[int]int) (rs keep.ReadServers) {
  42         rs.BlockToServers = make(map[blockdigest.BlockDigest][]keep.BlockServerInfo)
  43         for digest, replication := range digestToReplication {
  44                 rs.BlockToServers[blockdigest.MakeTestBlockDigest(digest)] =
  45                         make([]keep.BlockServerInfo, replication)
  46         }
  47         return
  48 }
  49
  50 // Verifies that
  51 // blocks.ToCollectionIndexSet(rc.BlockToCollectionIndices) returns
  52 // expectedCollections.
  53 func VerifyToCollectionIndexSet(
  54         t *testing.T,
  55         blocks []int,
  56         blockToCollectionIndices map[int][]int,
  57         expectedCollections []int) {
  58
  59         expected := CollectionIndexSetFromSlice(expectedCollections)
  60
  61         rc := collection.ReadCollections{
  62                 BlockToCollectionIndices: map[blockdigest.BlockDigest][]int{},
  63         }
  64         for digest, indices := range blockToCollectionIndices {
  65                 rc.BlockToCollectionIndices[blockdigest.MakeTestBlockDigest(digest)] = indices
  66         }
  67
  68         returned := make(CollectionIndexSet)
  69         BlockSetFromSlice(blocks).ToCollectionIndexSet(rc, &returned)
  70
  71         if !reflect.DeepEqual(returned, expected) {
  72                 t.Errorf("Expected %v.ToCollectionIndexSet(%v) to return \n %v \n but instead received \n %v",
  73                         blocks,
  74                         blockToCollectionIndices,
  75                         expectedCollections,
  76                         returned.ToSlice())
  77         }
  78 }
  79
  80 func TestToCollectionIndexSet(t *testing.T) {
  81         VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{6: []int{0}}, []int{0})
  82         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1}}, []int{1})
  83         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1, 9}}, []int{1, 9})
  84         VerifyToCollectionIndexSet(t, []int{5, 6},
  85                 map[int][]int{5: []int{2, 3}, 6: []int{3, 4}},
  86                 []int{2, 3, 4})
  87         VerifyToCollectionIndexSet(t, []int{5, 6},
  88                 map[int][]int{5: []int{8}, 6: []int{4}},
  89                 []int{4, 8})
  90         VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{5: []int{0}}, []int{})
  91 }
  92
  93 func TestSimpleSummary(t *testing.T) {
  94         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
  95                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
  96         })
  97         rc.Summarize()
  98         cIndex := rc.CollectionIndicesForTesting()
  99
 100         keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1})
 101
 102         expectedSummary := ReplicationSummary{
 103                 CollectionBlocksNotInKeep:  BlockSet{},
 104                 UnderReplicatedBlocks:      BlockSet{},
 105                 OverReplicatedBlocks:       BlockSet{},
 106                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1, 2}),
 107                 KeepBlocksNotInCollections: BlockSet{},
 108
 109                 CollectionsNotFullyInKeep:      CollectionIndexSet{},
 110                 UnderReplicatedCollections:     CollectionIndexSet{},
 111                 OverReplicatedCollections:      CollectionIndexSet{},
 112                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}),
 113         }
 114
 115         returnedSummary := SummarizeReplication(rc, keepInfo)
 116
 117         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 118                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
 119         }
 120 }
 121
 122 func TestMissingBlock(t *testing.T) {
 123         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 124                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
 125         })
 126         rc.Summarize()
 127         cIndex := rc.CollectionIndicesForTesting()
 128
 129         keepInfo := SpecifyReplication(map[int]int{1: 1})
 130
 131         expectedSummary := ReplicationSummary{
 132                 CollectionBlocksNotInKeep:  BlockSetFromSlice([]int{2}),
 133                 UnderReplicatedBlocks:      BlockSet{},
 134                 OverReplicatedBlocks:       BlockSet{},
 135                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1}),
 136                 KeepBlocksNotInCollections: BlockSet{},
 137
 138                 CollectionsNotFullyInKeep:      CollectionIndexSetFromSlice([]int{cIndex[0]}),
 139                 UnderReplicatedCollections:     CollectionIndexSet{},
 140                 OverReplicatedCollections:      CollectionIndexSet{},
 141                 CorrectlyReplicatedCollections: CollectionIndexSet{},
 142         }
 143
 144         returnedSummary := SummarizeReplication(rc, keepInfo)
 145
 146         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 147                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v",
 148                         expectedSummary,
 149                         returnedSummary)
 150         }
 151 }
 152
 153 func TestUnderAndOverReplicatedBlocks(t *testing.T) {
 154         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 155                 collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{1, 2}},
 156         })
 157         rc.Summarize()
 158         cIndex := rc.CollectionIndicesForTesting()
 159
 160         keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 3})
 161
 162         expectedSummary := ReplicationSummary{
 163                 CollectionBlocksNotInKeep:  BlockSet{},
 164                 UnderReplicatedBlocks:      BlockSetFromSlice([]int{1}),
 165                 OverReplicatedBlocks:       BlockSetFromSlice([]int{2}),
 166                 CorrectlyReplicatedBlocks:  BlockSet{},
 167                 KeepBlocksNotInCollections: BlockSet{},
 168
 169                 CollectionsNotFullyInKeep:      CollectionIndexSet{},
 170                 UnderReplicatedCollections:     CollectionIndexSetFromSlice([]int{cIndex[0]}),
 171                 OverReplicatedCollections:      CollectionIndexSetFromSlice([]int{cIndex[0]}),
 172                 CorrectlyReplicatedCollections: CollectionIndexSet{},
 173         }
 174
 175         returnedSummary := SummarizeReplication(rc, keepInfo)
 176
 177         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 178                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v",
 179                         expectedSummary,
 180                         returnedSummary)
 181         }
 182 }
 183
 184 func TestMixedReplication(t *testing.T) {
 185         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 186                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
 187                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{3, 4}},
 188                 collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{5, 6}},
 189         })
 190         rc.Summarize()
 191         cIndex := rc.CollectionIndicesForTesting()
 192
 193         keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1, 3: 1, 5: 1, 6: 3, 7: 2})
 194
 195         expectedSummary := ReplicationSummary{
 196                 CollectionBlocksNotInKeep:  BlockSetFromSlice([]int{4}),
 197                 UnderReplicatedBlocks:      BlockSetFromSlice([]int{5}),
 198                 OverReplicatedBlocks:       BlockSetFromSlice([]int{6}),
 199                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1, 2, 3}),
 200                 KeepBlocksNotInCollections: BlockSetFromSlice([]int{7}),
 201
 202                 CollectionsNotFullyInKeep:      CollectionIndexSetFromSlice([]int{cIndex[1]}),
 203                 UnderReplicatedCollections:     CollectionIndexSetFromSlice([]int{cIndex[2]}),
 204                 OverReplicatedCollections:      CollectionIndexSetFromSlice([]int{cIndex[2]}),
 205                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}),
 206         }
 207
 208         returnedSummary := SummarizeReplication(rc, keepInfo)
 209
 210         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 211                 t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUuid, rc.BlockToCollectionIndices)
 212         }
 213 }