services/datamanager/summary/summary_test.go

   1 package summary
   2
   3 import (
   4         "git.curoverse.com/arvados.git/sdk/go/blockdigest"
   5         "git.curoverse.com/arvados.git/services/datamanager/collection"
   6         "git.curoverse.com/arvados.git/services/datamanager/keep"
   7         "reflect"
   8         "sort"
   9         "testing"
  10 )
  11
  12 func BlockSetFromSlice(digests []int) (bs BlockSet) {
  13         bs = make(BlockSet)
  14         for _, digest := range digests {
  15                 bs.Insert(blockdigest.MakeTestDigestWithSize(digest))
  16         }
  17         return
  18 }
  19
  20 func CollectionIndexSetFromSlice(indices []int) (cis CollectionIndexSet) {
  21         cis = make(CollectionIndexSet)
  22         for _, index := range indices {
  23                 cis.Insert(index)
  24         }
  25         return
  26 }
  27
  28 func (cis CollectionIndexSet) ToSlice() (ints []int) {
  29         ints = make([]int, len(cis))
  30         i := 0
  31         for collectionIndex := range cis {
  32                 ints[i] = collectionIndex
  33                 i++
  34         }
  35         sort.Ints(ints)
  36         return
  37 }
  38
  39 // Helper method to meet interface expected by older tests.
  40 func SummarizeReplication(readCollections collection.ReadCollections,
  41         keepServerInfo keep.ReadServers) (rs ReplicationSummary) {
  42         return BucketReplication(readCollections, keepServerInfo).
  43                 SummarizeBuckets(readCollections)
  44 }
  45
  46 // Takes a map from block digest to replication level and represents
  47 // it in a keep.ReadServers structure.
  48 func SpecifyReplication(digestToReplication map[int]int) (rs keep.ReadServers) {
  49         rs.BlockToServers = make(map[blockdigest.DigestWithSize][]keep.BlockServerInfo)
  50         for digest, replication := range digestToReplication {
  51                 rs.BlockToServers[blockdigest.MakeTestDigestWithSize(digest)] =
  52                         make([]keep.BlockServerInfo, replication)
  53         }
  54         return
  55 }
  56
  57 // Verifies that
  58 // blocks.ToCollectionIndexSet(rc.BlockToCollectionIndices) returns
  59 // expectedCollections.
  60 func VerifyToCollectionIndexSet(
  61         t *testing.T,
  62         blocks []int,
  63         blockToCollectionIndices map[int][]int,
  64         expectedCollections []int) {
  65
  66         expected := CollectionIndexSetFromSlice(expectedCollections)
  67
  68         rc := collection.ReadCollections{
  69                 BlockToCollectionIndices: map[blockdigest.DigestWithSize][]int{},
  70         }
  71         for digest, indices := range blockToCollectionIndices {
  72                 rc.BlockToCollectionIndices[blockdigest.MakeTestDigestWithSize(digest)] = indices
  73         }
  74
  75         returned := make(CollectionIndexSet)
  76         BlockSetFromSlice(blocks).ToCollectionIndexSet(rc, &returned)
  77
  78         if !reflect.DeepEqual(returned, expected) {
  79                 t.Errorf("Expected %v.ToCollectionIndexSet(%v) to return \n %v \n but instead received \n %v",
  80                         blocks,
  81                         blockToCollectionIndices,
  82                         expectedCollections,
  83                         returned.ToSlice())
  84         }
  85 }
  86
  87 func TestToCollectionIndexSet(t *testing.T) {
  88         VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{6: []int{0}}, []int{0})
  89         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1}}, []int{1})
  90         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1, 9}}, []int{1, 9})
  91         VerifyToCollectionIndexSet(t, []int{5, 6},
  92                 map[int][]int{5: []int{2, 3}, 6: []int{3, 4}},
  93                 []int{2, 3, 4})
  94         VerifyToCollectionIndexSet(t, []int{5, 6},
  95                 map[int][]int{5: []int{8}, 6: []int{4}},
  96                 []int{4, 8})
  97         VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{5: []int{0}}, []int{})
  98 }
  99
 100 func TestSimpleSummary(t *testing.T) {
 101         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 102                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
 103         })
 104         rc.Summarize(nil)
 105         cIndex := rc.CollectionIndicesForTesting()
 106
 107         keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1})
 108
 109         expectedSummary := ReplicationSummary{
 110                 CollectionBlocksNotInKeep:  BlockSet{},
 111                 UnderReplicatedBlocks:      BlockSet{},
 112                 OverReplicatedBlocks:       BlockSet{},
 113                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1, 2}),
 114                 KeepBlocksNotInCollections: BlockSet{},
 115
 116                 CollectionsNotFullyInKeep:      CollectionIndexSet{},
 117                 UnderReplicatedCollections:     CollectionIndexSet{},
 118                 OverReplicatedCollections:      CollectionIndexSet{},
 119                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}),
 120         }
 121
 122         returnedSummary := SummarizeReplication(rc, keepInfo)
 123
 124         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 125                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
 126         }
 127 }
 128
 129 func TestMissingBlock(t *testing.T) {
 130         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 131                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
 132         })
 133         rc.Summarize(nil)
 134         cIndex := rc.CollectionIndicesForTesting()
 135
 136         keepInfo := SpecifyReplication(map[int]int{1: 1})
 137
 138         expectedSummary := ReplicationSummary{
 139                 CollectionBlocksNotInKeep:  BlockSetFromSlice([]int{2}),
 140                 UnderReplicatedBlocks:      BlockSet{},
 141                 OverReplicatedBlocks:       BlockSet{},
 142                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1}),
 143                 KeepBlocksNotInCollections: BlockSet{},
 144
 145                 CollectionsNotFullyInKeep:      CollectionIndexSetFromSlice([]int{cIndex[0]}),
 146                 UnderReplicatedCollections:     CollectionIndexSet{},
 147                 OverReplicatedCollections:      CollectionIndexSet{},
 148                 CorrectlyReplicatedCollections: CollectionIndexSet{},
 149         }
 150
 151         returnedSummary := SummarizeReplication(rc, keepInfo)
 152
 153         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 154                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v",
 155                         expectedSummary,
 156                         returnedSummary)
 157         }
 158 }
 159
 160 func TestUnderAndOverReplicatedBlocks(t *testing.T) {
 161         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 162                 collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{1, 2}},
 163         })
 164         rc.Summarize(nil)
 165         cIndex := rc.CollectionIndicesForTesting()
 166
 167         keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 3})
 168
 169         expectedSummary := ReplicationSummary{
 170                 CollectionBlocksNotInKeep:  BlockSet{},
 171                 UnderReplicatedBlocks:      BlockSetFromSlice([]int{1}),
 172                 OverReplicatedBlocks:       BlockSetFromSlice([]int{2}),
 173                 CorrectlyReplicatedBlocks:  BlockSet{},
 174                 KeepBlocksNotInCollections: BlockSet{},
 175
 176                 CollectionsNotFullyInKeep:      CollectionIndexSet{},
 177                 UnderReplicatedCollections:     CollectionIndexSetFromSlice([]int{cIndex[0]}),
 178                 OverReplicatedCollections:      CollectionIndexSetFromSlice([]int{cIndex[0]}),
 179                 CorrectlyReplicatedCollections: CollectionIndexSet{},
 180         }
 181
 182         returnedSummary := SummarizeReplication(rc, keepInfo)
 183
 184         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 185                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v",
 186                         expectedSummary,
 187                         returnedSummary)
 188         }
 189 }
 190
 191 func TestMixedReplication(t *testing.T) {
 192         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 193                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
 194                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{3, 4}},
 195                 collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{5, 6}},
 196         })
 197         rc.Summarize(nil)
 198         cIndex := rc.CollectionIndicesForTesting()
 199
 200         keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1, 3: 1, 5: 1, 6: 3, 7: 2})
 201
 202         expectedSummary := ReplicationSummary{
 203                 CollectionBlocksNotInKeep:  BlockSetFromSlice([]int{4}),
 204                 UnderReplicatedBlocks:      BlockSetFromSlice([]int{5}),
 205                 OverReplicatedBlocks:       BlockSetFromSlice([]int{6}),
 206                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1, 2, 3}),
 207                 KeepBlocksNotInCollections: BlockSetFromSlice([]int{7}),
 208
 209                 CollectionsNotFullyInKeep:      CollectionIndexSetFromSlice([]int{cIndex[1]}),
 210                 UnderReplicatedCollections:     CollectionIndexSetFromSlice([]int{cIndex[2]}),
 211                 OverReplicatedCollections:      CollectionIndexSetFromSlice([]int{cIndex[2]}),
 212                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}),
 213         }
 214
 215         returnedSummary := SummarizeReplication(rc, keepInfo)
 216
 217         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 218                 t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUUID, rc.BlockToCollectionIndices)
 219         }
 220 }