services/datamanager/summary/summary_test.go

   1 package summary
   2
   3 import (
   4         "git.curoverse.com/arvados.git/sdk/go/blockdigest"
   5         "git.curoverse.com/arvados.git/services/datamanager/collection"
   6         "git.curoverse.com/arvados.git/services/datamanager/keep"
   7         "reflect"
   8         "testing"
   9 )
  10
  11 func BlockSetFromSlice(digests []int) (bs BlockSet) {
  12         bs = make(BlockSet)
  13         for _, digest := range digests {
  14                 bs.Insert(blockdigest.MakeTestBlockDigest(digest))
  15         }
  16         return
  17 }
  18
  19 func CollectionIndexSetFromSlice(indices []int) (cis CollectionIndexSet) {
  20         cis = make(CollectionIndexSet)
  21         for _, index := range indices {
  22                 cis.Insert(index)
  23         }
  24         return
  25 }
  26
  27 func (cis CollectionIndexSet) ToSlice() (ints []int) {
  28         ints = make([]int, len(cis))
  29         i := 0
  30         for collectionIndex := range cis {
  31                 ints[i] = collectionIndex
  32                 i++
  33         }
  34         return
  35 }
  36
  37 func VerifyToCollectionIndexSet(
  38         t *testing.T,
  39         blocks []int,
  40         blockToCollectionIndices map[int][]int,
  41         expectedCollections []int) {
  42
  43         expected := CollectionIndexSetFromSlice(expectedCollections)
  44
  45         rc := collection.ReadCollections{
  46                 BlockToCollectionIndices: map[blockdigest.BlockDigest][]int{},
  47         }
  48         for digest, indices := range blockToCollectionIndices {
  49                 rc.BlockToCollectionIndices[blockdigest.MakeTestBlockDigest(digest)] = indices
  50         }
  51
  52         returned := make(CollectionIndexSet)
  53         BlockSetFromSlice(blocks).ToCollectionIndexSet(rc, &returned)
  54
  55         if !reflect.DeepEqual(returned, expected) {
  56                 t.Errorf("Expected %v.ToCollectionIndexSet(%v) to return \n %v \n but instead received \n %v",
  57                         blocks,
  58                         blockToCollectionIndices,
  59                         expectedCollections,
  60                         returned.ToSlice())
  61         }
  62 }
  63
  64 func TestToCollectionIndexSet(t *testing.T) {
  65         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1}}, []int{1})
  66         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1, 9}}, []int{1, 9})
  67         VerifyToCollectionIndexSet(t, []int{5, 6},
  68                 map[int][]int{5: []int{2, 3}, 6: []int{3, 4}},
  69                 []int{2, 3, 4})
  70 }
  71
  72 func TestSimpleSummary(t *testing.T) {
  73         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
  74                 collection.TestCollectionSpec{
  75                         ReplicationLevel: 1,
  76                         Blocks:           []int{1, 2},
  77                 },
  78         })
  79
  80         rc.Summarize()
  81
  82         // The internals aren't actually examined, so we can reuse the same one.
  83         dummyBlockServerInfo := keep.BlockServerInfo{}
  84
  85         blockDigest1 := blockdigest.MakeTestBlockDigest(1)
  86         blockDigest2 := blockdigest.MakeTestBlockDigest(2)
  87
  88         keepInfo := keep.ReadServers{
  89                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
  90                         blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
  91                         blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo},
  92                 },
  93         }
  94
  95         returnedSummary := SummarizeReplication(rc, keepInfo)
  96
  97         c := rc.UuidToCollection["col0"]
  98
  99         expectedSummary := ReplicationSummary{
 100                 CollectionBlocksNotInKeep:  BlockSet{},
 101                 UnderReplicatedBlocks:      BlockSet{},
 102                 OverReplicatedBlocks:       BlockSet{},
 103                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1, 2}),
 104                 KeepBlocksNotInCollections: BlockSet{},
 105
 106                 CollectionsNotFullyInKeep:  CollectionIndexSet{},
 107                 UnderReplicatedCollections: CollectionIndexSet{},
 108                 OverReplicatedCollections:  CollectionIndexSet{},
 109                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
 110                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 111         }
 112
 113         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 114                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
 115         }
 116 }
 117
 118 func TestMissingBlock(t *testing.T) {
 119         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 120                 collection.TestCollectionSpec{
 121                         ReplicationLevel: 1,
 122                         Blocks:           []int{1, 2},
 123                 },
 124         })
 125
 126         rc.Summarize()
 127
 128         // The internals aren't actually examined, so we can reuse the same one.
 129         dummyBlockServerInfo := keep.BlockServerInfo{}
 130
 131         blockDigest1 := blockdigest.MakeTestBlockDigest(1)
 132
 133         keepInfo := keep.ReadServers{
 134                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
 135                         blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
 136                 },
 137         }
 138
 139         returnedSummary := SummarizeReplication(rc, keepInfo)
 140
 141         c := rc.UuidToCollection["col0"]
 142
 143         expectedSummary := ReplicationSummary{
 144                 CollectionBlocksNotInKeep:  BlockSetFromSlice([]int{2}),
 145                 UnderReplicatedBlocks:      BlockSet{},
 146                 OverReplicatedBlocks:       BlockSet{},
 147                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1}),
 148                 KeepBlocksNotInCollections: BlockSet{},
 149
 150                 CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
 151                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 152                 UnderReplicatedCollections:     CollectionIndexSet{},
 153                 OverReplicatedCollections:      CollectionIndexSet{},
 154                 CorrectlyReplicatedCollections: CollectionIndexSet{},
 155         }
 156
 157         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 158                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
 159         }
 160 }
 161
 162 func TestUnderAndOverReplicatedBlocks(t *testing.T) {
 163         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 164                 collection.TestCollectionSpec{
 165                         ReplicationLevel: 2,
 166                         Blocks:           []int{1, 2},
 167                 },
 168         })
 169
 170         rc.Summarize()
 171
 172         // The internals aren't actually examined, so we can reuse the same one.
 173         dummyBlockServerInfo := keep.BlockServerInfo{}
 174
 175         blockDigest1 := blockdigest.MakeTestBlockDigest(1)
 176         blockDigest2 := blockdigest.MakeTestBlockDigest(2)
 177
 178         keepInfo := keep.ReadServers{
 179                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
 180                         blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
 181                         blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
 182                 },
 183         }
 184
 185         returnedSummary := SummarizeReplication(rc, keepInfo)
 186
 187         c := rc.UuidToCollection["col0"]
 188
 189         expectedSummary := ReplicationSummary{
 190                 CollectionBlocksNotInKeep:  BlockSet{},
 191                 UnderReplicatedBlocks:      BlockSetFromSlice([]int{1}),
 192                 OverReplicatedBlocks:       BlockSetFromSlice([]int{2}),
 193                 CorrectlyReplicatedBlocks:  BlockSet{},
 194                 KeepBlocksNotInCollections: BlockSet{},
 195
 196                 CollectionsNotFullyInKeep: CollectionIndexSet{},
 197                 UnderReplicatedCollections: CollectionIndexSetFromSlice(
 198                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 199                 OverReplicatedCollections: CollectionIndexSetFromSlice(
 200                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 201                 CorrectlyReplicatedCollections: CollectionIndexSet{},
 202         }
 203
 204         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 205                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
 206         }
 207 }
 208
 209 func TestMixedReplication(t *testing.T) {
 210         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 211                 collection.TestCollectionSpec{
 212                         ReplicationLevel: 1,
 213                         Blocks:           []int{1, 2},
 214                 },
 215                 collection.TestCollectionSpec{
 216                         ReplicationLevel: 1,
 217                         Blocks:           []int{3, 4},
 218                 },
 219                 collection.TestCollectionSpec{
 220                         ReplicationLevel: 2,
 221                         Blocks:           []int{5, 6},
 222                 },
 223         })
 224
 225         rc.Summarize()
 226
 227         // The internals aren't actually examined, so we can reuse the same one.
 228         dummyBlockServerInfo := keep.BlockServerInfo{}
 229
 230         keepInfo := keep.ReadServers{
 231                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
 232                         blockdigest.MakeTestBlockDigest(1): []keep.BlockServerInfo{dummyBlockServerInfo},
 233                         blockdigest.MakeTestBlockDigest(2): []keep.BlockServerInfo{dummyBlockServerInfo},
 234                         blockdigest.MakeTestBlockDigest(3): []keep.BlockServerInfo{dummyBlockServerInfo},
 235                         blockdigest.MakeTestBlockDigest(5): []keep.BlockServerInfo{dummyBlockServerInfo},
 236                         blockdigest.MakeTestBlockDigest(6): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
 237                         blockdigest.MakeTestBlockDigest(7): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo},
 238                 },
 239         }
 240
 241         returnedSummary := SummarizeReplication(rc, keepInfo)
 242
 243         c0 := rc.UuidToCollection["col0"]
 244         c1 := rc.UuidToCollection["col1"]
 245         c2 := rc.UuidToCollection["col2"]
 246
 247         expectedSummary := ReplicationSummary{
 248                 CollectionBlocksNotInKeep:  BlockSetFromSlice([]int{4}),
 249                 UnderReplicatedBlocks:      BlockSetFromSlice([]int{5}),
 250                 OverReplicatedBlocks:       BlockSetFromSlice([]int{6}),
 251                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1, 2, 3}),
 252                 KeepBlocksNotInCollections: BlockSetFromSlice([]int{7}),
 253
 254                 CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
 255                         []int{rc.CollectionUuidToIndex[c1.Uuid]}),
 256                 UnderReplicatedCollections: CollectionIndexSetFromSlice(
 257                         []int{rc.CollectionUuidToIndex[c2.Uuid]}),
 258                 OverReplicatedCollections: CollectionIndexSetFromSlice(
 259                         []int{rc.CollectionUuidToIndex[c2.Uuid]}),
 260                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
 261                         []int{rc.CollectionUuidToIndex[c0.Uuid]}),
 262         }
 263
 264         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 265                 t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUuid, rc.BlockToCollectionIndices)
 266         }
 267 }