services/datamanager/summary/summary_test.go

   1 package summary
   2
   3 import (
   4         "git.curoverse.com/arvados.git/sdk/go/blockdigest"
   5         "git.curoverse.com/arvados.git/services/datamanager/collection"
   6         "git.curoverse.com/arvados.git/services/datamanager/keep"
   7         "reflect"
   8         "testing"
   9 )
  10
  11 func BlockSetFromSlice(digests []int) (bs BlockSet) {
  12         bs = make(BlockSet)
  13         for _, digest := range digests {
  14                 bs.Insert(blockdigest.MakeTestBlockDigest(digest))
  15         }
  16         return
  17 }
  18
  19 func CollectionIndexSetFromSlice(indices []int) (cis CollectionIndexSet) {
  20         cis = make(CollectionIndexSet)
  21         for _, index := range indices {
  22                 cis.Insert(index)
  23         }
  24         return
  25 }
  26
  27 func (cis CollectionIndexSet) ToSlice() (ints []int) {
  28         ints = make([]int, len(cis))
  29         i := 0
  30         for collectionIndex := range cis {
  31                 ints[i] = collectionIndex
  32                 i++
  33         }
  34         return
  35 }
  36
  37 func VerifyToCollectionIndexSet(
  38         t *testing.T,
  39         blocks []int,
  40         blockToCollectionIndices map[int][]int,
  41         expectedCollections []int) {
  42
  43         expected := CollectionIndexSetFromSlice(expectedCollections)
  44
  45         rc := collection.ReadCollections{
  46                 BlockToCollectionIndices: map[blockdigest.BlockDigest][]int{},
  47         }
  48         for digest, indices := range blockToCollectionIndices {
  49                 rc.BlockToCollectionIndices[blockdigest.MakeTestBlockDigest(digest)] = indices
  50         }
  51
  52         returned := make(CollectionIndexSet)
  53         BlockSetFromSlice(blocks).ToCollectionIndexSet(rc, &returned)
  54
  55         if !reflect.DeepEqual(returned, expected) {
  56                 t.Errorf("Expected %v.ToCollectionIndexSet(%v) to return \n %v \n but instead received \n %v",
  57                         blocks,
  58                         blockToCollectionIndices,
  59                         expectedCollections,
  60                         returned.ToSlice())
  61         }
  62 }
  63
  64 func TestToCollectionIndexSet(t *testing.T) {
  65         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4:[]int{1}}, []int{1})
  66         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4:[]int{1,9}}, []int{1,9})
  67         VerifyToCollectionIndexSet(t, []int{5,6},
  68                 map[int][]int{5:[]int{2,3}, 6:[]int{3,4}},
  69                 []int{2,3,4})
  70 }
  71
  72
  73 func TestSimpleSummary(t *testing.T) {
  74         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
  75                 collection.TestCollectionSpec{
  76                         ReplicationLevel: 1,
  77                         Blocks:           []int{1, 2},
  78                 },
  79         })
  80
  81         rc.Summarize()
  82
  83         // The internals aren't actually examined, so we can reuse the same one.
  84         dummyBlockServerInfo := keep.BlockServerInfo{}
  85
  86         blockDigest1 := blockdigest.MakeTestBlockDigest(1)
  87         blockDigest2 := blockdigest.MakeTestBlockDigest(2)
  88
  89         keepInfo := keep.ReadServers{
  90                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
  91                         blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
  92                         blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo},
  93                 },
  94         }
  95
  96         returnedSummary := SummarizeReplication(rc, keepInfo)
  97
  98         c := rc.UuidToCollection["col0"]
  99
 100         expectedSummary := ReplicationSummary{
 101                 CollectionBlocksNotInKeep: BlockSet{},
 102                 UnderReplicatedBlocks:     BlockSet{},
 103                 OverReplicatedBlocks:      BlockSet{},
 104                 CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1,2}),
 105                 KeepBlocksNotInCollections: BlockSet{},
 106
 107                 CollectionsNotFullyInKeep:  CollectionIndexSet{},
 108                 UnderReplicatedCollections: CollectionIndexSet{},
 109                 OverReplicatedCollections:  CollectionIndexSet{},
 110                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
 111                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 112         }
 113
 114         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 115                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
 116         }
 117 }
 118
 119 func TestMissingBlock(t *testing.T) {
 120         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 121                 collection.TestCollectionSpec{
 122                         ReplicationLevel: 1,
 123                         Blocks:           []int{1, 2},
 124                 },
 125         })
 126
 127         rc.Summarize()
 128
 129         // The internals aren't actually examined, so we can reuse the same one.
 130         dummyBlockServerInfo := keep.BlockServerInfo{}
 131
 132         blockDigest1 := blockdigest.MakeTestBlockDigest(1)
 133
 134         keepInfo := keep.ReadServers{
 135                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
 136                         blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
 137                 },
 138         }
 139
 140         returnedSummary := SummarizeReplication(rc, keepInfo)
 141
 142         c := rc.UuidToCollection["col0"]
 143
 144         expectedSummary := ReplicationSummary{
 145                 CollectionBlocksNotInKeep: BlockSetFromSlice([]int{2}),
 146                 UnderReplicatedBlocks: BlockSet{},
 147                 OverReplicatedBlocks:  BlockSet{},
 148                 CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1}),
 149                 KeepBlocksNotInCollections: BlockSet{},
 150
 151                 CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
 152                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 153                 UnderReplicatedCollections:     CollectionIndexSet{},
 154                 OverReplicatedCollections:      CollectionIndexSet{},
 155                 CorrectlyReplicatedCollections: CollectionIndexSet{},
 156         }
 157
 158         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 159                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
 160         }
 161 }
 162
 163 func TestUnderAndOverReplicatedBlocks(t *testing.T) {
 164         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 165                 collection.TestCollectionSpec{
 166                         ReplicationLevel: 2,
 167                         Blocks:           []int{1, 2},
 168                 },
 169         })
 170
 171         rc.Summarize()
 172
 173         // The internals aren't actually examined, so we can reuse the same one.
 174         dummyBlockServerInfo := keep.BlockServerInfo{}
 175
 176         blockDigest1 := blockdigest.MakeTestBlockDigest(1)
 177         blockDigest2 := blockdigest.MakeTestBlockDigest(2)
 178
 179         keepInfo := keep.ReadServers{
 180                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
 181                         blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
 182                         blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
 183                 },
 184         }
 185
 186         returnedSummary := SummarizeReplication(rc, keepInfo)
 187
 188         c := rc.UuidToCollection["col0"]
 189
 190         expectedSummary := ReplicationSummary{
 191                 CollectionBlocksNotInKeep: BlockSet{},
 192                 UnderReplicatedBlocks: BlockSetFromSlice([]int{1}),
 193                 OverReplicatedBlocks: BlockSetFromSlice([]int{2}),
 194                 CorrectlyReplicatedBlocks:  BlockSet{},
 195                 KeepBlocksNotInCollections: BlockSet{},
 196
 197                 CollectionsNotFullyInKeep: CollectionIndexSet{},
 198                 UnderReplicatedCollections: CollectionIndexSetFromSlice(
 199                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 200                 OverReplicatedCollections: CollectionIndexSetFromSlice(
 201                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 202                 CorrectlyReplicatedCollections: CollectionIndexSet{},
 203         }
 204
 205         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 206                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
 207         }
 208 }
 209
 210 func TestMixedReplication(t *testing.T) {
 211         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 212                 collection.TestCollectionSpec{
 213                         ReplicationLevel: 1,
 214                         Blocks:           []int{1, 2},
 215                 },
 216                 collection.TestCollectionSpec{
 217                         ReplicationLevel: 1,
 218                         Blocks:           []int{3, 4},
 219                 },
 220                 collection.TestCollectionSpec{
 221                         ReplicationLevel: 2,
 222                         Blocks:           []int{5, 6},
 223                 },
 224         })
 225
 226         rc.Summarize()
 227
 228         // The internals aren't actually examined, so we can reuse the same one.
 229         dummyBlockServerInfo := keep.BlockServerInfo{}
 230
 231         keepInfo := keep.ReadServers{
 232                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
 233                         blockdigest.MakeTestBlockDigest(1): []keep.BlockServerInfo{dummyBlockServerInfo},
 234                         blockdigest.MakeTestBlockDigest(2): []keep.BlockServerInfo{dummyBlockServerInfo},
 235                         blockdigest.MakeTestBlockDigest(3): []keep.BlockServerInfo{dummyBlockServerInfo},
 236                         blockdigest.MakeTestBlockDigest(5): []keep.BlockServerInfo{dummyBlockServerInfo},
 237                         blockdigest.MakeTestBlockDigest(6): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
 238                         blockdigest.MakeTestBlockDigest(7): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo},
 239                 },
 240         }
 241
 242         returnedSummary := SummarizeReplication(rc, keepInfo)
 243
 244         c0 := rc.UuidToCollection["col0"]
 245         c1 := rc.UuidToCollection["col1"]
 246         c2 := rc.UuidToCollection["col2"]
 247
 248         expectedSummary := ReplicationSummary{
 249                 CollectionBlocksNotInKeep: BlockSetFromSlice([]int{4}),
 250                 UnderReplicatedBlocks: BlockSetFromSlice([]int{5}),
 251                 OverReplicatedBlocks: BlockSetFromSlice([]int{6}),
 252                 CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1,2,3}),
 253                 KeepBlocksNotInCollections: BlockSetFromSlice([]int{7}),
 254
 255                 CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
 256                         []int{rc.CollectionUuidToIndex[c1.Uuid]}),
 257                 UnderReplicatedCollections: CollectionIndexSetFromSlice(
 258                         []int{rc.CollectionUuidToIndex[c2.Uuid]}),
 259                 OverReplicatedCollections: CollectionIndexSetFromSlice(
 260                         []int{rc.CollectionUuidToIndex[c2.Uuid]}),
 261                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
 262                         []int{rc.CollectionUuidToIndex[c0.Uuid]}),
 263         }
 264
 265         tempCis := make(CollectionIndexSet)
 266         returnedSummary.CollectionBlocksNotInKeep.ToCollectionIndexSet(rc, &tempCis)
 267         t.Logf("blocks not in keep: %v, collections not fully in keep: %v",
 268                 returnedSummary.CollectionBlocksNotInKeep,
 269                 tempCis)
 270
 271         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 272                 t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUuid, rc.BlockToCollectionIndices)
 273         }
 274 }