services/datamanager/summary/summary_test.go

   1 package summary
   2
   3 import (
   4         "git.curoverse.com/arvados.git/sdk/go/blockdigest"
   5         "git.curoverse.com/arvados.git/services/datamanager/collection"
   6         "git.curoverse.com/arvados.git/services/datamanager/keep"
   7         "reflect"
   8         "sort"
   9         "testing"
  10 )
  11
  12 func BlockSetFromSlice(digests []int) (bs BlockSet) {
  13         bs = make(BlockSet)
  14         for _, digest := range digests {
  15                 bs.Insert(blockdigest.MakeTestBlockDigest(digest))
  16         }
  17         return
  18 }
  19
  20 func CollectionIndexSetFromSlice(indices []int) (cis CollectionIndexSet) {
  21         cis = make(CollectionIndexSet)
  22         for _, index := range indices {
  23                 cis.Insert(index)
  24         }
  25         return
  26 }
  27
  28 func (cis CollectionIndexSet) ToSlice() (ints []int) {
  29         ints = make([]int, len(cis))
  30         i := 0
  31         for collectionIndex := range cis {
  32                 ints[i] = collectionIndex
  33                 i++
  34         }
  35         sort.Ints(ints)
  36         return
  37 }
  38
  39 // Verifies that
  40 // blocks.ToCollectionIndexSet(rc.BlockToCollectionIndices) returns
  41 // expectedCollections.
  42 func VerifyToCollectionIndexSet(
  43         t *testing.T,
  44         blocks []int,
  45         blockToCollectionIndices map[int][]int,
  46         expectedCollections []int) {
  47
  48         expected := CollectionIndexSetFromSlice(expectedCollections)
  49
  50         rc := collection.ReadCollections{
  51                 BlockToCollectionIndices: map[blockdigest.BlockDigest][]int{},
  52         }
  53         for digest, indices := range blockToCollectionIndices {
  54                 rc.BlockToCollectionIndices[blockdigest.MakeTestBlockDigest(digest)] = indices
  55         }
  56
  57         returned := make(CollectionIndexSet)
  58         BlockSetFromSlice(blocks).ToCollectionIndexSet(rc, &returned)
  59
  60         if !reflect.DeepEqual(returned, expected) {
  61                 t.Errorf("Expected %v.ToCollectionIndexSet(%v) to return \n %v \n but instead received \n %v",
  62                         blocks,
  63                         blockToCollectionIndices,
  64                         expectedCollections,
  65                         returned.ToSlice())
  66         }
  67 }
  68
  69 func TestToCollectionIndexSet(t *testing.T) {
  70         VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{6: []int{0}}, []int{0})
  71         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1}}, []int{1})
  72         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1, 9}}, []int{1, 9})
  73         VerifyToCollectionIndexSet(t, []int{5, 6},
  74                 map[int][]int{5: []int{2, 3}, 6: []int{3, 4}},
  75                 []int{2, 3, 4})
  76         VerifyToCollectionIndexSet(t, []int{5, 6},
  77                 map[int][]int{5: []int{8}, 6: []int{4}},
  78                 []int{4, 8})
  79         VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{5: []int{0}}, []int{})
  80 }
  81
  82 func TestSimpleSummary(t *testing.T) {
  83         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
  84                 collection.TestCollectionSpec{
  85                         ReplicationLevel: 1,
  86                         Blocks:           []int{1, 2},
  87                 },
  88         })
  89
  90         rc.Summarize()
  91
  92         // The internals aren't actually examined, so we can reuse the same one.
  93         dummyBlockServerInfo := keep.BlockServerInfo{}
  94
  95         blockDigest1 := blockdigest.MakeTestBlockDigest(1)
  96         blockDigest2 := blockdigest.MakeTestBlockDigest(2)
  97
  98         keepInfo := keep.ReadServers{
  99                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
 100                         blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
 101                         blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo},
 102                 },
 103         }
 104
 105         returnedSummary := SummarizeReplication(rc, keepInfo)
 106
 107         c := rc.UuidToCollection["col0"]
 108
 109         expectedSummary := ReplicationSummary{
 110                 CollectionBlocksNotInKeep:  BlockSet{},
 111                 UnderReplicatedBlocks:      BlockSet{},
 112                 OverReplicatedBlocks:       BlockSet{},
 113                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1, 2}),
 114                 KeepBlocksNotInCollections: BlockSet{},
 115
 116                 CollectionsNotFullyInKeep:  CollectionIndexSet{},
 117                 UnderReplicatedCollections: CollectionIndexSet{},
 118                 OverReplicatedCollections:  CollectionIndexSet{},
 119                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
 120                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 121         }
 122
 123         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 124                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
 125         }
 126 }
 127
 128 func TestMissingBlock(t *testing.T) {
 129         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 130                 collection.TestCollectionSpec{
 131                         ReplicationLevel: 1,
 132                         Blocks:           []int{1, 2},
 133                 },
 134         })
 135
 136         rc.Summarize()
 137
 138         // The internals aren't actually examined, so we can reuse the same one.
 139         dummyBlockServerInfo := keep.BlockServerInfo{}
 140
 141         blockDigest1 := blockdigest.MakeTestBlockDigest(1)
 142
 143         keepInfo := keep.ReadServers{
 144                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
 145                         blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
 146                 },
 147         }
 148
 149         returnedSummary := SummarizeReplication(rc, keepInfo)
 150
 151         c := rc.UuidToCollection["col0"]
 152
 153         expectedSummary := ReplicationSummary{
 154                 CollectionBlocksNotInKeep:  BlockSetFromSlice([]int{2}),
 155                 UnderReplicatedBlocks:      BlockSet{},
 156                 OverReplicatedBlocks:       BlockSet{},
 157                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1}),
 158                 KeepBlocksNotInCollections: BlockSet{},
 159
 160                 CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
 161                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 162                 UnderReplicatedCollections:     CollectionIndexSet{},
 163                 OverReplicatedCollections:      CollectionIndexSet{},
 164                 CorrectlyReplicatedCollections: CollectionIndexSet{},
 165         }
 166
 167         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 168                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
 169         }
 170 }
 171
 172 func TestUnderAndOverReplicatedBlocks(t *testing.T) {
 173         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 174                 collection.TestCollectionSpec{
 175                         ReplicationLevel: 2,
 176                         Blocks:           []int{1, 2},
 177                 },
 178         })
 179
 180         rc.Summarize()
 181
 182         // The internals aren't actually examined, so we can reuse the same one.
 183         dummyBlockServerInfo := keep.BlockServerInfo{}
 184
 185         blockDigest1 := blockdigest.MakeTestBlockDigest(1)
 186         blockDigest2 := blockdigest.MakeTestBlockDigest(2)
 187
 188         keepInfo := keep.ReadServers{
 189                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
 190                         blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
 191                         blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
 192                 },
 193         }
 194
 195         returnedSummary := SummarizeReplication(rc, keepInfo)
 196
 197         c := rc.UuidToCollection["col0"]
 198
 199         expectedSummary := ReplicationSummary{
 200                 CollectionBlocksNotInKeep:  BlockSet{},
 201                 UnderReplicatedBlocks:      BlockSetFromSlice([]int{1}),
 202                 OverReplicatedBlocks:       BlockSetFromSlice([]int{2}),
 203                 CorrectlyReplicatedBlocks:  BlockSet{},
 204                 KeepBlocksNotInCollections: BlockSet{},
 205
 206                 CollectionsNotFullyInKeep: CollectionIndexSet{},
 207                 UnderReplicatedCollections: CollectionIndexSetFromSlice(
 208                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 209                 OverReplicatedCollections: CollectionIndexSetFromSlice(
 210                         []int{rc.CollectionUuidToIndex[c.Uuid]}),
 211                 CorrectlyReplicatedCollections: CollectionIndexSet{},
 212         }
 213
 214         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 215                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
 216         }
 217 }
 218
 219 func TestMixedReplication(t *testing.T) {
 220         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
 221                 collection.TestCollectionSpec{
 222                         ReplicationLevel: 1,
 223                         Blocks:           []int{1, 2},
 224                 },
 225                 collection.TestCollectionSpec{
 226                         ReplicationLevel: 1,
 227                         Blocks:           []int{3, 4},
 228                 },
 229                 collection.TestCollectionSpec{
 230                         ReplicationLevel: 2,
 231                         Blocks:           []int{5, 6},
 232                 },
 233         })
 234
 235         rc.Summarize()
 236
 237         // The internals aren't actually examined, so we can reuse the same one.
 238         dummyBlockServerInfo := keep.BlockServerInfo{}
 239
 240         keepInfo := keep.ReadServers{
 241                 BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
 242                         blockdigest.MakeTestBlockDigest(1): []keep.BlockServerInfo{dummyBlockServerInfo},
 243                         blockdigest.MakeTestBlockDigest(2): []keep.BlockServerInfo{dummyBlockServerInfo},
 244                         blockdigest.MakeTestBlockDigest(3): []keep.BlockServerInfo{dummyBlockServerInfo},
 245                         blockdigest.MakeTestBlockDigest(5): []keep.BlockServerInfo{dummyBlockServerInfo},
 246                         blockdigest.MakeTestBlockDigest(6): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
 247                         blockdigest.MakeTestBlockDigest(7): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo},
 248                 },
 249         }
 250
 251         returnedSummary := SummarizeReplication(rc, keepInfo)
 252
 253         c0 := rc.UuidToCollection["col0"]
 254         c1 := rc.UuidToCollection["col1"]
 255         c2 := rc.UuidToCollection["col2"]
 256
 257         expectedSummary := ReplicationSummary{
 258                 CollectionBlocksNotInKeep:  BlockSetFromSlice([]int{4}),
 259                 UnderReplicatedBlocks:      BlockSetFromSlice([]int{5}),
 260                 OverReplicatedBlocks:       BlockSetFromSlice([]int{6}),
 261                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1, 2, 3}),
 262                 KeepBlocksNotInCollections: BlockSetFromSlice([]int{7}),
 263
 264                 CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
 265                         []int{rc.CollectionUuidToIndex[c1.Uuid]}),
 266                 UnderReplicatedCollections: CollectionIndexSetFromSlice(
 267                         []int{rc.CollectionUuidToIndex[c2.Uuid]}),
 268                 OverReplicatedCollections: CollectionIndexSetFromSlice(
 269                         []int{rc.CollectionUuidToIndex[c2.Uuid]}),
 270                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
 271                         []int{rc.CollectionUuidToIndex[c0.Uuid]}),
 272         }
 273
 274         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
 275                 t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUuid, rc.BlockToCollectionIndices)
 276         }
 277 }