6260: so many golint complaints!!
[arvados.git] / services / datamanager / summary / summary_test.go
1 package summary
2
3 import (
4         "git.curoverse.com/arvados.git/sdk/go/blockdigest"
5         "git.curoverse.com/arvados.git/services/datamanager/collection"
6         "git.curoverse.com/arvados.git/services/datamanager/keep"
7         "reflect"
8         "sort"
9         "testing"
10 )
11
12 func BlockSetFromSlice(digests []int) (bs BlockSet) {
13         bs = make(BlockSet)
14         for _, digest := range digests {
15                 bs.Insert(blockdigest.MakeTestDigestWithSize(digest))
16         }
17         return
18 }
19
20 func CollectionIndexSetFromSlice(indices []int) (cis CollectionIndexSet) {
21         cis = make(CollectionIndexSet)
22         for _, index := range indices {
23                 cis.Insert(index)
24         }
25         return
26 }
27
28 func (cis CollectionIndexSet) ToSlice() (ints []int) {
29         ints = make([]int, len(cis))
30         i := 0
31         for collectionIndex := range cis {
32                 ints[i] = collectionIndex
33                 i++
34         }
35         sort.Ints(ints)
36         return
37 }
38
39 // Helper method to meet interface expected by older tests.
40 func SummarizeReplication(readCollections collection.ReadCollections,
41         keepServerInfo keep.ReadServers) (rs ReplicationSummary) {
42         return BucketReplication(readCollections, keepServerInfo).
43                 SummarizeBuckets(readCollections)
44 }
45
46 // Takes a map from block digest to replication level and represents
47 // it in a keep.ReadServers structure.
48 func SpecifyReplication(digestToReplication map[int]int) (rs keep.ReadServers) {
49         rs.BlockToServers = make(map[blockdigest.DigestWithSize][]keep.BlockServerInfo)
50         for digest, replication := range digestToReplication {
51                 rs.BlockToServers[blockdigest.MakeTestDigestWithSize(digest)] =
52                         make([]keep.BlockServerInfo, replication)
53         }
54         return
55 }
56
57 // Verifies that
58 // blocks.ToCollectionIndexSet(rc.BlockToCollectionIndices) returns
59 // expectedCollections.
60 func VerifyToCollectionIndexSet(
61         t *testing.T,
62         blocks []int,
63         blockToCollectionIndices map[int][]int,
64         expectedCollections []int) {
65
66         expected := CollectionIndexSetFromSlice(expectedCollections)
67
68         rc := collection.ReadCollections{
69                 BlockToCollectionIndices: map[blockdigest.DigestWithSize][]int{},
70         }
71         for digest, indices := range blockToCollectionIndices {
72                 rc.BlockToCollectionIndices[blockdigest.MakeTestDigestWithSize(digest)] = indices
73         }
74
75         returned := make(CollectionIndexSet)
76         BlockSetFromSlice(blocks).ToCollectionIndexSet(rc, &returned)
77
78         if !reflect.DeepEqual(returned, expected) {
79                 t.Errorf("Expected %v.ToCollectionIndexSet(%v) to return \n %v \n but instead received \n %v",
80                         blocks,
81                         blockToCollectionIndices,
82                         expectedCollections,
83                         returned.ToSlice())
84         }
85 }
86
87 func TestToCollectionIndexSet(t *testing.T) {
88         VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{6: []int{0}}, []int{0})
89         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1}}, []int{1})
90         VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1, 9}}, []int{1, 9})
91         VerifyToCollectionIndexSet(t, []int{5, 6},
92                 map[int][]int{5: []int{2, 3}, 6: []int{3, 4}},
93                 []int{2, 3, 4})
94         VerifyToCollectionIndexSet(t, []int{5, 6},
95                 map[int][]int{5: []int{8}, 6: []int{4}},
96                 []int{4, 8})
97         VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{5: []int{0}}, []int{})
98 }
99
100 func TestSimpleSummary(t *testing.T) {
101         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
102                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
103         })
104         rc.Summarize(nil)
105         cIndex := rc.CollectionIndicesForTesting()
106
107         keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1})
108
109         expectedSummary := ReplicationSummary{
110                 CollectionBlocksNotInKeep:  BlockSet{},
111                 UnderReplicatedBlocks:      BlockSet{},
112                 OverReplicatedBlocks:       BlockSet{},
113                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1, 2}),
114                 KeepBlocksNotInCollections: BlockSet{},
115
116                 CollectionsNotFullyInKeep:      CollectionIndexSet{},
117                 UnderReplicatedCollections:     CollectionIndexSet{},
118                 OverReplicatedCollections:      CollectionIndexSet{},
119                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}),
120         }
121
122         returnedSummary := SummarizeReplication(rc, keepInfo)
123
124         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
125                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
126         }
127 }
128
129 func TestMissingBlock(t *testing.T) {
130         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
131                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
132         })
133         rc.Summarize(nil)
134         cIndex := rc.CollectionIndicesForTesting()
135
136         keepInfo := SpecifyReplication(map[int]int{1: 1})
137
138         expectedSummary := ReplicationSummary{
139                 CollectionBlocksNotInKeep:  BlockSetFromSlice([]int{2}),
140                 UnderReplicatedBlocks:      BlockSet{},
141                 OverReplicatedBlocks:       BlockSet{},
142                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1}),
143                 KeepBlocksNotInCollections: BlockSet{},
144
145                 CollectionsNotFullyInKeep:      CollectionIndexSetFromSlice([]int{cIndex[0]}),
146                 UnderReplicatedCollections:     CollectionIndexSet{},
147                 OverReplicatedCollections:      CollectionIndexSet{},
148                 CorrectlyReplicatedCollections: CollectionIndexSet{},
149         }
150
151         returnedSummary := SummarizeReplication(rc, keepInfo)
152
153         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
154                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v",
155                         expectedSummary,
156                         returnedSummary)
157         }
158 }
159
160 func TestUnderAndOverReplicatedBlocks(t *testing.T) {
161         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
162                 collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{1, 2}},
163         })
164         rc.Summarize(nil)
165         cIndex := rc.CollectionIndicesForTesting()
166
167         keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 3})
168
169         expectedSummary := ReplicationSummary{
170                 CollectionBlocksNotInKeep:  BlockSet{},
171                 UnderReplicatedBlocks:      BlockSetFromSlice([]int{1}),
172                 OverReplicatedBlocks:       BlockSetFromSlice([]int{2}),
173                 CorrectlyReplicatedBlocks:  BlockSet{},
174                 KeepBlocksNotInCollections: BlockSet{},
175
176                 CollectionsNotFullyInKeep:      CollectionIndexSet{},
177                 UnderReplicatedCollections:     CollectionIndexSetFromSlice([]int{cIndex[0]}),
178                 OverReplicatedCollections:      CollectionIndexSetFromSlice([]int{cIndex[0]}),
179                 CorrectlyReplicatedCollections: CollectionIndexSet{},
180         }
181
182         returnedSummary := SummarizeReplication(rc, keepInfo)
183
184         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
185                 t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v",
186                         expectedSummary,
187                         returnedSummary)
188         }
189 }
190
191 func TestMixedReplication(t *testing.T) {
192         rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
193                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
194                 collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{3, 4}},
195                 collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{5, 6}},
196         })
197         rc.Summarize(nil)
198         cIndex := rc.CollectionIndicesForTesting()
199
200         keepInfo := SpecifyReplication(map[int]int{1: 1, 2: 1, 3: 1, 5: 1, 6: 3, 7: 2})
201
202         expectedSummary := ReplicationSummary{
203                 CollectionBlocksNotInKeep:  BlockSetFromSlice([]int{4}),
204                 UnderReplicatedBlocks:      BlockSetFromSlice([]int{5}),
205                 OverReplicatedBlocks:       BlockSetFromSlice([]int{6}),
206                 CorrectlyReplicatedBlocks:  BlockSetFromSlice([]int{1, 2, 3}),
207                 KeepBlocksNotInCollections: BlockSetFromSlice([]int{7}),
208
209                 CollectionsNotFullyInKeep:      CollectionIndexSetFromSlice([]int{cIndex[1]}),
210                 UnderReplicatedCollections:     CollectionIndexSetFromSlice([]int{cIndex[2]}),
211                 OverReplicatedCollections:      CollectionIndexSetFromSlice([]int{cIndex[2]}),
212                 CorrectlyReplicatedCollections: CollectionIndexSetFromSlice([]int{cIndex[0]}),
213         }
214
215         returnedSummary := SummarizeReplication(rc, keepInfo)
216
217         if !reflect.DeepEqual(returnedSummary, expectedSummary) {
218                 t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUUID, rc.BlockToCollectionIndices)
219         }
220 }