Fixed bug in BlockSet.ToCollectionIndexSet.

Lots of unittests in summary_test.go.

diff --git a/services/datamanager/summary/file.go b/services/datamanager/summary/file.go
index 3a853bca3e6d30001679f5c8da667973ad6fc205..15c369900ca57a42e837b1884e34372e4fa2a732 100644 (file)
--- a/services/datamanager/summary/file.go
+++ b/services/datamanager/summary/file.go
@@ -14,6 +14,13 @@ import (
        "os"
 )
 
+// Used to locally cache data read from servers to reduce execution
+// time when developing. Not for use in production.
+type serializedData struct {
+       ReadCollections collection.ReadCollections
+       KeepServerInfo  keep.ReadServers
+}
+
 var (
        writeDataTo  string
        readDataFrom string

diff --git a/services/datamanager/summary/summary.go b/services/datamanager/summary/summary.go
index 71d1affec0d1343c19a1934cd6365c7b3d59d67c..37e9373969efb826ba5c9f1efb895937f892df3f 100644 (file)
--- a/services/datamanager/summary/summary.go
+++ b/services/datamanager/summary/summary.go
@@ -16,37 +16,20 @@ func (bs BlockSet) Insert(digest blockdigest.BlockDigest) {
        bs[digest] = struct{}{}
 }
 
-func BlockSetFromSlice(digests []blockdigest.BlockDigest) (bs BlockSet) {
-       bs = make(BlockSet)
-       for _, digest := range digests {
-               bs.Insert(digest)
-       }
-       return
-}
-
-// We use the collection index to save space. To convert to & from the
-// uuid, use collection.ReadCollections' fields CollectionIndexToUuid
-// and CollectionUuidToIndex.
+// We use the collection index to save space. To convert to and from
+// the uuid, use collection.ReadCollections' fields
+// CollectionIndexToUuid and CollectionUuidToIndex.
 type CollectionIndexSet map[int]struct{}
 
 func (cis CollectionIndexSet) Insert(collectionIndex int) {
        cis[collectionIndex] = struct{}{}
 }
 
-func CollectionIndexSetFromSlice(indices []int) (cis CollectionIndexSet) {
-       cis = make(CollectionIndexSet)
-       for _, index := range indices {
-               cis.Insert(index)
-       }
-       return
-}
-
-
 func (bs BlockSet) ToCollectionIndexSet(
        readCollections collection.ReadCollections,
        collectionIndexSet *CollectionIndexSet) {
        for block := range bs {
-               for collectionIndex := range readCollections.BlockToCollectionIndices[block] {
+               for _,collectionIndex := range readCollections.BlockToCollectionIndices[block] {
                        collectionIndexSet.Insert(collectionIndex)
                }
        }
@@ -65,6 +48,7 @@ type ReplicationSummary struct {
        CorrectlyReplicatedCollections CollectionIndexSet
 }
 
+// This struct counts the elements in each set in ReplicationSummary.
 type ReplicationSummaryCounts struct {
        CollectionBlocksNotInKeep      int
        UnderReplicatedBlocks          int
@@ -77,12 +61,9 @@ type ReplicationSummaryCounts struct {
        CorrectlyReplicatedCollections int
 }
 
-type serializedData struct {
-       ReadCollections collection.ReadCollections
-       KeepServerInfo  keep.ReadServers
-}
-
 func (rs ReplicationSummary) ComputeCounts() (rsc ReplicationSummaryCounts) {
+       // TODO(misha): Consider replacing this brute-force approach by
+       // iterating through the fields using reflection.
        rsc.CollectionBlocksNotInKeep = len(rs.CollectionBlocksNotInKeep)
        rsc.UnderReplicatedBlocks = len(rs.UnderReplicatedBlocks)
        rsc.OverReplicatedBlocks = len(rs.OverReplicatedBlocks)

diff --git a/services/datamanager/summary/summary_test.go b/services/datamanager/summary/summary_test.go
new file mode 100644 (file)
index 0000000..ff73b87
--- /dev/null
+++ b/services/datamanager/summary/summary_test.go
@@ -0,0 +1,274 @@
+package summary
+
+import (
+       "git.curoverse.com/arvados.git/sdk/go/blockdigest"
+       "git.curoverse.com/arvados.git/services/datamanager/collection"
+       "git.curoverse.com/arvados.git/services/datamanager/keep"
+       "reflect"
+       "testing"
+)
+
+func BlockSetFromSlice(digests []int) (bs BlockSet) {
+       bs = make(BlockSet)
+       for _, digest := range digests {
+               bs.Insert(blockdigest.MakeTestBlockDigest(digest))
+       }
+       return
+}
+
+func CollectionIndexSetFromSlice(indices []int) (cis CollectionIndexSet) {
+       cis = make(CollectionIndexSet)
+       for _, index := range indices {
+               cis.Insert(index)
+       }
+       return
+}
+
+func (cis CollectionIndexSet) ToSlice() (ints []int) {
+       ints = make([]int, len(cis))
+       i := 0
+       for collectionIndex := range cis {
+               ints[i] = collectionIndex
+               i++
+       }
+       return
+}
+
+func VerifyToCollectionIndexSet(
+       t *testing.T,
+       blocks []int,
+       blockToCollectionIndices map[int][]int,
+       expectedCollections []int) {
+
+       expected := CollectionIndexSetFromSlice(expectedCollections)
+
+       rc := collection.ReadCollections{
+               BlockToCollectionIndices: map[blockdigest.BlockDigest][]int{},
+       }
+       for digest, indices := range blockToCollectionIndices {
+               rc.BlockToCollectionIndices[blockdigest.MakeTestBlockDigest(digest)] = indices
+       }
+
+       returned := make(CollectionIndexSet)
+       BlockSetFromSlice(blocks).ToCollectionIndexSet(rc, &returned)
+
+       if !reflect.DeepEqual(returned, expected) {
+               t.Errorf("Expected %v.ToCollectionIndexSet(%v) to return \n %v \n but instead received \n %v",
+                       blocks,
+                       blockToCollectionIndices,
+                       expectedCollections,
+                       returned.ToSlice())
+       }
+}
+
+func TestToCollectionIndexSet(t *testing.T) {
+       VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4:[]int{1}}, []int{1})
+       VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4:[]int{1,9}}, []int{1,9})
+       VerifyToCollectionIndexSet(t, []int{5,6},
+               map[int][]int{5:[]int{2,3}, 6:[]int{3,4}},
+               []int{2,3,4})
+}
+
+
+func TestSimpleSummary(t *testing.T) {
+       rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
+               collection.TestCollectionSpec{
+                       ReplicationLevel: 1,
+                       Blocks:           []int{1, 2},
+               },
+       })
+
+       rc.Summarize()
+
+       // The internals aren't actually examined, so we can reuse the same one.
+       dummyBlockServerInfo := keep.BlockServerInfo{}
+
+       blockDigest1 := blockdigest.MakeTestBlockDigest(1)
+       blockDigest2 := blockdigest.MakeTestBlockDigest(2)
+
+       keepInfo := keep.ReadServers{
+               BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
+                       blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
+                       blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo},
+               },
+       }
+
+       returnedSummary := SummarizeReplication(rc, keepInfo)
+
+       c := rc.UuidToCollection["col0"]
+
+       expectedSummary := ReplicationSummary{
+               CollectionBlocksNotInKeep: BlockSet{},
+               UnderReplicatedBlocks:     BlockSet{},
+               OverReplicatedBlocks:      BlockSet{},
+               CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1,2}),
+               KeepBlocksNotInCollections: BlockSet{},
+
+               CollectionsNotFullyInKeep:  CollectionIndexSet{},
+               UnderReplicatedCollections: CollectionIndexSet{},
+               OverReplicatedCollections:  CollectionIndexSet{},
+               CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
+                       []int{rc.CollectionUuidToIndex[c.Uuid]}),
+       }
+
+       if !reflect.DeepEqual(returnedSummary, expectedSummary) {
+               t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
+       }
+}
+
+func TestMissingBlock(t *testing.T) {
+       rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
+               collection.TestCollectionSpec{
+                       ReplicationLevel: 1,
+                       Blocks:           []int{1, 2},
+               },
+       })
+
+       rc.Summarize()
+
+       // The internals aren't actually examined, so we can reuse the same one.
+       dummyBlockServerInfo := keep.BlockServerInfo{}
+
+       blockDigest1 := blockdigest.MakeTestBlockDigest(1)
+
+       keepInfo := keep.ReadServers{
+               BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
+                       blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
+               },
+       }
+
+       returnedSummary := SummarizeReplication(rc, keepInfo)
+
+       c := rc.UuidToCollection["col0"]
+
+       expectedSummary := ReplicationSummary{
+               CollectionBlocksNotInKeep: BlockSetFromSlice([]int{2}),
+               UnderReplicatedBlocks: BlockSet{},
+               OverReplicatedBlocks:  BlockSet{},
+               CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1}),
+               KeepBlocksNotInCollections: BlockSet{},
+
+               CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
+                       []int{rc.CollectionUuidToIndex[c.Uuid]}),
+               UnderReplicatedCollections:     CollectionIndexSet{},
+               OverReplicatedCollections:      CollectionIndexSet{},
+               CorrectlyReplicatedCollections: CollectionIndexSet{},
+       }
+
+       if !reflect.DeepEqual(returnedSummary, expectedSummary) {
+               t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
+       }
+}
+
+func TestUnderAndOverReplicatedBlocks(t *testing.T) {
+       rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
+               collection.TestCollectionSpec{
+                       ReplicationLevel: 2,
+                       Blocks:           []int{1, 2},
+               },
+       })
+
+       rc.Summarize()
+
+       // The internals aren't actually examined, so we can reuse the same one.
+       dummyBlockServerInfo := keep.BlockServerInfo{}
+
+       blockDigest1 := blockdigest.MakeTestBlockDigest(1)
+       blockDigest2 := blockdigest.MakeTestBlockDigest(2)
+
+       keepInfo := keep.ReadServers{
+               BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
+                       blockDigest1: []keep.BlockServerInfo{dummyBlockServerInfo},
+                       blockDigest2: []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
+               },
+       }
+
+       returnedSummary := SummarizeReplication(rc, keepInfo)
+
+       c := rc.UuidToCollection["col0"]
+
+       expectedSummary := ReplicationSummary{
+               CollectionBlocksNotInKeep: BlockSet{},
+               UnderReplicatedBlocks: BlockSetFromSlice([]int{1}),
+               OverReplicatedBlocks: BlockSetFromSlice([]int{2}),
+               CorrectlyReplicatedBlocks:  BlockSet{},
+               KeepBlocksNotInCollections: BlockSet{},
+
+               CollectionsNotFullyInKeep: CollectionIndexSet{},
+               UnderReplicatedCollections: CollectionIndexSetFromSlice(
+                       []int{rc.CollectionUuidToIndex[c.Uuid]}),
+               OverReplicatedCollections: CollectionIndexSetFromSlice(
+                       []int{rc.CollectionUuidToIndex[c.Uuid]}),
+               CorrectlyReplicatedCollections: CollectionIndexSet{},
+       }
+
+       if !reflect.DeepEqual(returnedSummary, expectedSummary) {
+               t.Fatalf("Expected returnedSummary to look like %+v but instead it is %+v", expectedSummary, returnedSummary)
+       }
+}
+
+func TestMixedReplication(t *testing.T) {
+       rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
+               collection.TestCollectionSpec{
+                       ReplicationLevel: 1,
+                       Blocks:           []int{1, 2},
+               },
+               collection.TestCollectionSpec{
+                       ReplicationLevel: 1,
+                       Blocks:           []int{3, 4},
+               },
+               collection.TestCollectionSpec{
+                       ReplicationLevel: 2,
+                       Blocks:           []int{5, 6},
+               },
+       })
+
+       rc.Summarize()
+
+       // The internals aren't actually examined, so we can reuse the same one.
+       dummyBlockServerInfo := keep.BlockServerInfo{}
+
+       keepInfo := keep.ReadServers{
+               BlockToServers: map[blockdigest.BlockDigest][]keep.BlockServerInfo{
+                       blockdigest.MakeTestBlockDigest(1): []keep.BlockServerInfo{dummyBlockServerInfo},
+                       blockdigest.MakeTestBlockDigest(2): []keep.BlockServerInfo{dummyBlockServerInfo},
+                       blockdigest.MakeTestBlockDigest(3): []keep.BlockServerInfo{dummyBlockServerInfo},
+                       blockdigest.MakeTestBlockDigest(5): []keep.BlockServerInfo{dummyBlockServerInfo},
+                       blockdigest.MakeTestBlockDigest(6): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo, dummyBlockServerInfo},
+                       blockdigest.MakeTestBlockDigest(7): []keep.BlockServerInfo{dummyBlockServerInfo, dummyBlockServerInfo},
+               },
+       }
+
+       returnedSummary := SummarizeReplication(rc, keepInfo)
+
+       c0 := rc.UuidToCollection["col0"]
+       c1 := rc.UuidToCollection["col1"]
+       c2 := rc.UuidToCollection["col2"]
+
+       expectedSummary := ReplicationSummary{
+               CollectionBlocksNotInKeep: BlockSetFromSlice([]int{4}),
+               UnderReplicatedBlocks: BlockSetFromSlice([]int{5}),
+               OverReplicatedBlocks: BlockSetFromSlice([]int{6}),
+               CorrectlyReplicatedBlocks: BlockSetFromSlice([]int{1,2,3}),
+               KeepBlocksNotInCollections: BlockSetFromSlice([]int{7}),
+
+               CollectionsNotFullyInKeep: CollectionIndexSetFromSlice(
+                       []int{rc.CollectionUuidToIndex[c1.Uuid]}),
+               UnderReplicatedCollections: CollectionIndexSetFromSlice(
+                       []int{rc.CollectionUuidToIndex[c2.Uuid]}),
+               OverReplicatedCollections: CollectionIndexSetFromSlice(
+                       []int{rc.CollectionUuidToIndex[c2.Uuid]}),
+               CorrectlyReplicatedCollections: CollectionIndexSetFromSlice(
+                       []int{rc.CollectionUuidToIndex[c0.Uuid]}),
+       }
+
+       tempCis := make(CollectionIndexSet)
+       returnedSummary.CollectionBlocksNotInKeep.ToCollectionIndexSet(rc, &tempCis)
+       t.Logf("blocks not in keep: %v, collections not fully in keep: %v",
+               returnedSummary.CollectionBlocksNotInKeep,
+               tempCis)
+
+       if !reflect.DeepEqual(returnedSummary, expectedSummary) {
+               t.Fatalf("Expected returnedSummary to look like: \n%+v but instead it is: \n%+v. Index to UUID is %v. BlockToCollectionIndices is %v.", expectedSummary, returnedSummary, rc.CollectionIndexToUuid, rc.BlockToCollectionIndices)
+       }
+}