14285: Add deduplication ratio to keep-balance metrics.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 333a4fbde99b8470ed25fb45503baf2f2d5241a1..5f5f9afdc4741d6c8b74fb582f52a706fd5ea9be 100644 (file)
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -780,6 +780,26 @@ type balancerStats struct {
        trashes       int
        replHistogram []int
        classStats    map[string]replicationStats
+
+       // collectionBytes / collectionBlockBytes = deduplication ratio
+       collectionBytes      int64 // sum(bytes in referenced blocks) across all collections
+       collectionBlockBytes int64 // sum(block size) across all blocks referenced by collections
+       collectionBlockRefs  int64 // sum(number of blocks referenced) across all collections
+       collectionBlocks     int64 // number of blocks referenced by any collection
+}
+
+func (s *balancerStats) dedupByteRatio() float64 {
+       if s.collectionBlockBytes == 0 {
+               return 0
+       }
+       return float64(s.collectionBytes) / float64(s.collectionBlockBytes)
+}
+
+func (s *balancerStats) dedupBlockRatio() float64 {
+       if s.collectionBlocks == 0 {
+               return 0
+       }
+       return float64(s.collectionBlockRefs) / float64(s.collectionBlocks)
 }
 
 type replicationStats struct {
@@ -803,6 +823,13 @@ func (bal *Balancer) collectStatistics(results <-chan balanceResult) {
                surplus := result.have - result.want
                bytes := result.blkid.Size()
 
+               if rc := int64(result.blk.RefCount); rc > 0 {
+                       s.collectionBytes += rc * bytes
+                       s.collectionBlockBytes += bytes
+                       s.collectionBlockRefs += rc
+                       s.collectionBlocks++
+               }
+
                for class, state := range result.classState {
                        cs := s.classStats[class]
                        if state.unachievable {

diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index e1b92005f340674050ec2a4c34a91b9ca399e6a4..923ea47c429a6051377575376860b0bf51fa5915 100644 (file)
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -184,7 +184,8 @@ func (s *stubServer) serveFooBarFileCollections() *reqTracker {
                if strings.Contains(r.Form.Get("filters"), `modified_at`) {
                        io.WriteString(w, `{"items_available":0,"items":[]}`)
                } else {
-                       io.WriteString(w, `{"items_available":2,"items":[
+                       io.WriteString(w, `{"items_available":3,"items":[
+                               {"uuid":"zzzzz-4zz18-aaaaaaaaaaaaaaa","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
                                {"uuid":"zzzzz-4zz18-ehbhgtheo8909or","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
                                {"uuid":"zzzzz-4zz18-znfnqtbbv4spc3w","portable_data_hash":"1f4b0bc7583c2a7f9102c395f4ffc5e3+45","manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n","modified_at":"2014-02-03T17:22:54Z"}]}`)
                }
@@ -463,6 +464,8 @@ func (s *runSuite) TestCommit(c *check.C) {
        c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_total_bytes 15\n.*`)
        c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_sum [0-9\.]+\n.*`)
        c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count 1\n.*`)
+       c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_dedup_byte_ratio 1\.5\n.*`)
+       c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_dedup_block_ratio 1\.5\n.*`)
 }
 
 func (s *runSuite) TestRunForever(c *check.C) {

diff --git a/services/keep-balance/block_state.go b/services/keep-balance/block_state.go
index 22e89c019ab9fa5a5fb833bf84bbc63df7a4e93b..46e69059c9c796c5b23318f8f9b78b4f3c83651e 100644 (file)
--- a/services/keep-balance/block_state.go
+++ b/services/keep-balance/block_state.go
@@ -23,6 +23,7 @@ type Replica struct {
 // replicas actually stored (according to the keepstore indexes we
 // know about).
 type BlockState struct {
+       RefCount int
        Replicas []Replica
        Desired  map[string]int
        // TODO: Support combinations of classes ("private + durable")
@@ -42,6 +43,7 @@ func (bs *BlockState) addReplica(r Replica) {
 }
 
 func (bs *BlockState) increaseDesired(classes []string, n int) {
+       bs.RefCount++
        if len(classes) == 0 {
                classes = defaultClasses
        }

diff --git a/services/keep-balance/metrics.go b/services/keep-balance/metrics.go
index 0564d46cdae76c471129c4bfbf087ed2d06748db..5f3c98723d02a82e9410053657d3262dca3af1be 100644 (file)
--- a/services/keep-balance/metrics.go
+++ b/services/keep-balance/metrics.go
@@ -58,12 +58,14 @@ func (m *metrics) UpdateStats(s balancerStats) {
                Help  string
        }
        s2g := map[string]gauge{
-               "total":           {s.current, "current backend storage usage"},
-               "garbage":         {s.garbage, "garbage (unreferenced, old)"},
-               "transient":       {s.unref, "transient (unreferenced, new)"},
-               "overreplicated":  {s.overrep, "overreplicated"},
-               "underreplicated": {s.underrep, "underreplicated"},
-               "lost":            {s.lost, "lost"},
+               "total":             {s.current, "current backend storage usage"},
+               "garbage":           {s.garbage, "garbage (unreferenced, old)"},
+               "transient":         {s.unref, "transient (unreferenced, new)"},
+               "overreplicated":    {s.overrep, "overreplicated"},
+               "underreplicated":   {s.underrep, "underreplicated"},
+               "lost":              {s.lost, "lost"},
+               "dedup_byte_ratio":  {s.dedupByteRatio(), "deduplication ratio, bytes referenced / bytes stored"},
+               "dedup_block_ratio": {s.dedupBlockRatio(), "deduplication ratio, blocks referenced / blocks stored"},
        }
        m.setupOnce.Do(func() {
                // Register gauge(s) for each balancerStats field.
@@ -83,7 +85,7 @@ func (m *metrics) UpdateStats(s balancerStats) {
                                for _, sub := range []string{"blocks", "bytes", "replicas"} {
                                        addGauge(name+"_"+sub, sub+" of "+gauge.Help)
                                }
-                       case int, int64:
+                       case int, int64, float64:
                                addGauge(name, gauge.Help)
                        default:
                                panic(fmt.Sprintf("bad gauge type %T", gauge.Value))
@@ -101,6 +103,8 @@ func (m *metrics) UpdateStats(s balancerStats) {
                        m.statsGauges[name].Set(float64(val))
                case int64:
                        m.statsGauges[name].Set(float64(val))
+               case float64:
+                       m.statsGauges[name].Set(float64(val))
                default:
                        panic(fmt.Sprintf("bad gauge type %T", gauge.Value))
                }