From ad272d78c6238b3c5cc250e643b8fda632b11d70 Mon Sep 17 00:00:00 2001
From: Tom Clegg <tom@curoverse.com>
Date: Mon, 13 Jun 2016 10:31:39 -0400
Subject: [PATCH] 9395: Add CollectionBuffers config to keep-balance.

---
 services/keep-balance/balance.go | 6 +++---
 services/keep-balance/main.go    | 5 +++++
 services/keep-balance/usage.go   | 3 ++-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 0ca6e713c4..2d1a59e890 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -80,7 +80,7 @@ func (bal *Balancer) Run(config Config, runOptions RunOptions) (err error) {
 			return
 		}
 	}
-	if err = bal.GetCurrentState(&config.Client, config.CollectionBatchSize); err != nil {
+	if err = bal.GetCurrentState(&config.Client, config.CollectionBatchSize, config.CollectionBuffers); err != nil {
 		return
 	}
 	bal.ComputeChangeSets()
@@ -190,7 +190,7 @@ func (bal *Balancer) ClearTrashLists(c *arvados.Client) error {
 // collection manifests in the database (API server).
 //
 // It encodes the resulting information in BlockStateMap.
-func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize int) error {
+func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) error {
 	defer timeMe(bal.Logger, "GetCurrentState")()
 	bal.BlockStateMap = NewBlockStateMap()
 
@@ -225,7 +225,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize int) error {
 	// collQ buffers incoming collections so we can start fetching
 	// the next page without waiting for the current page to
 	// finish processing.
-	collQ := make(chan arvados.Collection, 1000)
+	collQ := make(chan arvados.Collection, bufs)
 
 	// Start a goroutine to process collections. (We could use a
 	// worker pool here, but even with a single worker we already
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 6e7d70d4e8..364bb3ffd3 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -31,6 +31,11 @@ type Config struct {
 
 	// Number of collections to request in each API call
 	CollectionBatchSize int
+
+	// Max collections to buffer in memory (bigger values consume
+	// more memory, but can reduce store-and-forward latency when
+	// fetching pages)
+	CollectionBuffers int
 }
 
 // RunOptions controls runtime behavior. The flags/options that belong
diff --git a/services/keep-balance/usage.go b/services/keep-balance/usage.go
index c5363641ca..2273f3a57f 100644
--- a/services/keep-balance/usage.go
+++ b/services/keep-balance/usage.go
@@ -17,7 +17,8 @@ var exampleConfigFile = []byte(`
 	    "disk"
 	],
 	"RunPeriod": "600s",
-	"CollectionBatchSize": 100000
+	"CollectionBatchSize": 100000,
+	"CollectionBuffers": 1000
     }`)
 
 func usage() {
-- 
2.30.2