From: Tom Clegg <tclegg@veritasgenetics.com>
Date: Fri, 17 Aug 2018 14:38:39 +0000 (-0400)
Subject: 13959: Merge branch 'master' into 13959-timeouts-and-logging
X-Git-Tag: 1.2.0~10^2~3
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/f739d736bbb60a8463f04f5d56c18d09157d820e

13959: Merge branch 'master' into 13959-timeouts-and-logging

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>
---

f739d736bbb60a8463f04f5d56c18d09157d820e
diff --cc sdk/go/dispatch/dispatch.go
index 152207ea94,4e25ba4f06..c8fb5aeb37
--- a/sdk/go/dispatch/dispatch.go
+++ b/sdk/go/dispatch/dispatch.go
@@@ -35,8 -29,9 +35,11 @@@ type Logger interface 
  type Dispatcher struct {
  	Arv *arvadosclient.ArvadosClient
  
 +	Logger Logger
 +
+ 	// Batch size for container queries
+ 	BatchSize int64
+ 
  	// Queue polling frequency
  	PollPeriod time.Duration
  
@@@ -174,8 -166,22 +181,22 @@@ func (d *Dispatcher) start(c arvados.Co
  }
  
  func (d *Dispatcher) checkForUpdates(filters [][]interface{}, todo map[string]*runTracker) bool {
+ 	var countList arvados.ContainerList
  	params := arvadosclient.Dict{
  		"filters": filters,
+ 		"count":   "exact",
+ 		"limit":   0,
+ 		"order":   []string{"priority desc"}}
+ 	err := d.Arv.List("containers", params, &countList)
+ 	if err != nil {
 -		log.Printf("error getting count of containers: %q", err)
++		d.Logger.Warnf("error getting count of containers: %q", err)
+ 		return false
+ 	}
+ 	itemsAvailable := countList.ItemsAvailable
+ 	params = arvadosclient.Dict{
+ 		"filters": filters,
+ 		"count":   "none",
+ 		"limit":   d.BatchSize,
  		"order":   []string{"priority desc"}}
  	offset := 0
  	for {
diff --cc services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index b12be91c91,36ef264963..ce0360261d
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@@ -182,7 -167,7 +185,8 @@@ func (disp *Dispatcher) setup() 
  	}
  	disp.Dispatcher = &dispatch.Dispatcher{
  		Arv:            arv,
 +		Logger:         disp.logger,
+ 		BatchSize:      disp.BatchSize,
  		RunContainer:   disp.runContainer,
  		PollPeriod:     time.Duration(disp.PollPeriod),
  		MinRetryPeriod: time.Duration(disp.MinRetryPeriod),