13959: Merge branch 'master' into 13959-timeouts-and-logging

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --cc sdk/go/dispatch/dispatch.go
index 152207ea94bb4bb4c4e21de36ca2b440691109a2,4e25ba4f0603699569402d619127dd4b9fd99fb1..c8fb5aeb37e97eb9a43846301e2a07ea63e28ae7
--- 1/sdk/go/dispatch/dispatch.go
--- 2/sdk/go/dispatch/dispatch.go
+++ b/sdk/go/dispatch/dispatch.go
@@@ -35,8 -29,9 +35,11 @@@ type Logger interface 
  type Dispatcher struct {
        Arv *arvadosclient.ArvadosClient
  
 +      Logger Logger
 +
+       // Batch size for container queries
+       BatchSize int64
+ 
        // Queue polling frequency
        PollPeriod time.Duration
  
@@@ -174,8 -166,22 +181,22 @@@ func (d *Dispatcher) start(c arvados.Co
  }
  
  func (d *Dispatcher) checkForUpdates(filters [][]interface{}, todo map[string]*runTracker) bool {
+       var countList arvados.ContainerList
        params := arvadosclient.Dict{
                "filters": filters,
 -              log.Printf("error getting count of containers: %q", err)
+               "count":   "exact",
+               "limit":   0,
+               "order":   []string{"priority desc"}}
+       err := d.Arv.List("containers", params, &countList)
+       if err != nil {
++              d.Logger.Warnf("error getting count of containers: %q", err)
+               return false
+       }
+       itemsAvailable := countList.ItemsAvailable
+       params = arvadosclient.Dict{
+               "filters": filters,
+               "count":   "none",
+               "limit":   d.BatchSize,
                "order":   []string{"priority desc"}}
        offset := 0
        for {

diff --cc services/crunch-dispatch-local/crunch-dispatch-local_test.go
Simple merge

diff --cc services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index b12be91c91e847fda0da2e76b8947abfac325100,36ef264963d760f04501fc25cee6916c62ef4bf2..ce0360261dab4aa3ab424d27c29c782e268b567f
--- 1/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
--- 2/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@@ -182,7 -167,7 +185,8 @@@ func (disp *Dispatcher) setup() 
        }
        disp.Dispatcher = &dispatch.Dispatcher{
                Arv:            arv,
 +              Logger:         disp.logger,
+               BatchSize:      disp.BatchSize,
                RunContainer:   disp.runContainer,
                PollPeriod:     time.Duration(disp.PollPeriod),
                MinRetryPeriod: time.Duration(disp.MinRetryPeriod),

diff --cc services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
Simple merge