From a2fe6f9367de3ee93064fbee3f2df78ce84aa318 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Fri, 10 Feb 2017 13:31:21 -0500 Subject: [PATCH] 10701: Get multiple pages of queue if necessary. --- sdk/go/dispatch/dispatch.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/sdk/go/dispatch/dispatch.go b/sdk/go/dispatch/dispatch.go index 5341369d01..5b19485c9e 100644 --- a/sdk/go/dispatch/dispatch.go +++ b/sdk/go/dispatch/dispatch.go @@ -118,30 +118,29 @@ func (d *Dispatcher) start(c arvados.Container) *runTracker { func (d *Dispatcher) checkForUpdates(filters [][]interface{}) { params := arvadosclient.Dict{ "filters": filters, - "order": []string{"priority desc"}, - "limit": "1000"} + "order": []string{"priority desc"}} var list arvados.ContainerList - err := d.Arv.List("containers", params, &list) - if err != nil { - log.Printf("Error getting list of containers: %q", err) - return - } - - if list.ItemsAvailable > len(list.Items) { - // TODO: support paging - log.Printf("Warning! %d containers are available but only received %d, paged requests are not yet supported, some containers may be ignored.", - list.ItemsAvailable, - len(list.Items)) + for offset, more := 0, true; more; offset += len(list.Items) { + params["offset"] = offset + err := d.Arv.List("containers", params, &list) + if err != nil { + log.Printf("Error getting list of containers: %q", err) + return + } + more = list.ItemsAvailable > len(list.Items) + d.checkListForUpdates(list.Items) } +} +func (d *Dispatcher) checkListForUpdates(containers []arvados.Container) { d.mtx.Lock() defer d.mtx.Unlock() if d.running == nil { d.running = make(map[string]*runTracker) } - for _, c := range list.Items { + for _, c := range containers { tracker, running := d.running[c.UUID] if c.LockedByUUID != "" && c.LockedByUUID != d.auth.UUID { log.Printf("debug: ignoring %s locked by %s", c.UUID, c.LockedByUUID) -- 2.30.2