14360: Fix log spam on normal race condition.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/lib/dispatchcloud/scheduler/run_queue.go b/lib/dispatchcloud/scheduler/run_queue.go
index 7b432adc629d0da69f3497e1fa5fd7f7a70176f9..c8c25131eabf5e6b8343d050802a72cc9d1fd024 100644 (file)
--- a/lib/dispatchcloud/scheduler/run_queue.go
+++ b/lib/dispatchcloud/scheduler/run_queue.go
@@ -131,6 +131,17 @@ func (sch *Scheduler) bgLock(logger logrus.FieldLogger, uuid string) {
                logger.Debug("locking in progress, doing nothing")
                return
        }
+       if ctr, ok := sch.queue.Get(uuid); !ok || ctr.State != arvados.ContainerStateQueued {
+               // This happens if the container has been cancelled or
+               // locked since runQueue called sch.queue.Entries(),
+               // possibly by a bgLock() call from a previous
+               // runQueue iteration. In any case, we will respond
+               // appropriately on the next runQueue iteration, which
+               // will have already been triggered by the queue
+               // update.
+               logger.WithField("State", ctr.State).Debug("container no longer queued by the time we decided to lock it, doing nothing")
+               return
+       }
        sch.locking[uuid] = true
        go func() {
                defer func() {