Merge branch '19095-singularity-bugfix'

[arvados.git] / lib / dispatchcloud / scheduler / run_queue.go

diff --git a/lib/dispatchcloud/scheduler/run_queue.go b/lib/dispatchcloud/scheduler/run_queue.go
index 9bbb064fee6a06bcc769b49fe8b7ae2b417b1af3..f729f0dc23a7f927eca6c39fe75734a4a2355ad9 100644 (file)
--- a/lib/dispatchcloud/scheduler/run_queue.go
+++ b/lib/dispatchcloud/scheduler/run_queue.go
@@ -20,7 +20,16 @@ func (sch *Scheduler) runQueue() {
                sorted = append(sorted, ent)
        }
        sort.Slice(sorted, func(i, j int) bool {
-               return sorted[i].Container.Priority > sorted[j].Container.Priority
+               if pi, pj := sorted[i].Container.Priority, sorted[j].Container.Priority; pi != pj {
+                       return pi > pj
+               } else {
+                       // When containers have identical priority,
+                       // start them in the order we first noticed
+                       // them. This avoids extra lock/unlock cycles
+                       // when we unlock the containers that don't
+                       // fit in the available pool.
+                       return sorted[i].FirstSeenAt.Before(sorted[j].FirstSeenAt)
+               }
        })
 
        running := sch.pool.Running()
@@ -34,7 +43,6 @@ func (sch *Scheduler) runQueue() {
        dontstart := map[arvados.InstanceType]bool{}
        var overquota []container.QueueEnt // entries that are unmappable because of worker pool quota
        var containerAllocatedWorkerBootingCount int
-       var longestWaitTimeCandidate, previousLongestWaitTimeCandidate float64
 
 tryrun:
        for i, ctr := range sorted {
@@ -46,11 +54,6 @@ tryrun:
                if _, running := running[ctr.UUID]; running || ctr.Priority < 1 {
                        continue
                }
-               previousLongestWaitTimeCandidate = longestWaitTimeCandidate
-               since := time.Since(ctr.CreatedAt).Seconds()
-               if since > longestWaitTimeCandidate {
-                       longestWaitTimeCandidate = since
-               }
                switch ctr.State {
                case arvados.ContainerStateQueued:
                        if unalloc[it] < 1 && sch.pool.AtQuota() {
@@ -72,20 +75,22 @@ tryrun:
                                // starve this one by using keeping
                                // idle workers alive on different
                                // instance types.
-                               logger.Debug("unlocking: AtQuota and no unalloc workers")
-                               sch.queue.Unlock(ctr.UUID)
+                               logger.Trace("overquota")
                                overquota = sorted[i:]
                                break tryrun
-                       } else if logger.Info("creating new instance"); sch.pool.Create(it) {
+                       } else if sch.pool.Create(it) {
                                // Success. (Note pool.Create works
                                // asynchronously and does its own
-                               // logging, so we don't need to.)
+                               // logging about the eventual outcome,
+                               // so we don't need to.)
+                               logger.Info("creating new instance")
                        } else {
                                // Failed despite not being at quota,
                                // e.g., cloud ops throttled.  TODO:
                                // avoid getting starved here if
                                // instances of a specific type always
                                // fail.
+                               logger.Trace("pool declined to create new instance")
                                continue
                        }
 
@@ -98,7 +103,6 @@ tryrun:
                                logger.Info("not restarting yet: crunch-run process from previous attempt has not exited")
                        } else if sch.pool.StartContainer(it, ctr) {
                                // Success.
-                               longestWaitTimeCandidate = previousLongestWaitTimeCandidate
                        } else {
                                containerAllocatedWorkerBootingCount += 1
                                dontstart[it] = true
@@ -108,7 +112,6 @@ tryrun:
 
        sch.mContainersAllocatedNotStarted.Set(float64(containerAllocatedWorkerBootingCount))
        sch.mContainersNotAllocatedOverQuota.Set(float64(len(overquota)))
-       sch.mLongestWaitTimeSinceQueue.Set(longestWaitTimeCandidate)
 
        if len(overquota) > 0 {
                // Unlock any containers that are unmappable while