X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/4355586821d71fed6a3fe95fea69f548797f77d8..e0d89f524f6ebdb5dc730402ff3c61e6d8160c9e:/lib/dispatchcloud/scheduler/run_queue.go

diff --git a/lib/dispatchcloud/scheduler/run_queue.go b/lib/dispatchcloud/scheduler/run_queue.go
index d77dcee947..f729f0dc23 100644
--- a/lib/dispatchcloud/scheduler/run_queue.go
+++ b/lib/dispatchcloud/scheduler/run_queue.go
@@ -20,7 +20,16 @@ func (sch *Scheduler) runQueue() {
 		sorted = append(sorted, ent)
 	}
 	sort.Slice(sorted, func(i, j int) bool {
-		return sorted[i].Container.Priority > sorted[j].Container.Priority
+		if pi, pj := sorted[i].Container.Priority, sorted[j].Container.Priority; pi != pj {
+			return pi > pj
+		} else {
+			// When containers have identical priority,
+			// start them in the order we first noticed
+			// them. This avoids extra lock/unlock cycles
+			// when we unlock the containers that don't
+			// fit in the available pool.
+			return sorted[i].FirstSeenAt.Before(sorted[j].FirstSeenAt)
+		}
 	})
 
 	running := sch.pool.Running()
@@ -33,6 +42,7 @@ func (sch *Scheduler) runQueue() {
 
 	dontstart := map[arvados.InstanceType]bool{}
 	var overquota []container.QueueEnt // entries that are unmappable because of worker pool quota
+	var containerAllocatedWorkerBootingCount int
 
 tryrun:
 	for i, ctr := range sorted {
@@ -61,30 +71,27 @@ tryrun:
 			if unalloc[it] > 0 {
 				unalloc[it]--
 			} else if sch.pool.AtQuota() {
-				logger.Debug("not starting: AtQuota and no unalloc workers")
+				// Don't let lower-priority containers
+				// starve this one by using keeping
+				// idle workers alive on different
+				// instance types.
+				logger.Trace("overquota")
 				overquota = sorted[i:]
 				break tryrun
-			} else {
+			} else if sch.pool.Create(it) {
+				// Success. (Note pool.Create works
+				// asynchronously and does its own
+				// logging about the eventual outcome,
+				// so we don't need to.)
 				logger.Info("creating new instance")
-				if !sch.pool.Create(it) {
-					// (Note pool.Create works
-					// asynchronously and logs its
-					// own failures, so we don't
-					// need to log this as a
-					// failure.)
-
-					sch.queue.Unlock(ctr.UUID)
-					// Don't let lower-priority
-					// containers starve this one
-					// by using keeping idle
-					// workers alive on different
-					// instance types.  TODO:
-					// avoid getting starved here
-					// if instances of a specific
-					// type always fail.
-					overquota = sorted[i:]
-					break tryrun
-				}
+			} else {
+				// Failed despite not being at quota,
+				// e.g., cloud ops throttled.  TODO:
+				// avoid getting starved here if
+				// instances of a specific type always
+				// fail.
+				logger.Trace("pool declined to create new instance")
+				continue
 			}
 
 			if dontstart[it] {
@@ -97,11 +104,15 @@ tryrun:
 			} else if sch.pool.StartContainer(it, ctr) {
 				// Success.
 			} else {
+				containerAllocatedWorkerBootingCount += 1
 				dontstart[it] = true
 			}
 		}
 	}
 
+	sch.mContainersAllocatedNotStarted.Set(float64(containerAllocatedWorkerBootingCount))
+	sch.mContainersNotAllocatedOverQuota.Set(float64(len(overquota)))
+
 	if len(overquota) > 0 {
 		// Unlock any containers that are unmappable while
 		// we're at quota.