From b30659d514ce281209fa7b99863413832fa8d44b Mon Sep 17 00:00:00 2001
From: Tom Clegg <tom@tomclegg.ca>
Date: Wed, 2 Sep 2020 08:56:38 -0400
Subject: [PATCH] 16795: Don't shutdown idle nodes just because Create failed.

Previous behavior would prematurely shutdown booting instances when
Create calls were being rate-limited.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>
---
 lib/dispatchcloud/scheduler/run_queue.go | 37 ++++++++++--------------
 1 file changed, 16 insertions(+), 21 deletions(-)

diff --git a/lib/dispatchcloud/scheduler/run_queue.go b/lib/dispatchcloud/scheduler/run_queue.go
index d77dcee947..0e8e1dc2ec 100644
--- a/lib/dispatchcloud/scheduler/run_queue.go
+++ b/lib/dispatchcloud/scheduler/run_queue.go
@@ -61,30 +61,25 @@ tryrun:
 			if unalloc[it] > 0 {
 				unalloc[it]--
 			} else if sch.pool.AtQuota() {
-				logger.Debug("not starting: AtQuota and no unalloc workers")
+				// Don't let lower-priority containers
+				// starve this one by using keeping
+				// idle workers alive on different
+				// instance types.
+				logger.Debug("unlocking: AtQuota and no unalloc workers")
+				sch.queue.Unlock(ctr.UUID)
 				overquota = sorted[i:]
 				break tryrun
+			} else if logger.Info("creating new instance"); sch.pool.Create(it) {
+				// Success. (Note pool.Create works
+				// asynchronously and does its own
+				// logging, so we don't need to.)
 			} else {
-				logger.Info("creating new instance")
-				if !sch.pool.Create(it) {
-					// (Note pool.Create works
-					// asynchronously and logs its
-					// own failures, so we don't
-					// need to log this as a
-					// failure.)
-
-					sch.queue.Unlock(ctr.UUID)
-					// Don't let lower-priority
-					// containers starve this one
-					// by using keeping idle
-					// workers alive on different
-					// instance types.  TODO:
-					// avoid getting starved here
-					// if instances of a specific
-					// type always fail.
-					overquota = sorted[i:]
-					break tryrun
-				}
+				// Failed despite not being at quota,
+				// e.g., cloud ops throttled.  TODO:
+				// avoid getting starved here if
+				// instances of a specific type always
+				// fail.
+				continue
 			}
 
 			if dontstart[it] {
-- 
2.39.5