From: Tom Clegg <tclegg@veritasgenetics.com>
Date: Tue, 30 Oct 2018 20:12:11 +0000 (-0400)
Subject: 14360: Fail Create calls fast during quota quarantine.
X-Git-Tag: 1.4.0~180^2~40
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/3654f222637b00728158300e700d615c4c913b29?hp=1c1e245bc3163a7841859d5ff75d0cff59dfc90c

14360: Fail Create calls fast during quota quarantine.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>
---

diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go
index 68174e6b71..b1b570bd90 100644
--- a/lib/dispatchcloud/worker/pool.go
+++ b/lib/dispatchcloud/worker/pool.go
@@ -124,6 +124,7 @@ type Pool struct {
 	loaded       bool                 // loaded list of instances from InstanceSet at least once
 	exited       map[string]time.Time // containers whose crunch-run proc has exited, but KillContainer has not been called
 	atQuotaUntil time.Time
+	atQuotaErr   cloud.QuotaError
 	stop         chan bool
 	mtx          sync.RWMutex
 	setupOnce    sync.Once
@@ -208,6 +209,9 @@ func (wp *Pool) Create(it arvados.InstanceType) error {
 	wp.setupOnce.Do(wp.setup)
 	wp.mtx.Lock()
 	defer wp.mtx.Unlock()
+	if time.Now().Before(wp.atQuotaUntil) {
+		return wp.atQuotaErr
+	}
 	tags := cloud.InstanceTags{tagKeyInstanceType: it.Name}
 	wp.creating[it]++
 	go func() {
@@ -217,6 +221,7 @@ func (wp *Pool) Create(it arvados.InstanceType) error {
 		defer wp.mtx.Unlock()
 		wp.creating[it]--
 		if err, ok := err.(cloud.QuotaError); ok && err.IsQuotaError() {
+			wp.atQuotaErr = err
 			wp.atQuotaUntil = time.Now().Add(time.Minute)
 		}
 		if err != nil {