X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d6446b03e2f5d5079a870bdd7b963456dc12b485..3108805fdc1984c90e3d69670d5911758eb64a1f:/lib/dispatchcloud/dispatcher_test.go

diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index 33d7f4e9ac..20185554b8 100644
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -207,8 +207,9 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
 		finishContainer(ctr)
 		return int(rand.Uint32() & 0x3)
 	}
+	var type4BrokenUntil time.Time
 	var countCapacityErrors int64
-	n := 0
+	vmCount := int32(0)
 	s.stubDriver.Queue = queue
 	s.stubDriver.SetupVM = func(stubvm *test.StubVM) error {
 		if pt := stubvm.Instance().ProviderType(); pt == test.InstanceType(6).ProviderType {
@@ -216,13 +217,25 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
 			atomic.AddInt64(&countCapacityErrors, 1)
 			return test.CapacityError{InstanceTypeSpecific: true}
 		}
-		n++
+		n := atomic.AddInt32(&vmCount, 1)
+		c.Logf("SetupVM: instance %s n=%d", stubvm.Instance(), n)
 		stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
 		stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
 		stubvm.ExecuteContainer = executeContainer
 		stubvm.CrashRunningContainer = finishContainer
 		stubvm.ExtraCrunchRunArgs = "'--runtime-engine=stub' '--foo' '--extra='\\''args'\\'''"
 		switch {
+		case stubvm.Instance().ProviderType() == test.InstanceType(4).ProviderType &&
+			(type4BrokenUntil.IsZero() || time.Now().Before(type4BrokenUntil)):
+			// Initially (at least 2*TimeoutBooting), all
+			// instances of this type are completely
+			// broken. This ensures the
+			// boot_outcomes{outcome="failure"} metric is
+			// not zero.
+			stubvm.Broken = time.Now()
+			if type4BrokenUntil.IsZero() {
+				type4BrokenUntil = time.Now().Add(2 * s.cluster.Containers.CloudVMs.TimeoutBooting.Duration())
+			}
 		case n%7 == 0:
 			// some instances start out OK but then stop
 			// running any commands
@@ -234,11 +247,6 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
 			// some instances start out OK but then start
 			// reporting themselves as broken
 			stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)) * time.Millisecond)
-		case n == 3:
-			// 1 instance is completely broken, ensuring
-			// the boot_outcomes{outcome="failure"} metric
-			// is not zero
-			stubvm.CrunchRunCrashRate = 1
 		default:
 			stubvm.CrunchRunCrashRate = 0.1
 			stubvm.ArvMountDeadlockRate = 0.1