X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/7c013f919b5db9336833dbe855349600449a993d..d90fffe9d937d3e05c04106904b18dc4da235bc6:/lib/dispatchcloud/dispatcher_test.go

diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index a9ed95c7c3..a981d83a26 100644
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -49,8 +49,10 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
 	s.stubDriver = &test.StubDriver{
 		HostKey:                   hostpriv,
 		AuthorizedKeys:            []ssh.PublicKey{dispatchpub},
+		ErrorRateCreate:           0.1,
 		ErrorRateDestroy:          0.1,
 		MinTimeBetweenCreateCalls: time.Millisecond,
+		QuotaMaxInstances:         10,
 	}
 
 	// We need the postgresql connection info from the integration
@@ -69,6 +71,7 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
 			DispatchPrivateKey:     string(dispatchprivraw),
 			StaleLockTimeout:       arvados.Duration(5 * time.Millisecond),
 			RuntimeEngine:          "stub",
+			MaxDispatchAttempts:    10,
 			CloudVMs: arvados.CloudVMsConfig{
 				Driver:               "test",
 				SyncInterval:         arvados.Duration(10 * time.Millisecond),
@@ -77,6 +80,7 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
 				TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
 				TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
 				MaxCloudOpsPerSecond: 500,
+				InitialQuotaEstimate: 8,
 				PollInterval:         arvados.Duration(5 * time.Millisecond),
 				ProbeInterval:        arvados.Duration(5 * time.Millisecond),
 				MaxProbesPerSecond:   1000,
@@ -101,9 +105,14 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
 	arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
 
 	arvClient, err := arvados.NewClientFromConfig(s.cluster)
-	c.Check(err, check.IsNil)
+	c.Assert(err, check.IsNil)
+	// Disable auto-retry
+	arvClient.Timeout = 0
 
-	s.error503Server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) }))
+	s.error503Server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		c.Logf("503 stub: returning 503")
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}))
 	arvClient.Client = &http.Client{
 		Transport: &http.Transport{
 			Proxy: s.arvClientProxy(c),
@@ -134,6 +143,7 @@ func (s *DispatcherSuite) TearDownTest(c *check.C) {
 func (s *DispatcherSuite) arvClientProxy(c *check.C) func(*http.Request) (*url.URL, error) {
 	return func(req *http.Request) (*url.URL, error) {
 		if req.URL.Path == "/503" {
+			c.Logf("arvClientProxy: proxying to 503 stub")
 			return url.Parse(s.error503Server.URL)
 		} else {
 			return nil, nil
@@ -149,6 +159,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
 	Drivers["test"] = s.stubDriver
 	s.disp.setupOnce.Do(s.disp.initialize)
 	queue := &test.Queue{
+		MaxDispatchAttempts: 5,
 		ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
 			return ChooseInstanceType(s.cluster, ctr)
 		},
@@ -183,6 +194,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
 		delete(waiting, ctr.UUID)
 		if len(waiting) == 100 {
 			// trigger scheduler maxConcurrency limit
+			c.Logf("test: requesting 503 in order to trigger maxConcurrency limit")
 			s.disp.ArvClient.RequestAndDecode(nil, "GET", "503", nil, nil)
 		}
 		if len(waiting) == 0 {
@@ -226,9 +238,9 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
 		select {
 		case <-done:
 			// loop will end because len(waiting)==0
-		case <-time.After(3 * time.Second):
+		case <-time.After(5 * time.Second):
 			if len(waiting) >= waswaiting {
-				c.Fatalf("timed out; no progress in 3s while waiting for %d containers: %q", len(waiting), waiting)
+				c.Fatalf("timed out; no progress in 5 s while waiting for %d containers: %q", len(waiting), waiting)
 			}
 		}
 	}
@@ -356,6 +368,7 @@ func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
 	sr := getInstances()
 	c.Check(len(sr.Items), check.Equals, 0)
 
+	s.stubDriver.ErrorRateCreate = 0
 	ch := s.disp.pool.Subscribe()
 	defer s.disp.pool.Unsubscribe(ch)
 	ok := s.disp.pool.Create(test.InstanceType(1))