20888: Applies suggested improvements.

[arvados.git] / lib / dispatchcloud / dispatcher_test.go

diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index a9ed95c7c3b5f581c4e3a60776f0d5c207c34db9..a981d83a26c8aae993660413b97f088ec3fdb9b8 100644 (file)
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -49,8 +49,10 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
        s.stubDriver = &test.StubDriver{
                HostKey:                   hostpriv,
                AuthorizedKeys:            []ssh.PublicKey{dispatchpub},
+               ErrorRateCreate:           0.1,
                ErrorRateDestroy:          0.1,
                MinTimeBetweenCreateCalls: time.Millisecond,
+               QuotaMaxInstances:         10,
        }
 
        // We need the postgresql connection info from the integration
@@ -69,6 +71,7 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
                        DispatchPrivateKey:     string(dispatchprivraw),
                        StaleLockTimeout:       arvados.Duration(5 * time.Millisecond),
                        RuntimeEngine:          "stub",
+                       MaxDispatchAttempts:    10,
                        CloudVMs: arvados.CloudVMsConfig{
                                Driver:               "test",
                                SyncInterval:         arvados.Duration(10 * time.Millisecond),
@@ -77,6 +80,7 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
                                TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
                                TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
                                MaxCloudOpsPerSecond: 500,
+                               InitialQuotaEstimate: 8,
                                PollInterval:         arvados.Duration(5 * time.Millisecond),
                                ProbeInterval:        arvados.Duration(5 * time.Millisecond),
                                MaxProbesPerSecond:   1000,
@@ -101,9 +105,14 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
        arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
 
        arvClient, err := arvados.NewClientFromConfig(s.cluster)
-       c.Check(err, check.IsNil)
+       c.Assert(err, check.IsNil)
+       // Disable auto-retry
+       arvClient.Timeout = 0
 
-       s.error503Server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) }))
+       s.error503Server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+               c.Logf("503 stub: returning 503")
+               w.WriteHeader(http.StatusServiceUnavailable)
+       }))
        arvClient.Client = &http.Client{
                Transport: &http.Transport{
                        Proxy: s.arvClientProxy(c),
@@ -134,6 +143,7 @@ func (s *DispatcherSuite) TearDownTest(c *check.C) {
 func (s *DispatcherSuite) arvClientProxy(c *check.C) func(*http.Request) (*url.URL, error) {
        return func(req *http.Request) (*url.URL, error) {
                if req.URL.Path == "/503" {
+                       c.Logf("arvClientProxy: proxying to 503 stub")
                        return url.Parse(s.error503Server.URL)
                } else {
                        return nil, nil
@@ -149,6 +159,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
        Drivers["test"] = s.stubDriver
        s.disp.setupOnce.Do(s.disp.initialize)
        queue := &test.Queue{
+               MaxDispatchAttempts: 5,
                ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
                        return ChooseInstanceType(s.cluster, ctr)
                },
@@ -183,6 +194,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
                delete(waiting, ctr.UUID)
                if len(waiting) == 100 {
                        // trigger scheduler maxConcurrency limit
+                       c.Logf("test: requesting 503 in order to trigger maxConcurrency limit")
                        s.disp.ArvClient.RequestAndDecode(nil, "GET", "503", nil, nil)
                }
                if len(waiting) == 0 {
@@ -226,9 +238,9 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
                select {
                case <-done:
                        // loop will end because len(waiting)==0
-               case <-time.After(3 * time.Second):
+               case <-time.After(5 * time.Second):
                        if len(waiting) >= waswaiting {
-                               c.Fatalf("timed out; no progress in 3s while waiting for %d containers: %q", len(waiting), waiting)
+                               c.Fatalf("timed out; no progress in 5 s while waiting for %d containers: %q", len(waiting), waiting)
                        }
                }
        }
@@ -356,6 +368,7 @@ func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
        sr := getInstances()
        c.Check(len(sr.Items), check.Equals, 0)
 
+       s.stubDriver.ErrorRateCreate = 0
        ch := s.disp.pool.Subscribe()
        defer s.disp.pool.Unsubscribe(ch)
        ok := s.disp.pool.Create(test.InstanceType(1))