18071: Use dblock to prevent multiple dispatchers from competing.

[arvados.git] / lib / dispatchcloud / dispatcher_test.go

diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index 3b45e75fd063f76f4bbe5f96b37025c738e8aad2..bdbdf1ef9d83f99a0fad85d829f19064eb1924d3 100644 (file)
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -15,6 +15,7 @@ import (
        "sync"
        "time"
 
+       "git.arvados.org/arvados.git/lib/config"
        "git.arvados.org/arvados.git/lib/dispatchcloud/test"
        "git.arvados.org/arvados.git/sdk/go/arvados"
        "git.arvados.org/arvados.git/sdk/go/arvadostest"
@@ -49,11 +50,22 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
                MinTimeBetweenCreateCalls: time.Millisecond,
        }
 
+       // We need the postgresql connection info from the integration
+       // test config.
+       cfg, err := config.NewLoader(nil, ctxlog.FromContext(s.ctx)).Load()
+       c.Assert(err, check.IsNil)
+       testcluster, err := cfg.GetCluster("")
+       c.Assert(err, check.IsNil)
+
        s.cluster = &arvados.Cluster{
                ManagementToken: "test-management-token",
+               PostgreSQL:      testcluster.PostgreSQL,
                Containers: arvados.ContainersConfig{
-                       DispatchPrivateKey: string(dispatchprivraw),
-                       StaleLockTimeout:   arvados.Duration(5 * time.Millisecond),
+                       CrunchRunCommand:       "crunch-run",
+                       CrunchRunArgumentsList: []string{"--foo", "--extra='args'"},
+                       DispatchPrivateKey:     string(dispatchprivraw),
+                       StaleLockTimeout:       arvados.Duration(5 * time.Millisecond),
+                       RuntimeEngine:          "stub",
                        CloudVMs: arvados.CloudVMsConfig{
                                Driver:               "test",
                                SyncInterval:         arvados.Duration(10 * time.Millisecond),
@@ -66,6 +78,7 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
                                ProbeInterval:        arvados.Duration(5 * time.Millisecond),
                                MaxProbesPerSecond:   1000,
                                TimeoutSignal:        arvados.Duration(3 * time.Millisecond),
+                               TimeoutStaleRunLock:  arvados.Duration(3 * time.Millisecond),
                                TimeoutTERM:          arvados.Duration(20 * time.Millisecond),
                                ResourceTags:         map[string]string{"testtag": "test value"},
                                TagKeyPrefix:         "test:",
@@ -160,6 +173,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
                stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
                stubvm.ExecuteContainer = executeContainer
                stubvm.CrashRunningContainer = finishContainer
+               stubvm.ExtraCrunchRunArgs = "'--runtime-engine=stub' '--foo' '--extra='\\''args'\\'''"
                switch n % 7 {
                case 0:
                        stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
@@ -169,8 +183,10 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
                        stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)) * time.Millisecond)
                default:
                        stubvm.CrunchRunCrashRate = 0.1
+                       stubvm.ArvMountDeadlockRate = 0.1
                }
        }
+       s.stubDriver.Bugf = c.Errorf
 
        start := time.Now()
        go s.disp.run()
@@ -214,6 +230,20 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
        c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="success"} [^0].*`)
        c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="shutdown"} [^0].*`)
        c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="unknown"} 0\n.*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds{quantile="0.95"} [0-9.]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_count [0-9]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_sum [0-9.]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds{quantile="0.95"} [0-9.]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_count [0-9]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_sum [0-9.]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_count [0-9]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_sum [0-9.]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_count [0-9]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_sum [0-9e+.]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_count{outcome="success"} [0-9]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_sum{outcome="success"} [0-9e+.]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_count{outcome="fail"} [0-9]*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_sum{outcome="fail"} [0-9e+.]*`)
 }
 
 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
@@ -304,7 +334,7 @@ func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
                time.Sleep(time.Millisecond)
        }
        c.Assert(len(sr.Items), check.Equals, 1)
-       c.Check(sr.Items[0].Instance, check.Matches, "stub.*")
+       c.Check(sr.Items[0].Instance, check.Matches, "inst.*")
        c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
        c.Check(sr.Items[0].Price, check.Equals, 0.123)
        c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")