15340: Fix uncounted Destroy ops.

[arvados.git] / lib / dispatchcloud / dispatcher_test.go

diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index b0033353c2e822d1e6883f7d72d117ac8474e84f..07fdc672282d5475308a9f010555f9a0efc241c6 100644 (file)
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -17,6 +17,7 @@ import (
 
        "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
        "git.curoverse.com/arvados.git/sdk/go/arvados"
+       "git.curoverse.com/arvados.git/sdk/go/arvadostest"
        "git.curoverse.com/arvados.git/sdk/go/ctxlog"
        "golang.org/x/crypto/ssh"
        check "gopkg.in/check.v1"
@@ -48,23 +49,26 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
        }
 
        s.cluster = &arvados.Cluster{
-               CloudVMs: arvados.CloudVMs{
-                       Driver:               "test",
-                       SyncInterval:         arvados.Duration(10 * time.Millisecond),
-                       TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
-                       TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
-                       TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
-                       TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
-                       MaxCloudOpsPerSecond: 500,
-               },
-               Dispatch: arvados.Dispatch{
-                       PrivateKey:         string(dispatchprivraw),
-                       PollInterval:       arvados.Duration(5 * time.Millisecond),
-                       ProbeInterval:      arvados.Duration(5 * time.Millisecond),
+               ManagementToken: "test-management-token",
+               Containers: arvados.ContainersConfig{
+                       DispatchPrivateKey: string(dispatchprivraw),
                        StaleLockTimeout:   arvados.Duration(5 * time.Millisecond),
-                       MaxProbesPerSecond: 1000,
-                       TimeoutSignal:      arvados.Duration(3 * time.Millisecond),
-                       TimeoutTERM:        arvados.Duration(20 * time.Millisecond),
+                       CloudVMs: arvados.CloudVMsConfig{
+                               Driver:               "test",
+                               SyncInterval:         arvados.Duration(10 * time.Millisecond),
+                               TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
+                               TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
+                               TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
+                               TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
+                               MaxCloudOpsPerSecond: 500,
+                               PollInterval:         arvados.Duration(5 * time.Millisecond),
+                               ProbeInterval:        arvados.Duration(5 * time.Millisecond),
+                               MaxProbesPerSecond:   1000,
+                               TimeoutSignal:        arvados.Duration(3 * time.Millisecond),
+                               TimeoutTERM:          arvados.Duration(20 * time.Millisecond),
+                               ResourceTags:         map[string]string{"testtag": "test value"},
+                               TagKeyPrefix:         "test:",
+                       },
                },
                InstanceTypes: arvados.InstanceTypeMap{
                        test.InstanceType(1).Name:  test.InstanceType(1),
@@ -75,16 +79,18 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
                        test.InstanceType(8).Name:  test.InstanceType(8),
                        test.InstanceType(16).Name: test.InstanceType(16),
                },
-               NodeProfiles: map[string]arvados.NodeProfile{
-                       "*": {
-                               Controller:    arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_API_HOST")},
-                               DispatchCloud: arvados.SystemServiceInstance{Listen: ":"},
-                       },
-               },
        }
+       arvadostest.SetServiceURL(&s.cluster.Services.DispatchCloud, "http://localhost:/")
+       arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
+
+       arvClient, err := arvados.NewClientFromConfig(s.cluster)
+       c.Check(err, check.IsNil)
+
        s.disp = &dispatcher{
-               Cluster: s.cluster,
-               Context: s.ctx,
+               Cluster:   s.cluster,
+               Context:   s.ctx,
+               ArvClient: arvClient,
+               AuthToken: arvadostest.AdminToken,
        }
        // Test cases can modify s.cluster before calling
        // initialize(), and then modify private state before calling
@@ -188,6 +194,19 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
                        c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
                }
        }
+
+       req := httptest.NewRequest("GET", "/metrics", nil)
+       req.Header.Set("Authorization", "Bearer "+s.cluster.ManagementToken)
+       resp := httptest.NewRecorder()
+       s.disp.ServeHTTP(resp, req)
+       c.Check(resp.Code, check.Equals, http.StatusOK)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Create"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="List"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Destroy"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="Create"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="List"} 0\n.*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="shutdown"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="unknown"} 0\n.*`)
 }
 
 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
@@ -232,7 +251,7 @@ func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
 
 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
        s.cluster.ManagementToken = "abcdefgh"
-       s.cluster.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
+       s.cluster.Containers.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
        drivers["test"] = s.stubDriver
        s.disp.setupOnce.Do(s.disp.initialize)
        s.disp.queue = &test.Queue{}