projects / arvados.git / blobdiff
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
16723: Reject invalid updates in API stub.
[arvados.git] / lib / dispatchcloud / dispatcher_test.go
diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index 44d5a0ae75b2b8b4400bdb090e0dfc9baaf90198..3b45e75fd063f76f4bbe5f96b37025c738e8aad2 100644 (file)
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -15,9 +15,11 @@ import (
        "sync"
        "time"
 
-       "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
-       "git.curoverse.com/arvados.git/sdk/go/arvados"
-       "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+       "git.arvados.org/arvados.git/lib/dispatchcloud/test"
+       "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadostest"
+       "git.arvados.org/arvados.git/sdk/go/ctxlog"
+       "github.com/prometheus/client_golang/prometheus"
        "golang.org/x/crypto/ssh"
        check "gopkg.in/check.v1"
 )
@@ -48,22 +50,26 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
        }
 
        s.cluster = &arvados.Cluster{
-               CloudVMs: arvados.CloudVMs{
-                       Driver:          "test",
-                       SyncInterval:    arvados.Duration(10 * time.Millisecond),
-                       TimeoutIdle:     arvados.Duration(150 * time.Millisecond),
-                       TimeoutBooting:  arvados.Duration(150 * time.Millisecond),
-                       TimeoutProbe:    arvados.Duration(15 * time.Millisecond),
-                       TimeoutShutdown: arvados.Duration(5 * time.Millisecond),
-               },
-               Dispatch: arvados.Dispatch{
-                       PrivateKey:         string(dispatchprivraw),
-                       PollInterval:       arvados.Duration(5 * time.Millisecond),
-                       ProbeInterval:      arvados.Duration(5 * time.Millisecond),
+               ManagementToken: "test-management-token",
+               Containers: arvados.ContainersConfig{
+                       DispatchPrivateKey: string(dispatchprivraw),
                        StaleLockTimeout:   arvados.Duration(5 * time.Millisecond),
-                       MaxProbesPerSecond: 1000,
-                       TimeoutSignal:      arvados.Duration(3 * time.Millisecond),
-                       TimeoutTERM:        arvados.Duration(20 * time.Millisecond),
+                       CloudVMs: arvados.CloudVMsConfig{
+                               Driver:               "test",
+                               SyncInterval:         arvados.Duration(10 * time.Millisecond),
+                               TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
+                               TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
+                               TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
+                               TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
+                               MaxCloudOpsPerSecond: 500,
+                               PollInterval:         arvados.Duration(5 * time.Millisecond),
+                               ProbeInterval:        arvados.Duration(5 * time.Millisecond),
+                               MaxProbesPerSecond:   1000,
+                               TimeoutSignal:        arvados.Duration(3 * time.Millisecond),
+                               TimeoutTERM:          arvados.Duration(20 * time.Millisecond),
+                               ResourceTags:         map[string]string{"testtag": "test value"},
+                               TagKeyPrefix:         "test:",
+                       },
                },
                InstanceTypes: arvados.InstanceTypeMap{
                        test.InstanceType(1).Name:  test.InstanceType(1),
@@ -74,16 +80,19 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
                        test.InstanceType(8).Name:  test.InstanceType(8),
                        test.InstanceType(16).Name: test.InstanceType(16),
                },
-               NodeProfiles: map[string]arvados.NodeProfile{
-                       "*": {
-                               Controller:    arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_API_HOST")},
-                               DispatchCloud: arvados.SystemServiceInstance{Listen: ":"},
-                       },
-               },
        }
+       arvadostest.SetServiceURL(&s.cluster.Services.DispatchCloud, "http://localhost:/")
+       arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
+
+       arvClient, err := arvados.NewClientFromConfig(s.cluster)
+       c.Check(err, check.IsNil)
+
        s.disp = &dispatcher{
-               Cluster: s.cluster,
-               Context: s.ctx,
+               Cluster:   s.cluster,
+               Context:   s.ctx,
+               ArvClient: arvClient,
+               AuthToken: arvadostest.AdminToken,
+               Registry:  prometheus.NewRegistry(),
        }
        // Test cases can modify s.cluster before calling
        // initialize(), and then modify private state before calling
@@ -100,12 +109,13 @@ func (s *DispatcherSuite) TearDownTest(c *check.C) {
 // a fake queue and cloud driver. The fake cloud driver injects
 // artificial errors in order to exercise a variety of code paths.
 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
-       drivers["test"] = s.stubDriver
+       Drivers["test"] = s.stubDriver
        s.disp.setupOnce.Do(s.disp.initialize)
        queue := &test.Queue{
                ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
                        return ChooseInstanceType(s.cluster, ctr)
                },
+               Logger: ctxlog.TestLogger(c),
        }
        for i := 0; i < 200; i++ {
                queue.Containers = append(queue.Containers, arvados.Container{
@@ -155,9 +165,10 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
                        stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
                case 1:
                        stubvm.CrunchRunMissing = true
+               case 2:
+                       stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)) * time.Millisecond)
                default:
                        stubvm.CrunchRunCrashRate = 0.1
-                       stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)+200) * time.Millisecond)
                }
        }
 
@@ -186,11 +197,28 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
                        c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
                }
        }
+
+       req := httptest.NewRequest("GET", "/metrics", nil)
+       req.Header.Set("Authorization", "Bearer "+s.cluster.ManagementToken)
+       resp := httptest.NewRecorder()
+       s.disp.ServeHTTP(resp, req)
+       c.Check(resp.Code, check.Equals, http.StatusOK)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Create"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="List"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Destroy"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="Create"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="List"} 0\n.*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="aborted"} 0.*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="disappeared"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="failure"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="success"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="shutdown"} [^0].*`)
+       c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="unknown"} 0\n.*`)
 }
 
 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
        s.cluster.ManagementToken = "abcdefgh"
-       drivers["test"] = s.stubDriver
+       Drivers["test"] = s.stubDriver
        s.disp.setupOnce.Do(s.disp.initialize)
        s.disp.queue = &test.Queue{}
        go s.disp.run()
@@ -212,7 +240,7 @@ func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
 
 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
        s.cluster.ManagementToken = ""
-       drivers["test"] = s.stubDriver
+       Drivers["test"] = s.stubDriver
        s.disp.setupOnce.Do(s.disp.initialize)
        s.disp.queue = &test.Queue{}
        go s.disp.run()
@@ -230,8 +258,8 @@ func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
 
 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
        s.cluster.ManagementToken = "abcdefgh"
-       s.cluster.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
-       drivers["test"] = s.stubDriver
+       s.cluster.Containers.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
+       Drivers["test"] = s.stubDriver
        s.disp.setupOnce.Do(s.disp.initialize)
        s.disp.queue = &test.Queue{}
        go s.disp.run()