1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
18 "git.arvados.org/arvados.git/lib/config"
19 "git.arvados.org/arvados.git/lib/dispatchcloud/test"
20 "git.arvados.org/arvados.git/sdk/go/arvados"
21 "git.arvados.org/arvados.git/sdk/go/arvadostest"
22 "git.arvados.org/arvados.git/sdk/go/ctxlog"
23 "github.com/prometheus/client_golang/prometheus"
24 "golang.org/x/crypto/ssh"
25 check "gopkg.in/check.v1"
28 var _ = check.Suite(&DispatcherSuite{})
30 type DispatcherSuite struct {
32 cancel context.CancelFunc
33 cluster *arvados.Cluster
34 stubDriver *test.StubDriver
38 func (s *DispatcherSuite) SetUpTest(c *check.C) {
39 s.ctx, s.cancel = context.WithCancel(context.Background())
40 s.ctx = ctxlog.Context(s.ctx, ctxlog.TestLogger(c))
41 dispatchpub, _ := test.LoadTestKey(c, "test/sshkey_dispatch")
42 dispatchprivraw, err := ioutil.ReadFile("test/sshkey_dispatch")
43 c.Assert(err, check.IsNil)
45 _, hostpriv := test.LoadTestKey(c, "test/sshkey_vm")
46 s.stubDriver = &test.StubDriver{
48 AuthorizedKeys: []ssh.PublicKey{dispatchpub},
49 ErrorRateDestroy: 0.1,
50 MinTimeBetweenCreateCalls: time.Millisecond,
53 // We need the postgresql connection info from the integration
55 cfg, err := config.NewLoader(nil, ctxlog.FromContext(s.ctx)).Load()
56 c.Assert(err, check.IsNil)
57 testcluster, err := cfg.GetCluster("")
58 c.Assert(err, check.IsNil)
60 s.cluster = &arvados.Cluster{
61 ManagementToken: "test-management-token",
62 PostgreSQL: testcluster.PostgreSQL,
63 Containers: arvados.ContainersConfig{
64 CrunchRunCommand: "crunch-run",
65 CrunchRunArgumentsList: []string{"--foo", "--extra='args'"},
66 DispatchPrivateKey: string(dispatchprivraw),
67 StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
68 RuntimeEngine: "stub",
69 CloudVMs: arvados.CloudVMsConfig{
71 SyncInterval: arvados.Duration(10 * time.Millisecond),
72 TimeoutIdle: arvados.Duration(150 * time.Millisecond),
73 TimeoutBooting: arvados.Duration(150 * time.Millisecond),
74 TimeoutProbe: arvados.Duration(15 * time.Millisecond),
75 TimeoutShutdown: arvados.Duration(5 * time.Millisecond),
76 MaxCloudOpsPerSecond: 500,
77 PollInterval: arvados.Duration(5 * time.Millisecond),
78 ProbeInterval: arvados.Duration(5 * time.Millisecond),
79 MaxProbesPerSecond: 1000,
80 TimeoutSignal: arvados.Duration(3 * time.Millisecond),
81 TimeoutStaleRunLock: arvados.Duration(3 * time.Millisecond),
82 TimeoutTERM: arvados.Duration(20 * time.Millisecond),
83 ResourceTags: map[string]string{"testtag": "test value"},
84 TagKeyPrefix: "test:",
87 InstanceTypes: arvados.InstanceTypeMap{
88 test.InstanceType(1).Name: test.InstanceType(1),
89 test.InstanceType(2).Name: test.InstanceType(2),
90 test.InstanceType(3).Name: test.InstanceType(3),
91 test.InstanceType(4).Name: test.InstanceType(4),
92 test.InstanceType(6).Name: test.InstanceType(6),
93 test.InstanceType(8).Name: test.InstanceType(8),
94 test.InstanceType(16).Name: test.InstanceType(16),
97 arvadostest.SetServiceURL(&s.cluster.Services.DispatchCloud, "http://localhost:/")
98 arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
100 arvClient, err := arvados.NewClientFromConfig(s.cluster)
101 c.Check(err, check.IsNil)
103 s.disp = &dispatcher{
106 ArvClient: arvClient,
107 AuthToken: arvadostest.AdminToken,
108 Registry: prometheus.NewRegistry(),
110 // Test cases can modify s.cluster before calling
111 // initialize(), and then modify private state before calling
115 func (s *DispatcherSuite) TearDownTest(c *check.C) {
120 // DispatchToStubDriver checks that the dispatcher wires everything
121 // together effectively. It uses a real scheduler and worker pool with
122 // a fake queue and cloud driver. The fake cloud driver injects
123 // artificial errors in order to exercise a variety of code paths.
124 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
125 Drivers["test"] = s.stubDriver
126 s.disp.setupOnce.Do(s.disp.initialize)
127 queue := &test.Queue{
128 ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
129 return ChooseInstanceType(s.cluster, ctr)
131 Logger: ctxlog.TestLogger(c),
133 for i := 0; i < 200; i++ {
134 queue.Containers = append(queue.Containers, arvados.Container{
135 UUID: test.ContainerUUID(i + 1),
136 State: arvados.ContainerStateQueued,
137 Priority: int64(i%20 + 1),
138 RuntimeConstraints: arvados.RuntimeConstraints{
139 RAM: int64(i%3+1) << 30,
147 done := make(chan struct{})
148 waiting := map[string]struct{}{}
149 for _, ctr := range queue.Containers {
150 waiting[ctr.UUID] = struct{}{}
152 finishContainer := func(ctr arvados.Container) {
155 if _, ok := waiting[ctr.UUID]; !ok {
156 c.Errorf("container completed twice: %s", ctr.UUID)
159 delete(waiting, ctr.UUID)
160 if len(waiting) == 0 {
164 executeContainer := func(ctr arvados.Container) int {
166 return int(rand.Uint32() & 0x3)
169 s.stubDriver.Queue = queue
170 s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
172 stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
173 stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
174 stubvm.ExecuteContainer = executeContainer
175 stubvm.CrashRunningContainer = finishContainer
176 stubvm.ExtraCrunchRunArgs = "'--runtime-engine=stub' '--foo' '--extra='\\''args'\\'''"
179 stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
181 stubvm.CrunchRunMissing = true
183 stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)) * time.Millisecond)
185 stubvm.CrunchRunCrashRate = 0.1
186 stubvm.ArvMountDeadlockRate = 0.1
189 s.stubDriver.Bugf = c.Errorf
193 err := s.disp.CheckHealth()
194 c.Check(err, check.IsNil)
198 c.Logf("containers finished (%s), waiting for instances to shutdown and queue to clear", time.Since(start))
199 case <-time.After(10 * time.Second):
200 c.Fatalf("timed out; still waiting for %d containers: %q", len(waiting), waiting)
203 deadline := time.Now().Add(5 * time.Second)
204 for range time.NewTicker(10 * time.Millisecond).C {
205 insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
206 c.Check(err, check.IsNil)
208 ents, _ := queue.Entries()
209 if len(ents) == 0 && len(insts) == 0 {
212 if time.Now().After(deadline) {
213 c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
217 req := httptest.NewRequest("GET", "/metrics", nil)
218 req.Header.Set("Authorization", "Bearer "+s.cluster.ManagementToken)
219 resp := httptest.NewRecorder()
220 s.disp.ServeHTTP(resp, req)
221 c.Check(resp.Code, check.Equals, http.StatusOK)
222 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Create"} [^0].*`)
223 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="List"} [^0].*`)
224 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Destroy"} [^0].*`)
225 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="Create"} [^0].*`)
226 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="List"} 0\n.*`)
227 c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="aborted"} 0.*`)
228 c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="disappeared"} [^0].*`)
229 c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="failure"} [^0].*`)
230 c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="success"} [^0].*`)
231 c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="shutdown"} [^0].*`)
232 c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="unknown"} 0\n.*`)
233 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds{quantile="0.95"} [0-9.]*`)
234 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_count [0-9]*`)
235 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_sum [0-9.]*`)
236 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds{quantile="0.95"} [0-9.]*`)
237 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_count [0-9]*`)
238 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_sum [0-9.]*`)
239 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_count [0-9]*`)
240 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_sum [0-9.]*`)
241 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_count [0-9]*`)
242 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_sum [0-9e+.]*`)
243 c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_count{outcome="success"} [0-9]*`)
244 c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_sum{outcome="success"} [0-9e+.]*`)
245 c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_count{outcome="fail"} [0-9]*`)
246 c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_sum{outcome="fail"} [0-9e+.]*`)
249 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
250 s.cluster.ManagementToken = "abcdefgh"
251 Drivers["test"] = s.stubDriver
252 s.disp.setupOnce.Do(s.disp.initialize)
253 s.disp.queue = &test.Queue{}
256 for _, token := range []string{"abc", ""} {
257 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
259 req.Header.Set("Authorization", "Bearer "+token)
261 resp := httptest.NewRecorder()
262 s.disp.ServeHTTP(resp, req)
264 c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
266 c.Check(resp.Code, check.Equals, http.StatusForbidden)
271 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
272 s.cluster.ManagementToken = ""
273 Drivers["test"] = s.stubDriver
274 s.disp.setupOnce.Do(s.disp.initialize)
275 s.disp.queue = &test.Queue{}
278 for _, token := range []string{"abc", ""} {
279 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
281 req.Header.Set("Authorization", "Bearer "+token)
283 resp := httptest.NewRecorder()
284 s.disp.ServeHTTP(resp, req)
285 c.Check(resp.Code, check.Equals, http.StatusForbidden)
289 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
290 s.cluster.ManagementToken = "abcdefgh"
291 s.cluster.Containers.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
292 Drivers["test"] = s.stubDriver
293 s.disp.setupOnce.Do(s.disp.initialize)
294 s.disp.queue = &test.Queue{}
297 type instance struct {
299 WorkerState string `json:"worker_state"`
301 LastContainerUUID string `json:"last_container_uuid"`
302 ArvadosInstanceType string `json:"arvados_instance_type"`
303 ProviderInstanceType string `json:"provider_instance_type"`
305 type instancesResponse struct {
308 getInstances := func() instancesResponse {
309 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
310 req.Header.Set("Authorization", "Bearer abcdefgh")
311 resp := httptest.NewRecorder()
312 s.disp.ServeHTTP(resp, req)
313 var sr instancesResponse
314 c.Check(resp.Code, check.Equals, http.StatusOK)
315 err := json.Unmarshal(resp.Body.Bytes(), &sr)
316 c.Check(err, check.IsNil)
321 c.Check(len(sr.Items), check.Equals, 0)
323 ch := s.disp.pool.Subscribe()
324 defer s.disp.pool.Unsubscribe(ch)
325 ok := s.disp.pool.Create(test.InstanceType(1))
326 c.Check(ok, check.Equals, true)
329 for deadline := time.Now().Add(time.Second); time.Now().Before(deadline); {
331 if len(sr.Items) > 0 {
334 time.Sleep(time.Millisecond)
336 c.Assert(len(sr.Items), check.Equals, 1)
337 c.Check(sr.Items[0].Instance, check.Matches, "inst.*")
338 c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
339 c.Check(sr.Items[0].Price, check.Equals, 0.123)
340 c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")
341 c.Check(sr.Items[0].ProviderInstanceType, check.Equals, test.InstanceType(1).ProviderType)
342 c.Check(sr.Items[0].ArvadosInstanceType, check.Equals, test.InstanceType(1).Name)