1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
20 "git.arvados.org/arvados.git/lib/config"
21 "git.arvados.org/arvados.git/lib/dispatchcloud/test"
22 "git.arvados.org/arvados.git/sdk/go/arvados"
23 "git.arvados.org/arvados.git/sdk/go/arvadostest"
24 "git.arvados.org/arvados.git/sdk/go/ctxlog"
25 "github.com/prometheus/client_golang/prometheus"
26 "golang.org/x/crypto/ssh"
27 check "gopkg.in/check.v1"
30 var _ = check.Suite(&DispatcherSuite{})
32 type DispatcherSuite struct {
34 cancel context.CancelFunc
35 cluster *arvados.Cluster
36 stubDriver *test.StubDriver
38 error503Server *httptest.Server
41 func (s *DispatcherSuite) SetUpTest(c *check.C) {
42 s.ctx, s.cancel = context.WithCancel(context.Background())
43 s.ctx = ctxlog.Context(s.ctx, ctxlog.TestLogger(c))
44 dispatchpub, _ := test.LoadTestKey(c, "test/sshkey_dispatch")
45 dispatchprivraw, err := ioutil.ReadFile("test/sshkey_dispatch")
46 c.Assert(err, check.IsNil)
48 _, hostpriv := test.LoadTestKey(c, "test/sshkey_vm")
49 s.stubDriver = &test.StubDriver{
51 AuthorizedKeys: []ssh.PublicKey{dispatchpub},
52 ErrorRateDestroy: 0.1,
53 MinTimeBetweenCreateCalls: time.Millisecond,
56 // We need the postgresql connection info from the integration
58 cfg, err := config.NewLoader(nil, ctxlog.FromContext(s.ctx)).Load()
59 c.Assert(err, check.IsNil)
60 testcluster, err := cfg.GetCluster("")
61 c.Assert(err, check.IsNil)
63 s.cluster = &arvados.Cluster{
64 ManagementToken: "test-management-token",
65 PostgreSQL: testcluster.PostgreSQL,
66 Containers: arvados.ContainersConfig{
67 CrunchRunCommand: "crunch-run",
68 CrunchRunArgumentsList: []string{"--foo", "--extra='args'"},
69 DispatchPrivateKey: string(dispatchprivraw),
70 StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
71 RuntimeEngine: "stub",
72 MaxDispatchAttempts: 10,
73 CloudVMs: arvados.CloudVMsConfig{
75 SyncInterval: arvados.Duration(10 * time.Millisecond),
76 TimeoutIdle: arvados.Duration(150 * time.Millisecond),
77 TimeoutBooting: arvados.Duration(150 * time.Millisecond),
78 TimeoutProbe: arvados.Duration(15 * time.Millisecond),
79 TimeoutShutdown: arvados.Duration(5 * time.Millisecond),
80 MaxCloudOpsPerSecond: 500,
81 InitialQuotaEstimate: 8,
82 PollInterval: arvados.Duration(5 * time.Millisecond),
83 ProbeInterval: arvados.Duration(5 * time.Millisecond),
84 MaxProbesPerSecond: 1000,
85 TimeoutSignal: arvados.Duration(3 * time.Millisecond),
86 TimeoutStaleRunLock: arvados.Duration(3 * time.Millisecond),
87 TimeoutTERM: arvados.Duration(20 * time.Millisecond),
88 ResourceTags: map[string]string{"testtag": "test value"},
89 TagKeyPrefix: "test:",
92 InstanceTypes: arvados.InstanceTypeMap{
93 test.InstanceType(1).Name: test.InstanceType(1),
94 test.InstanceType(2).Name: test.InstanceType(2),
95 test.InstanceType(3).Name: test.InstanceType(3),
96 test.InstanceType(4).Name: test.InstanceType(4),
97 test.InstanceType(6).Name: test.InstanceType(6),
98 test.InstanceType(8).Name: test.InstanceType(8),
99 test.InstanceType(16).Name: test.InstanceType(16),
102 arvadostest.SetServiceURL(&s.cluster.Services.DispatchCloud, "http://localhost:/")
103 arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
105 arvClient, err := arvados.NewClientFromConfig(s.cluster)
106 c.Assert(err, check.IsNil)
107 // Disable auto-retry
108 arvClient.Timeout = 0
110 s.error503Server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
111 c.Logf("503 stub: returning 503")
112 w.WriteHeader(http.StatusServiceUnavailable)
114 arvClient.Client = &http.Client{
115 Transport: &http.Transport{
116 Proxy: s.arvClientProxy(c),
117 TLSClientConfig: &tls.Config{
118 InsecureSkipVerify: true}}}
120 s.disp = &dispatcher{
123 ArvClient: arvClient,
124 AuthToken: arvadostest.AdminToken,
125 Registry: prometheus.NewRegistry(),
127 // Test cases can modify s.cluster before calling
128 // initialize(), and then modify private state before calling
132 func (s *DispatcherSuite) TearDownTest(c *check.C) {
135 s.error503Server.Close()
138 // Intercept outgoing API requests for "/503" and respond HTTP
139 // 503. This lets us force (*arvados.Client)Last503() to return
141 func (s *DispatcherSuite) arvClientProxy(c *check.C) func(*http.Request) (*url.URL, error) {
142 return func(req *http.Request) (*url.URL, error) {
143 if req.URL.Path == "/503" {
144 c.Logf("arvClientProxy: proxying to 503 stub")
145 return url.Parse(s.error503Server.URL)
152 // DispatchToStubDriver checks that the dispatcher wires everything
153 // together effectively. It uses a real scheduler and worker pool with
154 // a fake queue and cloud driver. The fake cloud driver injects
155 // artificial errors in order to exercise a variety of code paths.
156 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
157 Drivers["test"] = s.stubDriver
158 s.disp.setupOnce.Do(s.disp.initialize)
159 queue := &test.Queue{
160 MaxDispatchAttempts: 5,
161 ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
162 return ChooseInstanceType(s.cluster, ctr)
164 Logger: ctxlog.TestLogger(c),
166 for i := 0; i < 200; i++ {
167 queue.Containers = append(queue.Containers, arvados.Container{
168 UUID: test.ContainerUUID(i + 1),
169 State: arvados.ContainerStateQueued,
170 Priority: int64(i%20 + 1),
171 RuntimeConstraints: arvados.RuntimeConstraints{
172 RAM: int64(i%3+1) << 30,
180 done := make(chan struct{})
181 waiting := map[string]struct{}{}
182 for _, ctr := range queue.Containers {
183 waiting[ctr.UUID] = struct{}{}
185 finishContainer := func(ctr arvados.Container) {
188 if _, ok := waiting[ctr.UUID]; !ok {
189 c.Errorf("container completed twice: %s", ctr.UUID)
192 delete(waiting, ctr.UUID)
193 if len(waiting) == 100 {
194 // trigger scheduler maxConcurrency limit
195 c.Logf("test: requesting 503 in order to trigger maxConcurrency limit")
196 s.disp.ArvClient.RequestAndDecode(nil, "GET", "503", nil, nil)
198 if len(waiting) == 0 {
202 executeContainer := func(ctr arvados.Container) int {
204 return int(rand.Uint32() & 0x3)
207 s.stubDriver.Queue = queue
208 s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
210 stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
211 stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
212 stubvm.ExecuteContainer = executeContainer
213 stubvm.CrashRunningContainer = finishContainer
214 stubvm.ExtraCrunchRunArgs = "'--runtime-engine=stub' '--foo' '--extra='\\''args'\\'''"
217 stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
219 stubvm.CrunchRunMissing = true
221 stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)) * time.Millisecond)
223 stubvm.CrunchRunCrashRate = 0.1
224 stubvm.ArvMountDeadlockRate = 0.1
227 s.stubDriver.Bugf = c.Errorf
231 err := s.disp.CheckHealth()
232 c.Check(err, check.IsNil)
234 for len(waiting) > 0 {
235 waswaiting := len(waiting)
238 // loop will end because len(waiting)==0
239 case <-time.After(5 * time.Second):
240 if len(waiting) >= waswaiting {
241 c.Fatalf("timed out; no progress in 5 s while waiting for %d containers: %q", len(waiting), waiting)
245 c.Logf("containers finished (%s), waiting for instances to shutdown and queue to clear", time.Since(start))
247 deadline := time.Now().Add(5 * time.Second)
248 for range time.NewTicker(10 * time.Millisecond).C {
249 insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
250 c.Check(err, check.IsNil)
252 ents, _ := queue.Entries()
253 if len(ents) == 0 && len(insts) == 0 {
256 if time.Now().After(deadline) {
257 c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
261 req := httptest.NewRequest("GET", "/metrics", nil)
262 req.Header.Set("Authorization", "Bearer "+s.cluster.ManagementToken)
263 resp := httptest.NewRecorder()
264 s.disp.ServeHTTP(resp, req)
265 c.Check(resp.Code, check.Equals, http.StatusOK)
266 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Create"} [^0].*`)
267 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="List"} [^0].*`)
268 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Destroy"} [^0].*`)
269 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="Create"} [^0].*`)
270 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="List"} 0\n.*`)
271 c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="aborted"} [0-9]+\n.*`)
272 c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="disappeared"} [^0].*`)
273 c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="failure"} [^0].*`)
274 c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="success"} [^0].*`)
275 c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="shutdown"} [^0].*`)
276 c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="unknown"} 0\n.*`)
277 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds{quantile="0.95"} [0-9.]*`)
278 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_count [0-9]*`)
279 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_sum [0-9.]*`)
280 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds{quantile="0.95"} [0-9.]*`)
281 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_count [0-9]*`)
282 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_sum [0-9.]*`)
283 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_count [0-9]*`)
284 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_sum [0-9.]*`)
285 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_count [0-9]*`)
286 c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_sum [0-9e+.]*`)
287 c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_count{outcome="success"} [0-9]*`)
288 c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_sum{outcome="success"} [0-9e+.]*`)
289 c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_count{outcome="fail"} [0-9]*`)
290 c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_sum{outcome="fail"} [0-9e+.]*`)
291 c.Check(resp.Body.String(), check.Matches, `(?ms).*last_503_time [1-9][0-9e+.]*`)
292 c.Check(resp.Body.String(), check.Matches, `(?ms).*max_concurrent_containers [1-9][0-9e+.]*`)
295 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
296 s.cluster.ManagementToken = "abcdefgh"
297 Drivers["test"] = s.stubDriver
298 s.disp.setupOnce.Do(s.disp.initialize)
299 s.disp.queue = &test.Queue{}
302 for _, token := range []string{"abc", ""} {
303 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
305 req.Header.Set("Authorization", "Bearer "+token)
307 resp := httptest.NewRecorder()
308 s.disp.ServeHTTP(resp, req)
310 c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
312 c.Check(resp.Code, check.Equals, http.StatusForbidden)
317 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
318 s.cluster.ManagementToken = ""
319 Drivers["test"] = s.stubDriver
320 s.disp.setupOnce.Do(s.disp.initialize)
321 s.disp.queue = &test.Queue{}
324 for _, token := range []string{"abc", ""} {
325 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
327 req.Header.Set("Authorization", "Bearer "+token)
329 resp := httptest.NewRecorder()
330 s.disp.ServeHTTP(resp, req)
331 c.Check(resp.Code, check.Equals, http.StatusForbidden)
335 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
336 s.cluster.ManagementToken = "abcdefgh"
337 s.cluster.Containers.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
338 Drivers["test"] = s.stubDriver
339 s.disp.setupOnce.Do(s.disp.initialize)
340 s.disp.queue = &test.Queue{}
343 type instance struct {
345 WorkerState string `json:"worker_state"`
347 LastContainerUUID string `json:"last_container_uuid"`
348 ArvadosInstanceType string `json:"arvados_instance_type"`
349 ProviderInstanceType string `json:"provider_instance_type"`
351 type instancesResponse struct {
354 getInstances := func() instancesResponse {
355 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
356 req.Header.Set("Authorization", "Bearer abcdefgh")
357 resp := httptest.NewRecorder()
358 s.disp.ServeHTTP(resp, req)
359 var sr instancesResponse
360 c.Check(resp.Code, check.Equals, http.StatusOK)
361 err := json.Unmarshal(resp.Body.Bytes(), &sr)
362 c.Check(err, check.IsNil)
367 c.Check(len(sr.Items), check.Equals, 0)
369 ch := s.disp.pool.Subscribe()
370 defer s.disp.pool.Unsubscribe(ch)
371 ok := s.disp.pool.Create(test.InstanceType(1))
372 c.Check(ok, check.Equals, true)
375 for deadline := time.Now().Add(time.Second); time.Now().Before(deadline); {
377 if len(sr.Items) > 0 {
380 time.Sleep(time.Millisecond)
382 c.Assert(len(sr.Items), check.Equals, 1)
383 c.Check(sr.Items[0].Instance, check.Matches, "inst.*")
384 c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
385 c.Check(sr.Items[0].Price, check.Equals, 0.123)
386 c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")
387 c.Check(sr.Items[0].ProviderInstanceType, check.Equals, test.InstanceType(1).ProviderType)
388 c.Check(sr.Items[0].ArvadosInstanceType, check.Equals, test.InstanceType(1).Name)