Merge branch '16834-stale-run-lock'
[arvados.git] / lib / dispatchcloud / dispatcher_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package dispatchcloud
6
7 import (
8         "context"
9         "encoding/json"
10         "io/ioutil"
11         "math/rand"
12         "net/http"
13         "net/http/httptest"
14         "os"
15         "sync"
16         "time"
17
18         "git.arvados.org/arvados.git/lib/dispatchcloud/test"
19         "git.arvados.org/arvados.git/sdk/go/arvados"
20         "git.arvados.org/arvados.git/sdk/go/arvadostest"
21         "git.arvados.org/arvados.git/sdk/go/ctxlog"
22         "github.com/prometheus/client_golang/prometheus"
23         "golang.org/x/crypto/ssh"
24         check "gopkg.in/check.v1"
25 )
26
27 var _ = check.Suite(&DispatcherSuite{})
28
29 type DispatcherSuite struct {
30         ctx        context.Context
31         cancel     context.CancelFunc
32         cluster    *arvados.Cluster
33         stubDriver *test.StubDriver
34         disp       *dispatcher
35 }
36
37 func (s *DispatcherSuite) SetUpTest(c *check.C) {
38         s.ctx, s.cancel = context.WithCancel(context.Background())
39         s.ctx = ctxlog.Context(s.ctx, ctxlog.TestLogger(c))
40         dispatchpub, _ := test.LoadTestKey(c, "test/sshkey_dispatch")
41         dispatchprivraw, err := ioutil.ReadFile("test/sshkey_dispatch")
42         c.Assert(err, check.IsNil)
43
44         _, hostpriv := test.LoadTestKey(c, "test/sshkey_vm")
45         s.stubDriver = &test.StubDriver{
46                 HostKey:                   hostpriv,
47                 AuthorizedKeys:            []ssh.PublicKey{dispatchpub},
48                 ErrorRateDestroy:          0.1,
49                 MinTimeBetweenCreateCalls: time.Millisecond,
50         }
51
52         s.cluster = &arvados.Cluster{
53                 ManagementToken: "test-management-token",
54                 Containers: arvados.ContainersConfig{
55                         DispatchPrivateKey: string(dispatchprivraw),
56                         StaleLockTimeout:   arvados.Duration(5 * time.Millisecond),
57                         CloudVMs: arvados.CloudVMsConfig{
58                                 Driver:               "test",
59                                 SyncInterval:         arvados.Duration(10 * time.Millisecond),
60                                 TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
61                                 TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
62                                 TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
63                                 TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
64                                 MaxCloudOpsPerSecond: 500,
65                                 PollInterval:         arvados.Duration(5 * time.Millisecond),
66                                 ProbeInterval:        arvados.Duration(5 * time.Millisecond),
67                                 MaxProbesPerSecond:   1000,
68                                 TimeoutSignal:        arvados.Duration(3 * time.Millisecond),
69                                 TimeoutStaleRunLock:  arvados.Duration(3 * time.Millisecond),
70                                 TimeoutTERM:          arvados.Duration(20 * time.Millisecond),
71                                 ResourceTags:         map[string]string{"testtag": "test value"},
72                                 TagKeyPrefix:         "test:",
73                         },
74                 },
75                 InstanceTypes: arvados.InstanceTypeMap{
76                         test.InstanceType(1).Name:  test.InstanceType(1),
77                         test.InstanceType(2).Name:  test.InstanceType(2),
78                         test.InstanceType(3).Name:  test.InstanceType(3),
79                         test.InstanceType(4).Name:  test.InstanceType(4),
80                         test.InstanceType(6).Name:  test.InstanceType(6),
81                         test.InstanceType(8).Name:  test.InstanceType(8),
82                         test.InstanceType(16).Name: test.InstanceType(16),
83                 },
84         }
85         arvadostest.SetServiceURL(&s.cluster.Services.DispatchCloud, "http://localhost:/")
86         arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
87
88         arvClient, err := arvados.NewClientFromConfig(s.cluster)
89         c.Check(err, check.IsNil)
90
91         s.disp = &dispatcher{
92                 Cluster:   s.cluster,
93                 Context:   s.ctx,
94                 ArvClient: arvClient,
95                 AuthToken: arvadostest.AdminToken,
96                 Registry:  prometheus.NewRegistry(),
97         }
98         // Test cases can modify s.cluster before calling
99         // initialize(), and then modify private state before calling
100         // go run().
101 }
102
103 func (s *DispatcherSuite) TearDownTest(c *check.C) {
104         s.cancel()
105         s.disp.Close()
106 }
107
108 // DispatchToStubDriver checks that the dispatcher wires everything
109 // together effectively. It uses a real scheduler and worker pool with
110 // a fake queue and cloud driver. The fake cloud driver injects
111 // artificial errors in order to exercise a variety of code paths.
112 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
113         Drivers["test"] = s.stubDriver
114         s.disp.setupOnce.Do(s.disp.initialize)
115         queue := &test.Queue{
116                 ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
117                         return ChooseInstanceType(s.cluster, ctr)
118                 },
119                 Logger: ctxlog.TestLogger(c),
120         }
121         for i := 0; i < 200; i++ {
122                 queue.Containers = append(queue.Containers, arvados.Container{
123                         UUID:     test.ContainerUUID(i + 1),
124                         State:    arvados.ContainerStateQueued,
125                         Priority: int64(i%20 + 1),
126                         RuntimeConstraints: arvados.RuntimeConstraints{
127                                 RAM:   int64(i%3+1) << 30,
128                                 VCPUs: i%8 + 1,
129                         },
130                 })
131         }
132         s.disp.queue = queue
133
134         var mtx sync.Mutex
135         done := make(chan struct{})
136         waiting := map[string]struct{}{}
137         for _, ctr := range queue.Containers {
138                 waiting[ctr.UUID] = struct{}{}
139         }
140         finishContainer := func(ctr arvados.Container) {
141                 mtx.Lock()
142                 defer mtx.Unlock()
143                 if _, ok := waiting[ctr.UUID]; !ok {
144                         c.Errorf("container completed twice: %s", ctr.UUID)
145                         return
146                 }
147                 delete(waiting, ctr.UUID)
148                 if len(waiting) == 0 {
149                         close(done)
150                 }
151         }
152         executeContainer := func(ctr arvados.Container) int {
153                 finishContainer(ctr)
154                 return int(rand.Uint32() & 0x3)
155         }
156         n := 0
157         s.stubDriver.Queue = queue
158         s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
159                 n++
160                 stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
161                 stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
162                 stubvm.ExecuteContainer = executeContainer
163                 stubvm.CrashRunningContainer = finishContainer
164                 switch n % 7 {
165                 case 0:
166                         stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
167                 case 1:
168                         stubvm.CrunchRunMissing = true
169                 case 2:
170                         stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)) * time.Millisecond)
171                 default:
172                         stubvm.CrunchRunCrashRate = 0.1
173                         stubvm.ArvMountDeadlockRate = 0.1
174                 }
175         }
176         s.stubDriver.Bugf = c.Errorf
177
178         start := time.Now()
179         go s.disp.run()
180         err := s.disp.CheckHealth()
181         c.Check(err, check.IsNil)
182
183         select {
184         case <-done:
185                 c.Logf("containers finished (%s), waiting for instances to shutdown and queue to clear", time.Since(start))
186         case <-time.After(10 * time.Second):
187                 c.Fatalf("timed out; still waiting for %d containers: %q", len(waiting), waiting)
188         }
189
190         deadline := time.Now().Add(5 * time.Second)
191         for range time.NewTicker(10 * time.Millisecond).C {
192                 insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
193                 c.Check(err, check.IsNil)
194                 queue.Update()
195                 ents, _ := queue.Entries()
196                 if len(ents) == 0 && len(insts) == 0 {
197                         break
198                 }
199                 if time.Now().After(deadline) {
200                         c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
201                 }
202         }
203
204         req := httptest.NewRequest("GET", "/metrics", nil)
205         req.Header.Set("Authorization", "Bearer "+s.cluster.ManagementToken)
206         resp := httptest.NewRecorder()
207         s.disp.ServeHTTP(resp, req)
208         c.Check(resp.Code, check.Equals, http.StatusOK)
209         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Create"} [^0].*`)
210         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="List"} [^0].*`)
211         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Destroy"} [^0].*`)
212         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="Create"} [^0].*`)
213         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="List"} 0\n.*`)
214         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="aborted"} 0.*`)
215         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="disappeared"} [^0].*`)
216         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="failure"} [^0].*`)
217         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="success"} [^0].*`)
218         c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="shutdown"} [^0].*`)
219         c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="unknown"} 0\n.*`)
220         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds{quantile="0.95"} [0-9.]*`)
221         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_count [0-9]*`)
222         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_sum [0-9.]*`)
223         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds{quantile="0.95"} [0-9.]*`)
224         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_count [0-9]*`)
225         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_sum [0-9.]*`)
226 }
227
228 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
229         s.cluster.ManagementToken = "abcdefgh"
230         Drivers["test"] = s.stubDriver
231         s.disp.setupOnce.Do(s.disp.initialize)
232         s.disp.queue = &test.Queue{}
233         go s.disp.run()
234
235         for _, token := range []string{"abc", ""} {
236                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
237                 if token != "" {
238                         req.Header.Set("Authorization", "Bearer "+token)
239                 }
240                 resp := httptest.NewRecorder()
241                 s.disp.ServeHTTP(resp, req)
242                 if token == "" {
243                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
244                 } else {
245                         c.Check(resp.Code, check.Equals, http.StatusForbidden)
246                 }
247         }
248 }
249
250 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
251         s.cluster.ManagementToken = ""
252         Drivers["test"] = s.stubDriver
253         s.disp.setupOnce.Do(s.disp.initialize)
254         s.disp.queue = &test.Queue{}
255         go s.disp.run()
256
257         for _, token := range []string{"abc", ""} {
258                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
259                 if token != "" {
260                         req.Header.Set("Authorization", "Bearer "+token)
261                 }
262                 resp := httptest.NewRecorder()
263                 s.disp.ServeHTTP(resp, req)
264                 c.Check(resp.Code, check.Equals, http.StatusForbidden)
265         }
266 }
267
268 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
269         s.cluster.ManagementToken = "abcdefgh"
270         s.cluster.Containers.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
271         Drivers["test"] = s.stubDriver
272         s.disp.setupOnce.Do(s.disp.initialize)
273         s.disp.queue = &test.Queue{}
274         go s.disp.run()
275
276         type instance struct {
277                 Instance             string
278                 WorkerState          string `json:"worker_state"`
279                 Price                float64
280                 LastContainerUUID    string `json:"last_container_uuid"`
281                 ArvadosInstanceType  string `json:"arvados_instance_type"`
282                 ProviderInstanceType string `json:"provider_instance_type"`
283         }
284         type instancesResponse struct {
285                 Items []instance
286         }
287         getInstances := func() instancesResponse {
288                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
289                 req.Header.Set("Authorization", "Bearer abcdefgh")
290                 resp := httptest.NewRecorder()
291                 s.disp.ServeHTTP(resp, req)
292                 var sr instancesResponse
293                 c.Check(resp.Code, check.Equals, http.StatusOK)
294                 err := json.Unmarshal(resp.Body.Bytes(), &sr)
295                 c.Check(err, check.IsNil)
296                 return sr
297         }
298
299         sr := getInstances()
300         c.Check(len(sr.Items), check.Equals, 0)
301
302         ch := s.disp.pool.Subscribe()
303         defer s.disp.pool.Unsubscribe(ch)
304         ok := s.disp.pool.Create(test.InstanceType(1))
305         c.Check(ok, check.Equals, true)
306         <-ch
307
308         for deadline := time.Now().Add(time.Second); time.Now().Before(deadline); {
309                 sr = getInstances()
310                 if len(sr.Items) > 0 {
311                         break
312                 }
313                 time.Sleep(time.Millisecond)
314         }
315         c.Assert(len(sr.Items), check.Equals, 1)
316         c.Check(sr.Items[0].Instance, check.Matches, "inst.*")
317         c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
318         c.Check(sr.Items[0].Price, check.Equals, 0.123)
319         c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")
320         c.Check(sr.Items[0].ProviderInstanceType, check.Equals, test.InstanceType(1).ProviderType)
321         c.Check(sr.Items[0].ArvadosInstanceType, check.Equals, test.InstanceType(1).Name)
322 }