1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
18 "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
19 "git.curoverse.com/arvados.git/sdk/go/arvados"
20 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
21 "git.curoverse.com/arvados.git/sdk/go/ctxlog"
22 "golang.org/x/crypto/ssh"
23 check "gopkg.in/check.v1"
26 var _ = check.Suite(&DispatcherSuite{})
28 type DispatcherSuite struct {
30 cancel context.CancelFunc
31 cluster *arvados.Cluster
32 stubDriver *test.StubDriver
36 func (s *DispatcherSuite) SetUpTest(c *check.C) {
37 s.ctx, s.cancel = context.WithCancel(context.Background())
38 s.ctx = ctxlog.Context(s.ctx, ctxlog.TestLogger(c))
39 dispatchpub, _ := test.LoadTestKey(c, "test/sshkey_dispatch")
40 dispatchprivraw, err := ioutil.ReadFile("test/sshkey_dispatch")
41 c.Assert(err, check.IsNil)
43 _, hostpriv := test.LoadTestKey(c, "test/sshkey_vm")
44 s.stubDriver = &test.StubDriver{
46 AuthorizedKeys: []ssh.PublicKey{dispatchpub},
47 ErrorRateDestroy: 0.1,
48 MinTimeBetweenCreateCalls: time.Millisecond,
51 s.cluster = &arvados.Cluster{
52 ManagementToken: "test-management-token",
53 Containers: arvados.ContainersConfig{
54 DispatchPrivateKey: string(dispatchprivraw),
55 StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
56 CloudVMs: arvados.CloudVMsConfig{
58 SyncInterval: arvados.Duration(10 * time.Millisecond),
59 TimeoutIdle: arvados.Duration(150 * time.Millisecond),
60 TimeoutBooting: arvados.Duration(150 * time.Millisecond),
61 TimeoutProbe: arvados.Duration(15 * time.Millisecond),
62 TimeoutShutdown: arvados.Duration(5 * time.Millisecond),
63 MaxCloudOpsPerSecond: 500,
64 PollInterval: arvados.Duration(5 * time.Millisecond),
65 ProbeInterval: arvados.Duration(5 * time.Millisecond),
66 MaxProbesPerSecond: 1000,
67 TimeoutSignal: arvados.Duration(3 * time.Millisecond),
68 TimeoutTERM: arvados.Duration(20 * time.Millisecond),
69 ResourceTags: map[string]string{"testtag": "test value"},
70 TagKeyPrefix: "test:",
73 InstanceTypes: arvados.InstanceTypeMap{
74 test.InstanceType(1).Name: test.InstanceType(1),
75 test.InstanceType(2).Name: test.InstanceType(2),
76 test.InstanceType(3).Name: test.InstanceType(3),
77 test.InstanceType(4).Name: test.InstanceType(4),
78 test.InstanceType(6).Name: test.InstanceType(6),
79 test.InstanceType(8).Name: test.InstanceType(8),
80 test.InstanceType(16).Name: test.InstanceType(16),
83 arvadostest.SetServiceURL(&s.cluster.Services.DispatchCloud, "http://localhost:/")
84 arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
86 arvClient, err := arvados.NewClientFromConfig(s.cluster)
87 c.Check(err, check.IsNil)
93 AuthToken: arvadostest.AdminToken,
95 // Test cases can modify s.cluster before calling
96 // initialize(), and then modify private state before calling
100 func (s *DispatcherSuite) TearDownTest(c *check.C) {
105 // DispatchToStubDriver checks that the dispatcher wires everything
106 // together effectively. It uses a real scheduler and worker pool with
107 // a fake queue and cloud driver. The fake cloud driver injects
108 // artificial errors in order to exercise a variety of code paths.
109 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
110 Drivers["test"] = s.stubDriver
111 s.disp.setupOnce.Do(s.disp.initialize)
112 queue := &test.Queue{
113 ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
114 return ChooseInstanceType(s.cluster, ctr)
117 for i := 0; i < 200; i++ {
118 queue.Containers = append(queue.Containers, arvados.Container{
119 UUID: test.ContainerUUID(i + 1),
120 State: arvados.ContainerStateQueued,
121 Priority: int64(i%20 + 1),
122 RuntimeConstraints: arvados.RuntimeConstraints{
123 RAM: int64(i%3+1) << 30,
131 done := make(chan struct{})
132 waiting := map[string]struct{}{}
133 for _, ctr := range queue.Containers {
134 waiting[ctr.UUID] = struct{}{}
136 finishContainer := func(ctr arvados.Container) {
139 if _, ok := waiting[ctr.UUID]; !ok {
140 c.Errorf("container completed twice: %s", ctr.UUID)
143 delete(waiting, ctr.UUID)
144 if len(waiting) == 0 {
148 executeContainer := func(ctr arvados.Container) int {
150 return int(rand.Uint32() & 0x3)
153 s.stubDriver.Queue = queue
154 s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
156 stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
157 stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
158 stubvm.ExecuteContainer = executeContainer
159 stubvm.CrashRunningContainer = finishContainer
162 stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
164 stubvm.CrunchRunMissing = true
166 stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)) * time.Millisecond)
168 stubvm.CrunchRunCrashRate = 0.1
174 err := s.disp.CheckHealth()
175 c.Check(err, check.IsNil)
179 c.Logf("containers finished (%s), waiting for instances to shutdown and queue to clear", time.Since(start))
180 case <-time.After(10 * time.Second):
181 c.Fatalf("timed out; still waiting for %d containers: %q", len(waiting), waiting)
184 deadline := time.Now().Add(5 * time.Second)
185 for range time.NewTicker(10 * time.Millisecond).C {
186 insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
187 c.Check(err, check.IsNil)
189 ents, _ := queue.Entries()
190 if len(ents) == 0 && len(insts) == 0 {
193 if time.Now().After(deadline) {
194 c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
198 req := httptest.NewRequest("GET", "/metrics", nil)
199 req.Header.Set("Authorization", "Bearer "+s.cluster.ManagementToken)
200 resp := httptest.NewRecorder()
201 s.disp.ServeHTTP(resp, req)
202 c.Check(resp.Code, check.Equals, http.StatusOK)
203 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Create"} [^0].*`)
204 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="List"} [^0].*`)
205 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="Create"} [^0].*`)
206 c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="List"} 0\n.*`)
207 c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="shutdown"} [^0].*`)
208 c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="unknown"} 0\n.*`)
211 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
212 s.cluster.ManagementToken = "abcdefgh"
213 Drivers["test"] = s.stubDriver
214 s.disp.setupOnce.Do(s.disp.initialize)
215 s.disp.queue = &test.Queue{}
218 for _, token := range []string{"abc", ""} {
219 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
221 req.Header.Set("Authorization", "Bearer "+token)
223 resp := httptest.NewRecorder()
224 s.disp.ServeHTTP(resp, req)
226 c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
228 c.Check(resp.Code, check.Equals, http.StatusForbidden)
233 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
234 s.cluster.ManagementToken = ""
235 Drivers["test"] = s.stubDriver
236 s.disp.setupOnce.Do(s.disp.initialize)
237 s.disp.queue = &test.Queue{}
240 for _, token := range []string{"abc", ""} {
241 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
243 req.Header.Set("Authorization", "Bearer "+token)
245 resp := httptest.NewRecorder()
246 s.disp.ServeHTTP(resp, req)
247 c.Check(resp.Code, check.Equals, http.StatusForbidden)
251 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
252 s.cluster.ManagementToken = "abcdefgh"
253 s.cluster.Containers.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
254 Drivers["test"] = s.stubDriver
255 s.disp.setupOnce.Do(s.disp.initialize)
256 s.disp.queue = &test.Queue{}
259 type instance struct {
261 WorkerState string `json:"worker_state"`
263 LastContainerUUID string `json:"last_container_uuid"`
264 ArvadosInstanceType string `json:"arvados_instance_type"`
265 ProviderInstanceType string `json:"provider_instance_type"`
267 type instancesResponse struct {
270 getInstances := func() instancesResponse {
271 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
272 req.Header.Set("Authorization", "Bearer abcdefgh")
273 resp := httptest.NewRecorder()
274 s.disp.ServeHTTP(resp, req)
275 var sr instancesResponse
276 c.Check(resp.Code, check.Equals, http.StatusOK)
277 err := json.Unmarshal(resp.Body.Bytes(), &sr)
278 c.Check(err, check.IsNil)
283 c.Check(len(sr.Items), check.Equals, 0)
285 ch := s.disp.pool.Subscribe()
286 defer s.disp.pool.Unsubscribe(ch)
287 ok := s.disp.pool.Create(test.InstanceType(1))
288 c.Check(ok, check.Equals, true)
291 for deadline := time.Now().Add(time.Second); time.Now().Before(deadline); {
293 if len(sr.Items) > 0 {
296 time.Sleep(time.Millisecond)
298 c.Assert(len(sr.Items), check.Equals, 1)
299 c.Check(sr.Items[0].Instance, check.Matches, "stub.*")
300 c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
301 c.Check(sr.Items[0].Price, check.Equals, 0.123)
302 c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")
303 c.Check(sr.Items[0].ProviderInstanceType, check.Equals, test.InstanceType(1).ProviderType)
304 c.Check(sr.Items[0].ArvadosInstanceType, check.Equals, test.InstanceType(1).Name)