1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
17 "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
18 "git.curoverse.com/arvados.git/sdk/go/arvados"
19 "github.com/Sirupsen/logrus"
20 "golang.org/x/crypto/ssh"
21 check "gopkg.in/check.v1"
24 var _ = check.Suite(&DispatcherSuite{})
26 type DispatcherSuite struct {
27 cluster *arvados.Cluster
28 instanceSet *test.LameInstanceSet
29 stubDriver *test.StubDriver
33 func (s *DispatcherSuite) SetUpSuite(c *check.C) {
34 if os.Getenv("ARVADOS_DEBUG") != "" {
35 logrus.StandardLogger().SetLevel(logrus.DebugLevel)
39 func (s *DispatcherSuite) SetUpTest(c *check.C) {
40 dispatchpub, _ := test.LoadTestKey(c, "test/sshkey_dispatch")
41 dispatchprivraw, err := ioutil.ReadFile("test/sshkey_dispatch")
42 c.Assert(err, check.IsNil)
44 _, hostpriv := test.LoadTestKey(c, "test/sshkey_vm")
45 s.stubDriver = &test.StubDriver{
47 AuthorizedKeys: []ssh.PublicKey{dispatchpub},
48 ErrorRateDestroy: 0.1,
51 s.cluster = &arvados.Cluster{
52 CloudVMs: arvados.CloudVMs{
54 SyncInterval: arvados.Duration(10 * time.Millisecond),
55 TimeoutIdle: arvados.Duration(30 * time.Millisecond),
56 TimeoutBooting: arvados.Duration(30 * time.Millisecond),
57 TimeoutProbe: arvados.Duration(15 * time.Millisecond),
58 TimeoutShutdown: arvados.Duration(5 * time.Millisecond),
60 Dispatch: arvados.Dispatch{
61 PrivateKey: dispatchprivraw,
62 PollInterval: arvados.Duration(5 * time.Millisecond),
63 ProbeInterval: arvados.Duration(5 * time.Millisecond),
64 StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
65 MaxProbesPerSecond: 1000,
67 InstanceTypes: arvados.InstanceTypeMap{
68 test.InstanceType(1).Name: test.InstanceType(1),
69 test.InstanceType(2).Name: test.InstanceType(2),
70 test.InstanceType(3).Name: test.InstanceType(3),
71 test.InstanceType(4).Name: test.InstanceType(4),
72 test.InstanceType(6).Name: test.InstanceType(6),
73 test.InstanceType(8).Name: test.InstanceType(8),
74 test.InstanceType(16).Name: test.InstanceType(16),
76 NodeProfiles: map[string]arvados.NodeProfile{
78 Controller: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_API_HOST")},
79 DispatchCloud: arvados.SystemServiceInstance{Listen: ":"},
83 s.disp = &dispatcher{Cluster: s.cluster}
84 // Test cases can modify s.cluster before calling
85 // initialize(), and then modify private state before calling
89 func (s *DispatcherSuite) TearDownTest(c *check.C) {
93 // DispatchToStubDriver checks that the dispatcher wires everything
94 // together effectively. It uses a real scheduler and worker pool with
95 // a fake queue and cloud driver. The fake cloud driver injects
96 // artificial errors in order to exercise a variety of code paths.
97 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
98 drivers["test"] = s.stubDriver
99 s.disp.setupOnce.Do(s.disp.initialize)
100 queue := &test.Queue{
101 ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
102 return ChooseInstanceType(s.cluster, ctr)
105 for i := 0; i < 200; i++ {
106 queue.Containers = append(queue.Containers, arvados.Container{
107 UUID: test.ContainerUUID(i + 1),
108 State: arvados.ContainerStateQueued,
109 Priority: int64(i%20 + 1),
110 RuntimeConstraints: arvados.RuntimeConstraints{
111 RAM: int64(i%3+1) << 30,
119 done := make(chan struct{})
120 waiting := map[string]struct{}{}
121 for _, ctr := range queue.Containers {
122 waiting[ctr.UUID] = struct{}{}
124 onComplete := func(uuid string) {
127 if _, ok := waiting[uuid]; !ok {
128 c.Logf("container completed twice: %s -- perhaps completed after stub instance was killed?", uuid)
130 delete(waiting, uuid)
131 if len(waiting) == 0 {
136 s.stubDriver.Queue = queue
137 s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
139 stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
140 stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
141 stubvm.CtrExit = int(rand.Uint32() & 0x3)
144 stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
146 stubvm.CrunchRunMissing = true
148 stubvm.CrunchRunCrashRate = 0.1
150 stubvm.OnComplete = onComplete
151 stubvm.OnCancel = onComplete
156 err := s.disp.CheckHealth()
157 c.Check(err, check.IsNil)
161 c.Logf("containers finished (%s), waiting for instances to shutdown and queue to clear", time.Since(start))
162 case <-time.After(10 * time.Second):
163 c.Fatalf("timed out; still waiting for %d containers: %q", len(waiting), waiting)
166 deadline := time.Now().Add(time.Second)
167 for range time.NewTicker(10 * time.Millisecond).C {
168 insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
169 c.Check(err, check.IsNil)
171 ents, _ := queue.Entries()
172 if len(ents) == 0 && len(insts) == 0 {
175 if time.Now().After(deadline) {
176 c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
181 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
182 drivers["test"] = s.stubDriver
183 s.cluster.ManagementToken = "abcdefgh"
184 for _, token := range []string{"abc", ""} {
185 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
187 req.Header.Set("Authorization", "Bearer "+token)
189 resp := httptest.NewRecorder()
190 s.disp.ServeHTTP(resp, req)
192 c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
194 c.Check(resp.Code, check.Equals, http.StatusForbidden)
199 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
200 drivers["test"] = s.stubDriver
201 s.cluster.ManagementToken = ""
202 for _, token := range []string{"abc", ""} {
203 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
205 req.Header.Set("Authorization", "Bearer "+token)
207 resp := httptest.NewRecorder()
208 s.disp.ServeHTTP(resp, req)
209 c.Check(resp.Code, check.Equals, http.StatusForbidden)
213 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
214 s.cluster.ManagementToken = "abcdefgh"
215 s.cluster.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
216 drivers["test"] = s.stubDriver
218 type instance struct {
222 LastContainerUUID string
223 ArvadosInstanceType string
224 ProviderInstanceType string
226 type instancesResponse struct {
229 getInstances := func() instancesResponse {
230 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
231 req.Header.Set("Authorization", "Bearer abcdefgh")
232 resp := httptest.NewRecorder()
233 s.disp.ServeHTTP(resp, req)
234 var sr instancesResponse
235 c.Check(resp.Code, check.Equals, http.StatusOK)
236 err := json.Unmarshal(resp.Body.Bytes(), &sr)
237 c.Check(err, check.IsNil)
242 c.Check(len(sr.Items), check.Equals, 0)
244 ch := s.disp.pool.Subscribe()
245 defer s.disp.pool.Unsubscribe(ch)
246 err := s.disp.pool.Create(test.InstanceType(1))
247 c.Check(err, check.IsNil)
251 c.Assert(len(sr.Items), check.Equals, 1)
252 c.Check(sr.Items[0].Instance, check.Matches, "stub.*")
253 c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
254 c.Check(sr.Items[0].Price, check.Equals, 0.123)
255 c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")
256 c.Check(sr.Items[0].ProviderInstanceType, check.Equals, test.InstanceType(1).ProviderType)
257 c.Check(sr.Items[0].ArvadosInstanceType, check.Equals, test.InstanceType(1).Name)