1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
12 "git.arvados.org/arvados.git/lib/cloud"
13 "git.arvados.org/arvados.git/lib/dispatchcloud/test"
14 "git.arvados.org/arvados.git/sdk/go/arvados"
15 "git.arvados.org/arvados.git/sdk/go/ctxlog"
16 "github.com/prometheus/client_golang/prometheus"
17 check "gopkg.in/check.v1"
20 const GiB arvados.ByteSize = 1 << 30
22 var _ = check.Suite(&PoolSuite{})
24 type lessChecker struct {
28 func (*lessChecker) Check(params []interface{}, names []string) (result bool, error string) {
29 return params[0].(int) < params[1].(int), ""
32 var less = &lessChecker{&check.CheckerInfo{Name: "less", Params: []string{"obtained", "expected"}}}
34 type PoolSuite struct{}
36 func (suite *PoolSuite) TestResumeAfterRestart(c *check.C) {
37 type1 := test.InstanceType(1)
38 type2 := test.InstanceType(2)
39 type3 := test.InstanceType(3)
40 waitForIdle := func(pool *Pool, notify <-chan struct{}) {
41 timeout := time.NewTimer(time.Second)
43 instances := pool.Instances()
44 sort.Slice(instances, func(i, j int) bool {
45 return strings.Compare(instances[i].ArvadosInstanceType, instances[j].ArvadosInstanceType) < 0
47 if len(instances) == 3 &&
48 instances[0].ArvadosInstanceType == type1.Name &&
49 instances[0].WorkerState == StateIdle.String() &&
50 instances[1].ArvadosInstanceType == type1.Name &&
51 instances[1].WorkerState == StateIdle.String() &&
52 instances[2].ArvadosInstanceType == type2.Name &&
53 instances[2].WorkerState == StateIdle.String() {
58 c.Logf("pool.Instances() == %#v", instances)
66 logger := ctxlog.TestLogger(c)
67 driver := &test.StubDriver{}
68 instanceSetID := cloud.InstanceSetID("test-instance-set-id")
69 is, err := driver.InstanceSet(nil, instanceSetID, nil, logger)
70 c.Assert(err, check.IsNil)
72 newExecutor := func(cloud.Instance) Executor {
74 response: map[string]stubResp{
75 "crunch-run --list": {},
81 cluster := &arvados.Cluster{
82 Containers: arvados.ContainersConfig{
83 CloudVMs: arvados.CloudVMsConfig{
84 BootProbeCommand: "true",
85 MaxProbesPerSecond: 1000,
86 ProbeInterval: arvados.Duration(time.Millisecond * 10),
87 SyncInterval: arvados.Duration(time.Millisecond * 10),
88 TagKeyPrefix: "testprefix:",
91 InstanceTypes: arvados.InstanceTypeMap{
98 pool := NewPool(logger, arvados.NewClientFromEnv(), prometheus.NewRegistry(), instanceSetID, is, newExecutor, nil, cluster)
99 notify := pool.Subscribe()
100 defer pool.Unsubscribe(notify)
104 waitForIdle(pool, notify)
105 var heldInstanceID cloud.InstanceID
106 for _, inst := range pool.Instances() {
107 if inst.ArvadosInstanceType == type2.Name {
108 heldInstanceID = cloud.InstanceID(inst.Instance)
109 pool.SetIdleBehavior(heldInstanceID, IdleBehaviorHold)
112 // Wait for the tags to save to the cloud provider
113 tagKey := cluster.Containers.CloudVMs.TagKeyPrefix + tagKeyIdleBehavior
114 deadline := time.Now().Add(time.Second)
117 defer pool.mtx.RUnlock()
118 for _, wkr := range pool.workers {
119 if wkr.instType == type2 {
120 return wkr.instance.Tags()[tagKey] == string(IdleBehaviorHold)
125 if time.Now().After(deadline) {
128 time.Sleep(time.Millisecond * 10)
132 c.Log("------- starting new pool, waiting to recover state")
134 pool2 := NewPool(logger, arvados.NewClientFromEnv(), prometheus.NewRegistry(), instanceSetID, is, newExecutor, nil, cluster)
135 notify2 := pool2.Subscribe()
136 defer pool2.Unsubscribe(notify2)
137 waitForIdle(pool2, notify2)
138 for _, inst := range pool2.Instances() {
139 if inst.ArvadosInstanceType == type2.Name {
140 c.Check(inst.Instance, check.Equals, heldInstanceID)
141 c.Check(inst.IdleBehavior, check.Equals, IdleBehaviorHold)
143 c.Check(inst.IdleBehavior, check.Equals, IdleBehaviorRun)
149 func (suite *PoolSuite) TestDrain(c *check.C) {
150 logger := ctxlog.TestLogger(c)
151 driver := test.StubDriver{}
152 instanceSet, err := driver.InstanceSet(nil, "test-instance-set-id", nil, logger)
153 c.Assert(err, check.IsNil)
155 ac := arvados.NewClientFromEnv()
157 type1 := test.InstanceType(1)
161 newExecutor: func(cloud.Instance) Executor { return &stubExecutor{} },
162 instanceSet: &throttledInstanceSet{InstanceSet: instanceSet},
163 instanceTypes: arvados.InstanceTypeMap{
167 notify := pool.Subscribe()
168 defer pool.Unsubscribe(notify)
172 // Wait for the instance to either return from its Create
173 // call, or show up in a poll.
174 suite.wait(c, pool, notify, func() bool {
176 defer pool.mtx.RUnlock()
177 return len(pool.workers) == 1
182 idleBehavior IdleBehavior
185 {StateIdle, IdleBehaviorHold, false},
186 {StateIdle, IdleBehaviorDrain, false},
187 {StateIdle, IdleBehaviorRun, true},
190 for _, test := range tests {
191 for _, wkr := range pool.workers {
192 wkr.state = test.state
193 wkr.idleBehavior = test.idleBehavior
196 // Try to start a container
197 started := pool.StartContainer(type1, arvados.Container{UUID: "testcontainer"})
198 c.Check(started, check.Equals, test.result)
202 func (suite *PoolSuite) TestNodeCreateThrottle(c *check.C) {
203 logger := ctxlog.TestLogger(c)
204 driver := test.StubDriver{HoldCloudOps: true}
205 instanceSet, err := driver.InstanceSet(nil, "test-instance-set-id", nil, logger)
206 c.Assert(err, check.IsNil)
208 type1 := test.InstanceType(1)
211 instanceSet: &throttledInstanceSet{InstanceSet: instanceSet},
212 maxConcurrentInstanceCreateOps: 1,
213 instanceTypes: arvados.InstanceTypeMap{
218 c.Check(pool.Unallocated()[type1], check.Equals, 0)
219 res := pool.Create(type1)
220 c.Check(pool.Unallocated()[type1], check.Equals, 1)
221 c.Check(res, check.Equals, true)
223 res = pool.Create(type1)
224 c.Check(pool.Unallocated()[type1], check.Equals, 1)
225 c.Check(res, check.Equals, false)
227 pool.instanceSet.throttleCreate.err = nil
228 pool.maxConcurrentInstanceCreateOps = 2
230 res = pool.Create(type1)
231 c.Check(pool.Unallocated()[type1], check.Equals, 2)
232 c.Check(res, check.Equals, true)
234 pool.instanceSet.throttleCreate.err = nil
235 pool.maxConcurrentInstanceCreateOps = 0
237 res = pool.Create(type1)
238 c.Check(pool.Unallocated()[type1], check.Equals, 3)
239 c.Check(res, check.Equals, true)
242 func (suite *PoolSuite) TestCreateUnallocShutdown(c *check.C) {
243 logger := ctxlog.TestLogger(c)
244 driver := test.StubDriver{HoldCloudOps: true}
245 instanceSet, err := driver.InstanceSet(nil, "test-instance-set-id", nil, logger)
246 c.Assert(err, check.IsNil)
248 type1 := arvados.InstanceType{Name: "a1s", ProviderType: "a1.small", VCPUs: 1, RAM: 1 * GiB, Price: .01}
249 type2 := arvados.InstanceType{Name: "a2m", ProviderType: "a2.medium", VCPUs: 2, RAM: 2 * GiB, Price: .02}
250 type3 := arvados.InstanceType{Name: "a2l", ProviderType: "a2.large", VCPUs: 4, RAM: 4 * GiB, Price: .04}
253 newExecutor: func(cloud.Instance) Executor { return &stubExecutor{} },
254 instanceSet: &throttledInstanceSet{InstanceSet: instanceSet},
255 instanceTypes: arvados.InstanceTypeMap{
261 notify := pool.Subscribe()
262 defer pool.Unsubscribe(notify)
263 notify2 := pool.Subscribe()
264 defer pool.Unsubscribe(notify2)
266 c.Check(pool.Unallocated()[type1], check.Equals, 0)
267 c.Check(pool.Unallocated()[type2], check.Equals, 0)
268 c.Check(pool.Unallocated()[type3], check.Equals, 0)
273 c.Check(pool.Unallocated()[type1], check.Equals, 1)
274 c.Check(pool.Unallocated()[type2], check.Equals, 2)
275 c.Check(pool.Unallocated()[type3], check.Equals, 1)
277 // Unblock the pending Create calls.
278 go driver.ReleaseCloudOps(4)
280 // Wait for each instance to either return from its Create
281 // call, or show up in a poll.
282 suite.wait(c, pool, notify, func() bool {
284 defer pool.mtx.RUnlock()
285 return len(pool.workers) == 4
288 // Place type3 node on admin-hold
289 ivs := suite.instancesByType(pool, type3)
290 c.Assert(ivs, check.HasLen, 1)
291 type3instanceID := ivs[0].Instance
292 err = pool.SetIdleBehavior(type3instanceID, IdleBehaviorHold)
293 c.Check(err, check.IsNil)
295 // Check admin-hold behavior: refuse to shutdown, and don't
296 // report as Unallocated ("available now or soon").
297 c.Check(pool.Shutdown(type3), check.Equals, false)
298 suite.wait(c, pool, notify, func() bool {
299 return pool.Unallocated()[type3] == 0
301 c.Check(suite.instancesByType(pool, type3), check.HasLen, 1)
303 // Shutdown both type2 nodes
304 c.Check(pool.Shutdown(type2), check.Equals, true)
305 suite.wait(c, pool, notify, func() bool {
306 return pool.Unallocated()[type1] == 1 && pool.Unallocated()[type2] == 1
308 c.Check(pool.Shutdown(type2), check.Equals, true)
309 suite.wait(c, pool, notify, func() bool {
310 return pool.Unallocated()[type1] == 1 && pool.Unallocated()[type2] == 0
312 c.Check(pool.Shutdown(type2), check.Equals, false)
314 // Consume any waiting notifications to ensure the
315 // next one we get is from Shutdown.
324 // Shutdown type1 node
325 c.Check(pool.Shutdown(type1), check.Equals, true)
326 suite.wait(c, pool, notify, func() bool {
327 return pool.Unallocated()[type1] == 0 && pool.Unallocated()[type2] == 0 && pool.Unallocated()[type3] == 0
331 case <-time.After(time.Second):
332 c.Error("notify did not receive")
335 // Put type3 node back in service.
336 err = pool.SetIdleBehavior(type3instanceID, IdleBehaviorRun)
337 c.Check(err, check.IsNil)
338 suite.wait(c, pool, notify, func() bool {
339 return pool.Unallocated()[type3] == 1
342 // Check admin-drain behavior: shut down right away, and don't
343 // report as Unallocated.
344 err = pool.SetIdleBehavior(type3instanceID, IdleBehaviorDrain)
345 c.Check(err, check.IsNil)
346 suite.wait(c, pool, notify, func() bool {
347 return pool.Unallocated()[type3] == 0
349 suite.wait(c, pool, notify, func() bool {
350 ivs := suite.instancesByType(pool, type3)
351 return len(ivs) == 1 && ivs[0].WorkerState == StateShutdown.String()
354 // Unblock all pending Destroy calls. Pool calls Destroy again
355 // if a node still appears in the provider list after a
356 // previous attempt, so there might be more than 4 Destroy
358 go driver.ReleaseCloudOps(4444)
360 // Sync until all instances disappear from the provider list.
361 suite.wait(c, pool, notify, func() bool {
362 pool.getInstancesAndSync()
363 return len(pool.Instances()) == 0
367 func (suite *PoolSuite) instancesByType(pool *Pool, it arvados.InstanceType) []InstanceView {
368 var ivs []InstanceView
369 for _, iv := range pool.Instances() {
370 if iv.ArvadosInstanceType == it.Name {
371 ivs = append(ivs, iv)
377 func (suite *PoolSuite) wait(c *check.C, pool *Pool, notify <-chan struct{}, ready func() bool) {
378 timeout := time.NewTimer(time.Second).C
387 c.Check(ready(), check.Equals, true)