3 from __future__ import absolute_import, print_function
8 import arvados.errors as arverror
14 from libcloud.common.exceptions import BaseHTTPError
16 import arvnodeman.computenode.dispatch as dispatch
17 from arvnodeman.computenode.driver import BaseComputeNodeDriver
18 from . import testutil
20 class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
21 def make_mocks(self, arvados_effect=None):
22 if arvados_effect is None:
23 arvados_effect = [testutil.arvados_node_mock()]
24 self.arvados_effect = arvados_effect
25 self.timer = testutil.MockTimer()
26 self.api_client = mock.MagicMock(name='api_client')
27 self.api_client.nodes().create().execute.side_effect = arvados_effect
28 self.api_client.nodes().update().execute.side_effect = arvados_effect
29 self.cloud_client = mock.MagicMock(name='cloud_client')
30 self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1)
31 self.cloud_client.is_cloud_exception = BaseComputeNodeDriver.is_cloud_exception
33 def make_actor(self, arv_node=None):
34 if not hasattr(self, 'timer'):
35 self.make_mocks(arvados_effect=[arv_node] if arv_node else None)
36 self.setup_actor = dispatch.ComputeNodeSetupActor.start(
37 self.timer, self.api_client, self.cloud_client,
38 testutil.MockSize(1), arv_node).proxy()
40 def assert_node_properties_updated(self, uuid=None,
41 size=testutil.MockSize(1)):
42 self.api_client.nodes().update.assert_any_call(
43 uuid=(uuid or self.arvados_effect[-1]['uuid']),
48 'price': size.price}}})
50 def test_creation_without_arvados_node(self):
52 finished = threading.Event()
53 self.setup_actor.subscribe(lambda _: finished.set())
54 self.assertEqual(self.arvados_effect[-1],
55 self.setup_actor.arvados_node.get(self.TIMEOUT))
56 assert(finished.wait(self.TIMEOUT))
57 self.assertEqual(1, self.api_client.nodes().create().execute.call_count)
58 self.assertEqual(1, self.api_client.nodes().update().execute.call_count)
59 self.assert_node_properties_updated()
60 self.assertEqual(self.cloud_client.create_node(),
61 self.setup_actor.cloud_node.get(self.TIMEOUT))
63 def test_creation_with_arvados_node(self):
64 self.make_mocks(arvados_effect=[testutil.arvados_node_mock()]*2)
65 self.make_actor(testutil.arvados_node_mock())
66 finished = threading.Event()
67 self.setup_actor.subscribe(lambda _: finished.set())
68 self.assertEqual(self.arvados_effect[-1],
69 self.setup_actor.arvados_node.get(self.TIMEOUT))
70 assert(finished.wait(self.TIMEOUT))
71 self.assert_node_properties_updated()
72 self.assertEqual(2, self.api_client.nodes().update().execute.call_count)
73 self.assertEqual(self.cloud_client.create_node(),
74 self.setup_actor.cloud_node.get(self.TIMEOUT))
76 def test_failed_arvados_calls_retried(self):
78 arverror.ApiError(httplib2.Response({'status': '500'}), ""),
79 testutil.arvados_node_mock(),
82 self.wait_for_assignment(self.setup_actor, 'arvados_node')
84 def test_failed_cloud_calls_retried(self):
86 self.cloud_client.create_node.side_effect = [
87 Exception("test cloud creation error"),
88 self.cloud_client.create_node.return_value,
91 self.wait_for_assignment(self.setup_actor, 'cloud_node')
93 def test_unknown_basehttperror_not_retried(self):
95 self.cloud_client.create_node.side_effect = [
96 BaseHTTPError(400, "Unknown"),
97 self.cloud_client.create_node.return_value,
100 finished = threading.Event()
101 self.setup_actor.subscribe(lambda _: finished.set())
102 assert(finished.wait(self.TIMEOUT))
103 self.assertEqual(0, self.cloud_client.post_create_node.call_count)
105 def test_known_basehttperror_retried(self):
107 self.cloud_client.create_node.side_effect = [
108 BaseHTTPError(400, "InstanceLimitExceeded"),
109 self.cloud_client.create_node.return_value,
112 self.wait_for_assignment(self.setup_actor, 'cloud_node')
113 self.assertEqual(1, self.cloud_client.post_create_node.call_count)
115 def test_failed_post_create_retried(self):
117 self.cloud_client.post_create_node.side_effect = [
118 Exception("test cloud post-create error"), None]
120 done = self.FUTURE_CLASS()
121 self.setup_actor.subscribe(done.set)
122 done.get(self.TIMEOUT)
123 self.assertEqual(2, self.cloud_client.post_create_node.call_count)
125 def test_stop_when_no_cloud_node(self):
127 arverror.ApiError(httplib2.Response({'status': '500'}), ""))
130 self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT))
132 self.setup_actor.actor_ref.actor_stopped.wait(self.TIMEOUT))
134 def test_no_stop_when_cloud_node(self):
136 self.wait_for_assignment(self.setup_actor, 'cloud_node')
138 self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT))
139 self.assertTrue(self.stop_proxy(self.setup_actor),
140 "actor was stopped by stop_if_no_cloud_node")
142 def test_subscribe(self):
144 arverror.ApiError(httplib2.Response({'status': '500'}), ""))
146 subscriber = mock.Mock(name='subscriber_mock')
147 self.setup_actor.subscribe(subscriber)
148 retry_resp = [testutil.arvados_node_mock()]
149 self.api_client.nodes().create().execute.side_effect = retry_resp
150 self.api_client.nodes().update().execute.side_effect = retry_resp
151 self.wait_for_assignment(self.setup_actor, 'cloud_node')
152 self.assertEqual(self.setup_actor.actor_ref.actor_urn,
153 subscriber.call_args[0][0].actor_ref.actor_urn)
155 def test_late_subscribe(self):
157 subscriber = mock.Mock(name='subscriber_mock')
158 self.wait_for_assignment(self.setup_actor, 'cloud_node')
159 self.setup_actor.subscribe(subscriber).get(self.TIMEOUT)
160 self.stop_proxy(self.setup_actor)
161 self.assertEqual(self.setup_actor.actor_ref.actor_urn,
162 subscriber.call_args[0][0].actor_ref.actor_urn)
165 class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin):
166 def make_mocks(self, cloud_node=None, arvados_node=None,
167 shutdown_open=True, node_broken=False):
168 self.timer = testutil.MockTimer()
169 self.shutdowns = testutil.MockShutdownTimer()
170 self.shutdowns._set_state(shutdown_open, 300)
171 self.cloud_client = mock.MagicMock(name='cloud_client')
172 self.cloud_client.broken.return_value = node_broken
173 self.arvados_client = mock.MagicMock(name='arvados_client')
174 self.updates = mock.MagicMock(name='update_mock')
175 if cloud_node is None:
176 cloud_node = testutil.cloud_node_mock()
177 self.cloud_node = cloud_node
178 self.arvados_node = arvados_node
180 def make_actor(self, cancellable=True, start_time=None):
181 if not hasattr(self, 'timer'):
183 if start_time is None:
184 start_time = time.time()
185 monitor_actor = dispatch.ComputeNodeMonitorActor.start(
186 self.cloud_node, start_time, self.shutdowns,
187 testutil.cloud_node_fqdn, self.timer, self.updates, self.cloud_client,
189 self.shutdown_actor = self.ACTOR_CLASS.start(
190 self.timer, self.cloud_client, self.arvados_client, monitor_actor,
192 self.monitor_actor = monitor_actor.proxy()
194 def check_success_flag(self, expected, allow_msg_count=1):
195 # allow_msg_count is the number of internal messages that may
196 # need to be handled for shutdown to finish.
197 for try_num in range(1 + allow_msg_count):
198 last_flag = self.shutdown_actor.success.get(self.TIMEOUT)
199 if last_flag is expected:
202 self.fail("success flag {} is not {}".format(last_flag, expected))
204 def test_uncancellable_shutdown(self, *mocks):
205 self.make_mocks(shutdown_open=False)
206 self.cloud_client.destroy_node.return_value = False
207 self.make_actor(cancellable=False)
208 self.check_success_flag(None, 0)
209 self.shutdowns._set_state(True, 600)
210 self.cloud_client.destroy_node.return_value = True
211 self.check_success_flag(True)
213 def test_arvados_node_cleaned_after_shutdown(self, *mocks):
214 cloud_node = testutil.cloud_node_mock(62)
215 arv_node = testutil.arvados_node_mock(62)
216 self.make_mocks(cloud_node, arv_node)
218 self.check_success_flag(True, 3)
219 update_mock = self.arvados_client.nodes().update
220 self.assertTrue(update_mock.called)
221 update_kwargs = update_mock.call_args_list[0][1]
222 self.assertEqual(arv_node['uuid'], update_kwargs.get('uuid'))
223 self.assertIn('body', update_kwargs)
224 for clear_key in ['slot_number', 'hostname', 'ip_address',
225 'first_ping_at', 'last_ping_at']:
226 self.assertIn(clear_key, update_kwargs['body'])
227 self.assertIsNone(update_kwargs['body'][clear_key])
228 self.assertTrue(update_mock().execute.called)
230 def test_arvados_node_not_cleaned_after_shutdown_cancelled(self, *mocks):
231 cloud_node = testutil.cloud_node_mock(61)
232 arv_node = testutil.arvados_node_mock(61)
233 self.make_mocks(cloud_node, arv_node, shutdown_open=False)
234 self.cloud_client.destroy_node.return_value = False
235 self.make_actor(cancellable=True)
236 self.shutdown_actor.cancel_shutdown("test")
237 self.check_success_flag(False, 2)
238 self.assertFalse(self.arvados_client.nodes().update.called)
241 class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin,
243 ACTOR_CLASS = dispatch.ComputeNodeShutdownActor
245 def test_easy_shutdown(self):
246 self.make_actor(start_time=0)
247 self.check_success_flag(True)
248 self.assertTrue(self.cloud_client.destroy_node.called)
250 def test_shutdown_retries_when_cloud_fails(self):
252 self.cloud_client.destroy_node.return_value = False
253 self.make_actor(start_time=0)
254 self.assertIsNone(self.shutdown_actor.success.get(self.TIMEOUT))
255 self.cloud_client.destroy_node.return_value = True
256 self.check_success_flag(True)
258 def test_shutdown_cancelled_when_cloud_fails_on_broken_node(self):
259 self.make_mocks(node_broken=True)
260 self.cloud_client.destroy_node.return_value = False
261 self.make_actor(start_time=0)
262 self.check_success_flag(False, 2)
263 self.assertEqual(1, self.cloud_client.destroy_node.call_count)
264 self.assertEqual(self.ACTOR_CLASS.NODE_BROKEN,
265 self.shutdown_actor.cancel_reason.get(self.TIMEOUT))
267 def test_late_subscribe(self):
269 subscriber = mock.Mock(name='subscriber_mock')
270 self.shutdown_actor.subscribe(subscriber).get(self.TIMEOUT)
271 self.stop_proxy(self.shutdown_actor)
272 self.assertTrue(subscriber.called)
273 self.assertEqual(self.shutdown_actor.actor_ref.actor_urn,
274 subscriber.call_args[0][0].actor_ref.actor_urn)
277 class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
279 def make_actor(self):
280 self.driver = mock.MagicMock(name='driver_mock')
281 self.updater = dispatch.ComputeNodeUpdateActor.start(self.driver).proxy()
283 def test_node_sync(self):
285 cloud_node = testutil.cloud_node_mock()
286 arv_node = testutil.arvados_node_mock()
287 self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
288 self.driver().sync_node.assert_called_with(cloud_node, arv_node)
291 def test_node_sync_error(self):
293 cloud_node = testutil.cloud_node_mock()
294 arv_node = testutil.arvados_node_mock()
295 self.driver().sync_node.side_effect = (IOError, Exception, True)
296 self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
297 self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
298 self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
299 self.driver().sync_node.assert_called_with(cloud_node, arv_node)
301 class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin,
303 def make_mocks(self, node_num):
304 self.shutdowns = testutil.MockShutdownTimer()
305 self.shutdowns._set_state(False, 300)
306 self.timer = mock.MagicMock(name='timer_mock')
307 self.updates = mock.MagicMock(name='update_mock')
308 self.cloud_mock = testutil.cloud_node_mock(node_num)
309 self.subscriber = mock.Mock(name='subscriber_mock')
310 self.cloud_client = mock.MagicMock(name='cloud_client')
311 self.cloud_client.broken.return_value = False
313 def make_actor(self, node_num=1, arv_node=None, start_time=None):
314 if not hasattr(self, 'cloud_mock'):
315 self.make_mocks(node_num)
316 if start_time is None:
317 start_time = time.time()
318 self.node_actor = dispatch.ComputeNodeMonitorActor.start(
319 self.cloud_mock, start_time, self.shutdowns,
320 testutil.cloud_node_fqdn, self.timer, self.updates, self.cloud_client,
321 arv_node, boot_fail_after=300).proxy()
322 self.node_actor.subscribe(self.subscriber).get(self.TIMEOUT)
324 def node_state(self, *states):
325 return self.node_actor.in_state(*states).get(self.TIMEOUT)
327 def test_in_state_when_unpaired(self):
329 self.assertTrue(self.node_state('unpaired'))
331 def test_in_state_when_pairing_stale(self):
332 self.make_actor(arv_node=testutil.arvados_node_mock(
333 job_uuid=None, age=90000))
334 self.assertTrue(self.node_state('down'))
336 def test_in_state_when_no_state_available(self):
337 self.make_actor(arv_node=testutil.arvados_node_mock(
338 crunch_worker_state=None))
339 print(self.node_actor.get_state().get())
340 self.assertTrue(self.node_state('idle'))
342 def test_in_state_when_no_state_available_old(self):
343 self.make_actor(arv_node=testutil.arvados_node_mock(
344 crunch_worker_state=None, age=90000))
345 print(self.node_actor.get_state().get())
346 self.assertTrue(self.node_state('down'))
348 def test_in_idle_state(self):
349 self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None))
350 self.assertTrue(self.node_state('idle'))
351 self.assertFalse(self.node_state('busy'))
352 self.assertTrue(self.node_state('idle', 'busy'))
354 def test_in_busy_state(self):
355 self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True))
356 self.assertFalse(self.node_state('idle'))
357 self.assertTrue(self.node_state('busy'))
358 self.assertTrue(self.node_state('idle', 'busy'))
360 def test_init_shutdown_scheduling(self):
362 self.assertTrue(self.timer.schedule.called)
363 self.assertEqual(300, self.timer.schedule.call_args[0][0])
365 def test_shutdown_window_close_scheduling(self):
367 self.shutdowns._set_state(False, 600)
368 self.timer.schedule.reset_mock()
369 self.node_actor.consider_shutdown().get(self.TIMEOUT)
370 self.stop_proxy(self.node_actor)
371 self.assertTrue(self.timer.schedule.called)
372 self.assertEqual(600, self.timer.schedule.call_args[0][0])
373 self.assertFalse(self.subscriber.called)
375 def test_shutdown_subscription(self):
376 self.make_actor(start_time=0)
377 self.shutdowns._set_state(True, 600)
378 self.node_actor.consider_shutdown().get(self.TIMEOUT)
379 self.assertTrue(self.subscriber.called)
380 self.assertEqual(self.node_actor.actor_ref.actor_urn,
381 self.subscriber.call_args[0][0].actor_ref.actor_urn)
383 def test_no_shutdown_booting(self):
385 self.shutdowns._set_state(True, 600)
386 self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT),
387 (False, "node state is ('unpaired', 'open', 'boot wait', 'idle exceeded')"))
389 def test_shutdown_without_arvados_node(self):
390 self.make_actor(start_time=0)
391 self.shutdowns._set_state(True, 600)
392 self.assertEquals((True, "node state is ('unpaired', 'open', 'boot exceeded', 'idle exceeded')"),
393 self.node_actor.shutdown_eligible().get(self.TIMEOUT))
395 def test_shutdown_missing(self):
396 arv_node = testutil.arvados_node_mock(10, job_uuid=None,
397 crunch_worker_state="down",
398 last_ping_at='1970-01-01T01:02:03.04050607Z')
399 self.make_actor(10, arv_node)
400 self.shutdowns._set_state(True, 600)
401 self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"),
402 self.node_actor.shutdown_eligible().get(self.TIMEOUT))
404 def test_shutdown_running_broken(self):
405 arv_node = testutil.arvados_node_mock(12, job_uuid=None,
406 crunch_worker_state="down")
407 self.make_actor(12, arv_node)
408 self.shutdowns._set_state(True, 600)
409 self.cloud_client.broken.return_value = True
410 self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"),
411 self.node_actor.shutdown_eligible().get(self.TIMEOUT))
413 def test_shutdown_missing_broken(self):
414 arv_node = testutil.arvados_node_mock(11, job_uuid=None,
415 crunch_worker_state="down",
416 last_ping_at='1970-01-01T01:02:03.04050607Z')
417 self.make_actor(11, arv_node)
418 self.shutdowns._set_state(True, 600)
419 self.cloud_client.broken.return_value = True
420 self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT), (True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"))
422 def test_no_shutdown_when_window_closed(self):
423 self.make_actor(3, testutil.arvados_node_mock(3, job_uuid=None))
424 self.assertEquals((False, "node state is ('idle', 'closed', 'boot wait', 'idle exceeded')"),
425 self.node_actor.shutdown_eligible().get(self.TIMEOUT))
427 def test_no_shutdown_when_node_running_job(self):
428 self.make_actor(4, testutil.arvados_node_mock(4, job_uuid=True))
429 self.shutdowns._set_state(True, 600)
430 self.assertEquals((False, "node state is ('busy', 'open', 'boot wait', 'idle exceeded')"),
431 self.node_actor.shutdown_eligible().get(self.TIMEOUT))
433 def test_shutdown_when_node_state_unknown(self):
434 self.make_actor(5, testutil.arvados_node_mock(
435 5, crunch_worker_state=None))
436 self.shutdowns._set_state(True, 600)
437 self.assertEquals((True, "node state is ('idle', 'open', 'boot wait', 'idle exceeded')"),
438 self.node_actor.shutdown_eligible().get(self.TIMEOUT))
440 def test_no_shutdown_when_node_state_stale(self):
441 self.make_actor(6, testutil.arvados_node_mock(6, age=90000))
442 self.shutdowns._set_state(True, 600)
443 self.assertEquals((False, "node state is stale"),
444 self.node_actor.shutdown_eligible().get(self.TIMEOUT))
446 def test_arvados_node_match(self):
448 arv_node = testutil.arvados_node_mock(
449 2, hostname='compute-two.zzzzz.arvadosapi.com')
450 self.cloud_client.node_id.return_value = '2'
451 pair_id = self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)
452 self.assertEqual(self.cloud_mock.id, pair_id)
453 self.stop_proxy(self.node_actor)
454 self.updates.sync_node.assert_called_with(self.cloud_mock, arv_node)
456 def test_arvados_node_mismatch(self):
458 arv_node = testutil.arvados_node_mock(1)
460 self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
462 def test_arvados_node_mismatch_first_ping_too_early(self):
464 arv_node = testutil.arvados_node_mock(
465 4, first_ping_at='1971-03-02T14:15:16.1717282Z')
467 self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
469 def test_update_cloud_node(self):
472 self.cloud_mock.id = '1'
473 self.node_actor.update_cloud_node(self.cloud_mock)
474 current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
475 self.assertEqual([testutil.ip_address_mock(2)],
476 current_cloud.private_ips)
478 def test_missing_cloud_node_update(self):
480 self.node_actor.update_cloud_node(None)
481 current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
482 self.assertEqual([testutil.ip_address_mock(1)],
483 current_cloud.private_ips)
485 def test_update_arvados_node(self):
487 job_uuid = 'zzzzz-jjjjj-updatejobnode00'
488 new_arvados = testutil.arvados_node_mock(3, job_uuid)
489 self.node_actor.update_arvados_node(new_arvados)
490 current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
491 self.assertEqual(job_uuid, current_arvados['job_uuid'])
493 def test_missing_arvados_node_update(self):
494 self.make_actor(4, testutil.arvados_node_mock(4))
495 self.node_actor.update_arvados_node(None)
496 current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
497 self.assertEqual(testutil.ip_address_mock(4),
498 current_arvados['ip_address'])
500 def test_update_arvados_node_syncs_when_fqdn_mismatch(self):
502 self.cloud_mock.extra['testname'] = 'cloudfqdn.zzzzz.arvadosapi.com'
504 arv_node = testutil.arvados_node_mock(5)
505 self.node_actor.update_arvados_node(arv_node).get(self.TIMEOUT)
506 self.assertEqual(1, self.updates.sync_node.call_count)
508 def test_update_arvados_node_skips_sync_when_fqdn_match(self):
510 arv_node = testutil.arvados_node_mock(6)
511 self.cloud_mock.extra['testname'] ='{n[hostname]}.{n[domain]}'.format(
514 self.node_actor.update_arvados_node(arv_node).get(self.TIMEOUT)
515 self.assertEqual(0, self.updates.sync_node.call_count)