3 from __future__ import absolute_import, print_function
8 import arvados.errors as arverror
13 import arvnodeman.computenode.dispatch as dispatch
14 from . import testutil
16 class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
17 def make_mocks(self, arvados_effect=None):
18 if arvados_effect is None:
19 arvados_effect = [testutil.arvados_node_mock()]
20 self.arvados_effect = arvados_effect
21 self.timer = testutil.MockTimer()
22 self.api_client = mock.MagicMock(name='api_client')
23 self.api_client.nodes().create().execute.side_effect = arvados_effect
24 self.api_client.nodes().update().execute.side_effect = arvados_effect
25 self.cloud_client = mock.MagicMock(name='cloud_client')
26 self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1)
28 def make_actor(self, arv_node=None):
29 if not hasattr(self, 'timer'):
30 self.make_mocks(arvados_effect=[arv_node])
31 self.setup_actor = dispatch.ComputeNodeSetupActor.start(
32 self.timer, self.api_client, self.cloud_client,
33 testutil.MockSize(1), arv_node).proxy()
35 def test_creation_without_arvados_node(self):
37 self.assertEqual(self.arvados_effect[-1],
38 self.setup_actor.arvados_node.get(self.TIMEOUT))
39 self.assertTrue(self.api_client.nodes().create().execute.called)
40 self.assertEqual(self.cloud_client.create_node(),
41 self.setup_actor.cloud_node.get(self.TIMEOUT))
43 def test_creation_with_arvados_node(self):
44 self.make_actor(testutil.arvados_node_mock())
45 self.assertEqual(self.arvados_effect[-1],
46 self.setup_actor.arvados_node.get(self.TIMEOUT))
47 self.assertTrue(self.api_client.nodes().update().execute.called)
48 self.assertEqual(self.cloud_client.create_node(),
49 self.setup_actor.cloud_node.get(self.TIMEOUT))
51 def test_failed_arvados_calls_retried(self):
53 arverror.ApiError(httplib2.Response({'status': '500'}), ""),
54 testutil.arvados_node_mock(),
57 self.wait_for_assignment(self.setup_actor, 'arvados_node')
59 def test_failed_cloud_calls_retried(self):
61 self.cloud_client.create_node.side_effect = [
62 Exception("test cloud creation error"),
63 self.cloud_client.create_node.return_value,
66 self.wait_for_assignment(self.setup_actor, 'cloud_node')
68 def test_failed_post_create_retried(self):
70 self.cloud_client.post_create_node.side_effect = [
71 Exception("test cloud post-create error"), None]
73 done = self.FUTURE_CLASS()
74 self.setup_actor.subscribe(done.set)
75 done.get(self.TIMEOUT)
76 self.assertEqual(2, self.cloud_client.post_create_node.call_count)
78 def test_stop_when_no_cloud_node(self):
80 arverror.ApiError(httplib2.Response({'status': '500'}), ""))
83 self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT))
85 self.setup_actor.actor_ref.actor_stopped.wait(self.TIMEOUT))
87 def test_no_stop_when_cloud_node(self):
89 self.wait_for_assignment(self.setup_actor, 'cloud_node')
91 self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT))
92 self.assertTrue(self.stop_proxy(self.setup_actor),
93 "actor was stopped by stop_if_no_cloud_node")
95 def test_subscribe(self):
97 arverror.ApiError(httplib2.Response({'status': '500'}), ""))
99 subscriber = mock.Mock(name='subscriber_mock')
100 self.setup_actor.subscribe(subscriber)
101 self.api_client.nodes().create().execute.side_effect = [
102 testutil.arvados_node_mock()]
103 self.wait_for_assignment(self.setup_actor, 'cloud_node')
104 self.assertEqual(self.setup_actor.actor_ref.actor_urn,
105 subscriber.call_args[0][0].actor_ref.actor_urn)
107 def test_late_subscribe(self):
109 subscriber = mock.Mock(name='subscriber_mock')
110 self.wait_for_assignment(self.setup_actor, 'cloud_node')
111 self.setup_actor.subscribe(subscriber).get(self.TIMEOUT)
112 self.stop_proxy(self.setup_actor)
113 self.assertEqual(self.setup_actor.actor_ref.actor_urn,
114 subscriber.call_args[0][0].actor_ref.actor_urn)
117 class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin):
118 def make_mocks(self, cloud_node=None, arvados_node=None,
119 shutdown_open=True, node_broken=False):
120 self.timer = testutil.MockTimer()
121 self.shutdowns = testutil.MockShutdownTimer()
122 self.shutdowns._set_state(shutdown_open, 300)
123 self.cloud_client = mock.MagicMock(name='cloud_client')
124 self.cloud_client.broken.return_value = node_broken
125 self.arvados_client = mock.MagicMock(name='arvados_client')
126 self.updates = mock.MagicMock(name='update_mock')
127 if cloud_node is None:
128 cloud_node = testutil.cloud_node_mock()
129 self.cloud_node = cloud_node
130 self.arvados_node = arvados_node
132 def make_actor(self, cancellable=True, start_time=None):
133 if not hasattr(self, 'timer'):
135 if start_time is None:
136 start_time = time.time()
137 monitor_actor = dispatch.ComputeNodeMonitorActor.start(
138 self.cloud_node, start_time, self.shutdowns,
139 testutil.cloud_node_fqdn, self.timer, self.updates, self.cloud_client,
141 self.shutdown_actor = self.ACTOR_CLASS.start(
142 self.timer, self.cloud_client, self.arvados_client, monitor_actor,
144 self.monitor_actor = monitor_actor.proxy()
146 def check_success_flag(self, expected, allow_msg_count=1):
147 # allow_msg_count is the number of internal messages that may
148 # need to be handled for shutdown to finish.
149 for try_num in range(1 + allow_msg_count):
150 last_flag = self.shutdown_actor.success.get(self.TIMEOUT)
151 if last_flag is expected:
154 self.fail("success flag {} is not {}".format(last_flag, expected))
156 def test_uncancellable_shutdown(self, *mocks):
157 self.make_mocks(shutdown_open=False)
158 self.cloud_client.destroy_node.return_value = False
159 self.make_actor(cancellable=False)
160 self.check_success_flag(None, 0)
161 self.shutdowns._set_state(True, 600)
162 self.cloud_client.destroy_node.return_value = True
163 self.check_success_flag(True)
165 def test_arvados_node_cleaned_after_shutdown(self, *mocks):
166 cloud_node = testutil.cloud_node_mock(62)
167 arv_node = testutil.arvados_node_mock(62)
168 self.make_mocks(cloud_node, arv_node)
170 self.check_success_flag(True, 3)
171 update_mock = self.arvados_client.nodes().update
172 self.assertTrue(update_mock.called)
173 update_kwargs = update_mock.call_args_list[0][1]
174 self.assertEqual(arv_node['uuid'], update_kwargs.get('uuid'))
175 self.assertIn('body', update_kwargs)
176 for clear_key in ['slot_number', 'hostname', 'ip_address',
177 'first_ping_at', 'last_ping_at']:
178 self.assertIn(clear_key, update_kwargs['body'])
179 self.assertIsNone(update_kwargs['body'][clear_key])
180 self.assertTrue(update_mock().execute.called)
182 def test_arvados_node_not_cleaned_after_shutdown_cancelled(self, *mocks):
183 cloud_node = testutil.cloud_node_mock(61)
184 arv_node = testutil.arvados_node_mock(61)
185 self.make_mocks(cloud_node, arv_node, shutdown_open=False)
186 self.make_actor(cancellable=True)
187 self.check_success_flag(False, 2)
188 self.assertFalse(self.arvados_client.nodes().update.called)
191 class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin,
193 ACTOR_CLASS = dispatch.ComputeNodeShutdownActor
195 def test_easy_shutdown(self):
196 self.make_actor(start_time=0)
197 self.check_success_flag(True)
198 self.assertTrue(self.cloud_client.destroy_node.called)
200 def test_shutdown_cancelled_when_window_closes(self):
201 self.make_mocks(shutdown_open=False)
203 self.check_success_flag(False, 2)
204 self.assertFalse(self.cloud_client.destroy_node.called)
205 self.assertEqual(self.ACTOR_CLASS.WINDOW_CLOSED,
206 self.shutdown_actor.cancel_reason.get(self.TIMEOUT))
208 def test_shutdown_retries_when_cloud_fails(self):
210 self.cloud_client.destroy_node.return_value = False
211 self.make_actor(start_time=0)
212 self.assertIsNone(self.shutdown_actor.success.get(self.TIMEOUT))
213 self.cloud_client.destroy_node.return_value = True
214 self.check_success_flag(True)
216 def test_shutdown_cancelled_when_cloud_fails_on_broken_node(self):
217 self.make_mocks(node_broken=True)
218 self.cloud_client.destroy_node.return_value = False
219 self.make_actor(start_time=0)
220 self.check_success_flag(False, 2)
221 self.assertEqual(1, self.cloud_client.destroy_node.call_count)
222 self.assertEqual(self.ACTOR_CLASS.NODE_BROKEN,
223 self.shutdown_actor.cancel_reason.get(self.TIMEOUT))
225 def test_late_subscribe(self):
227 subscriber = mock.Mock(name='subscriber_mock')
228 self.shutdown_actor.subscribe(subscriber).get(self.TIMEOUT)
229 self.stop_proxy(self.shutdown_actor)
230 self.assertTrue(subscriber.called)
231 self.assertEqual(self.shutdown_actor.actor_ref.actor_urn,
232 subscriber.call_args[0][0].actor_ref.actor_urn)
235 class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
237 def make_actor(self):
238 self.driver = mock.MagicMock(name='driver_mock')
239 self.updater = dispatch.ComputeNodeUpdateActor.start(self.driver).proxy()
241 def test_node_sync(self):
243 cloud_node = testutil.cloud_node_mock()
244 arv_node = testutil.arvados_node_mock()
245 self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
246 self.driver().sync_node.assert_called_with(cloud_node, arv_node)
249 class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin,
251 def make_mocks(self, node_num):
252 self.shutdowns = testutil.MockShutdownTimer()
253 self.shutdowns._set_state(False, 300)
254 self.timer = mock.MagicMock(name='timer_mock')
255 self.updates = mock.MagicMock(name='update_mock')
256 self.cloud_mock = testutil.cloud_node_mock(node_num)
257 self.subscriber = mock.Mock(name='subscriber_mock')
258 self.cloud_client = mock.MagicMock(name='cloud_client')
259 self.cloud_client.broken.return_value = False
261 def make_actor(self, node_num=1, arv_node=None, start_time=None):
262 if not hasattr(self, 'cloud_mock'):
263 self.make_mocks(node_num)
264 if start_time is None:
265 start_time = time.time()
266 self.node_actor = dispatch.ComputeNodeMonitorActor.start(
267 self.cloud_mock, start_time, self.shutdowns,
268 testutil.cloud_node_fqdn, self.timer, self.updates, self.cloud_client,
269 arv_node, boot_fail_after=300).proxy()
270 self.node_actor.subscribe(self.subscriber).get(self.TIMEOUT)
272 def node_state(self, *states):
273 return self.node_actor.in_state(*states).get(self.TIMEOUT)
275 def test_in_state_when_unpaired(self):
277 self.assertIsNone(self.node_state('idle', 'busy'))
279 def test_in_state_when_pairing_stale(self):
280 self.make_actor(arv_node=testutil.arvados_node_mock(
281 job_uuid=None, age=90000))
282 self.assertIsNone(self.node_state('idle', 'busy'))
284 def test_in_state_when_no_state_available(self):
285 self.make_actor(arv_node=testutil.arvados_node_mock(
286 crunch_worker_state=None))
287 self.assertIsNone(self.node_state('idle', 'busy'))
289 def test_in_idle_state(self):
290 self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None))
291 self.assertTrue(self.node_state('idle'))
292 self.assertFalse(self.node_state('busy'))
293 self.assertTrue(self.node_state('idle', 'busy'))
295 def test_in_busy_state(self):
296 self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True))
297 self.assertFalse(self.node_state('idle'))
298 self.assertTrue(self.node_state('busy'))
299 self.assertTrue(self.node_state('idle', 'busy'))
301 def test_init_shutdown_scheduling(self):
303 self.assertTrue(self.timer.schedule.called)
304 self.assertEqual(300, self.timer.schedule.call_args[0][0])
306 def test_shutdown_window_close_scheduling(self):
308 self.shutdowns._set_state(False, 600)
309 self.timer.schedule.reset_mock()
310 self.node_actor.consider_shutdown().get(self.TIMEOUT)
311 self.stop_proxy(self.node_actor)
312 self.assertTrue(self.timer.schedule.called)
313 self.assertEqual(600, self.timer.schedule.call_args[0][0])
314 self.assertFalse(self.subscriber.called)
316 def test_shutdown_subscription(self):
317 self.make_actor(start_time=0)
318 self.shutdowns._set_state(True, 600)
319 self.node_actor.consider_shutdown().get(self.TIMEOUT)
320 self.assertTrue(self.subscriber.called)
321 self.assertEqual(self.node_actor.actor_ref.actor_urn,
322 self.subscriber.call_args[0][0].actor_ref.actor_urn)
324 def test_no_shutdown_booting(self):
326 self.shutdowns._set_state(True, 600)
327 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
329 def test_shutdown_without_arvados_node(self):
330 self.make_actor(start_time=0)
331 self.shutdowns._set_state(True, 600)
332 self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
334 def test_no_shutdown_missing(self):
335 arv_node = testutil.arvados_node_mock(10, job_uuid=None,
336 crunch_worker_state="down",
337 last_ping_at='1970-01-01T01:02:03.04050607Z')
338 self.make_actor(10, arv_node)
339 self.shutdowns._set_state(True, 600)
340 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
342 def test_no_shutdown_running_broken(self):
343 arv_node = testutil.arvados_node_mock(12, job_uuid=None,
344 crunch_worker_state="down")
345 self.make_actor(12, arv_node)
346 self.shutdowns._set_state(True, 600)
347 self.cloud_client.broken.return_value = True
348 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
350 def test_shutdown_missing_broken(self):
351 arv_node = testutil.arvados_node_mock(11, job_uuid=None,
352 crunch_worker_state="down",
353 last_ping_at='1970-01-01T01:02:03.04050607Z')
354 self.make_actor(11, arv_node)
355 self.shutdowns._set_state(True, 600)
356 self.cloud_client.broken.return_value = True
357 self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
359 def test_no_shutdown_when_window_closed(self):
360 self.make_actor(3, testutil.arvados_node_mock(3, job_uuid=None))
361 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
363 def test_no_shutdown_when_node_running_job(self):
364 self.make_actor(4, testutil.arvados_node_mock(4, job_uuid=True))
365 self.shutdowns._set_state(True, 600)
366 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
368 def test_no_shutdown_when_node_state_unknown(self):
369 self.make_actor(5, testutil.arvados_node_mock(
370 5, crunch_worker_state=None))
371 self.shutdowns._set_state(True, 600)
372 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
374 def test_no_shutdown_when_node_state_stale(self):
375 self.make_actor(6, testutil.arvados_node_mock(6, age=90000))
376 self.shutdowns._set_state(True, 600)
377 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
379 def test_arvados_node_match(self):
381 arv_node = testutil.arvados_node_mock(
382 2, hostname='compute-two.zzzzz.arvadosapi.com')
383 pair_id = self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)
384 self.assertEqual(self.cloud_mock.id, pair_id)
385 self.stop_proxy(self.node_actor)
386 self.updates.sync_node.assert_called_with(self.cloud_mock, arv_node)
388 def test_arvados_node_mismatch(self):
390 arv_node = testutil.arvados_node_mock(1)
392 self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
394 def test_arvados_node_mismatch_first_ping_too_early(self):
396 arv_node = testutil.arvados_node_mock(
397 4, first_ping_at='1971-03-02T14:15:16.1717282Z')
399 self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
401 def test_update_cloud_node(self):
404 self.cloud_mock.id = '1'
405 self.node_actor.update_cloud_node(self.cloud_mock)
406 current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
407 self.assertEqual([testutil.ip_address_mock(2)],
408 current_cloud.private_ips)
410 def test_missing_cloud_node_update(self):
412 self.node_actor.update_cloud_node(None)
413 current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
414 self.assertEqual([testutil.ip_address_mock(1)],
415 current_cloud.private_ips)
417 def test_update_arvados_node(self):
419 job_uuid = 'zzzzz-jjjjj-updatejobnode00'
420 new_arvados = testutil.arvados_node_mock(3, job_uuid)
421 self.node_actor.update_arvados_node(new_arvados)
422 current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
423 self.assertEqual(job_uuid, current_arvados['job_uuid'])
425 def test_missing_arvados_node_update(self):
426 self.make_actor(4, testutil.arvados_node_mock(4))
427 self.node_actor.update_arvados_node(None)
428 current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
429 self.assertEqual(testutil.ip_address_mock(4),
430 current_arvados['ip_address'])
432 def test_update_arvados_node_syncs_when_fqdn_mismatch(self):
434 self.cloud_mock.extra['testname'] = 'cloudfqdn.zzzzz.arvadosapi.com'
436 arv_node = testutil.arvados_node_mock(5)
437 self.node_actor.update_arvados_node(arv_node).get(self.TIMEOUT)
438 self.assertEqual(1, self.updates.sync_node.call_count)
440 def test_update_arvados_node_skips_sync_when_fqdn_match(self):
442 arv_node = testutil.arvados_node_mock(6)
443 self.cloud_mock.extra['testname'] ='{n[hostname]}.{n[domain]}'.format(
446 self.node_actor.update_arvados_node(arv_node).get(self.TIMEOUT)
447 self.assertEqual(0, self.updates.sync_node.call_count)