3 from __future__ import absolute_import, print_function
8 import arvados.errors as arverror
13 import arvnodeman.computenode.dispatch as dispatch
14 from . import testutil
16 class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
17 def make_mocks(self, arvados_effect=None, cloud_effect=None):
18 if arvados_effect is None:
19 arvados_effect = [testutil.arvados_node_mock()]
20 self.arvados_effect = arvados_effect
21 self.timer = testutil.MockTimer()
22 self.api_client = mock.MagicMock(name='api_client')
23 self.api_client.nodes().create().execute.side_effect = arvados_effect
24 self.api_client.nodes().update().execute.side_effect = arvados_effect
25 self.cloud_client = mock.MagicMock(name='cloud_client')
26 self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1)
28 def make_actor(self, arv_node=None):
29 if not hasattr(self, 'timer'):
30 self.make_mocks(arvados_effect=[arv_node])
31 self.setup_actor = dispatch.ComputeNodeSetupActor.start(
32 self.timer, self.api_client, self.cloud_client,
33 testutil.MockSize(1), arv_node).proxy()
35 def test_creation_without_arvados_node(self):
37 self.assertEqual(self.arvados_effect[-1],
38 self.setup_actor.arvados_node.get(self.TIMEOUT))
39 self.assertTrue(self.api_client.nodes().create().execute.called)
40 self.assertEqual(self.cloud_client.create_node(),
41 self.setup_actor.cloud_node.get(self.TIMEOUT))
43 def test_creation_with_arvados_node(self):
44 self.make_actor(testutil.arvados_node_mock())
45 self.assertEqual(self.arvados_effect[-1],
46 self.setup_actor.arvados_node.get(self.TIMEOUT))
47 self.assertTrue(self.api_client.nodes().update().execute.called)
48 self.assertEqual(self.cloud_client.create_node(),
49 self.setup_actor.cloud_node.get(self.TIMEOUT))
51 def test_failed_calls_retried(self):
53 arverror.ApiError(httplib2.Response({'status': '500'}), ""),
54 testutil.arvados_node_mock(),
57 self.wait_for_assignment(self.setup_actor, 'cloud_node')
59 def test_stop_when_no_cloud_node(self):
61 arverror.ApiError(httplib2.Response({'status': '500'}), ""))
63 self.setup_actor.stop_if_no_cloud_node()
65 self.setup_actor.actor_ref.actor_stopped.wait(self.TIMEOUT))
67 def test_no_stop_when_cloud_node(self):
69 self.wait_for_assignment(self.setup_actor, 'cloud_node')
70 self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT)
71 self.assertTrue(self.stop_proxy(self.setup_actor),
72 "actor was stopped by stop_if_no_cloud_node")
74 def test_subscribe(self):
76 arverror.ApiError(httplib2.Response({'status': '500'}), ""))
78 subscriber = mock.Mock(name='subscriber_mock')
79 self.setup_actor.subscribe(subscriber)
80 self.api_client.nodes().create().execute.side_effect = [
81 testutil.arvados_node_mock()]
82 self.wait_for_assignment(self.setup_actor, 'cloud_node')
83 self.assertEqual(self.setup_actor.actor_ref.actor_urn,
84 subscriber.call_args[0][0].actor_ref.actor_urn)
86 def test_late_subscribe(self):
88 subscriber = mock.Mock(name='subscriber_mock')
89 self.wait_for_assignment(self.setup_actor, 'cloud_node')
90 self.setup_actor.subscribe(subscriber).get(self.TIMEOUT)
91 self.stop_proxy(self.setup_actor)
92 self.assertEqual(self.setup_actor.actor_ref.actor_urn,
93 subscriber.call_args[0][0].actor_ref.actor_urn)
96 class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin):
97 def make_mocks(self, cloud_node=None, arvados_node=None,
99 self.timer = testutil.MockTimer()
100 self.shutdowns = testutil.MockShutdownTimer()
101 self.shutdowns._set_state(shutdown_open, 300)
102 self.cloud_client = mock.MagicMock(name='cloud_client')
103 self.updates = mock.MagicMock(name='update_mock')
104 if cloud_node is None:
105 cloud_node = testutil.cloud_node_mock()
106 self.cloud_node = cloud_node
107 self.arvados_node = arvados_node
109 def make_actor(self):
110 if not hasattr(self, 'timer'):
112 monitor_actor = dispatch.ComputeNodeMonitorActor.start(
113 self.cloud_node, time.time(), self.shutdowns, self.timer,
114 self.updates, self.arvados_node)
115 self.shutdown_actor = self.ACTOR_CLASS.start(
116 self.timer, self.cloud_client, monitor_actor).proxy()
117 self.monitor_actor = monitor_actor.proxy()
119 def check_success_flag(self, expected, allow_msg_count=1):
120 # allow_msg_count is the number of internal messages that may
121 # need to be handled for shutdown to finish.
122 for try_num in range(1 + allow_msg_count):
123 last_flag = self.shutdown_actor.success.get(self.TIMEOUT)
124 if last_flag is expected:
127 self.fail("success flag {} is not {}".format(last_flag, expected))
130 class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin,
132 ACTOR_CLASS = dispatch.ComputeNodeShutdownActor
134 def test_easy_shutdown(self):
136 self.check_success_flag(True)
137 self.assertTrue(self.cloud_client.destroy_node.called)
139 def test_shutdown_cancelled_when_window_closes(self):
140 self.make_mocks(shutdown_open=False)
142 self.check_success_flag(False, 2)
143 self.assertFalse(self.cloud_client.destroy_node.called)
145 def test_shutdown_retries_when_cloud_fails(self):
147 self.cloud_client.destroy_node.return_value = False
149 self.assertIsNone(self.shutdown_actor.success.get(self.TIMEOUT))
150 self.cloud_client.destroy_node.return_value = True
151 self.check_success_flag(True)
153 def test_late_subscribe(self):
155 subscriber = mock.Mock(name='subscriber_mock')
156 self.shutdown_actor.subscribe(subscriber).get(self.TIMEOUT)
157 self.stop_proxy(self.shutdown_actor)
158 self.assertTrue(subscriber.called)
159 self.assertEqual(self.shutdown_actor.actor_ref.actor_urn,
160 subscriber.call_args[0][0].actor_ref.actor_urn)
163 class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
165 def make_actor(self):
166 self.driver = mock.MagicMock(name='driver_mock')
167 self.updater = dispatch.ComputeNodeUpdateActor.start(self.driver).proxy()
169 def test_node_sync(self):
171 cloud_node = testutil.cloud_node_mock()
172 arv_node = testutil.arvados_node_mock()
173 self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
174 self.driver().sync_node.assert_called_with(cloud_node, arv_node)
177 class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin,
179 def make_mocks(self, node_num):
180 self.shutdowns = testutil.MockShutdownTimer()
181 self.shutdowns._set_state(False, 300)
182 self.timer = mock.MagicMock(name='timer_mock')
183 self.updates = mock.MagicMock(name='update_mock')
184 self.cloud_mock = testutil.cloud_node_mock(node_num)
185 self.subscriber = mock.Mock(name='subscriber_mock')
187 def make_actor(self, node_num=1, arv_node=None, start_time=None):
188 if not hasattr(self, 'cloud_mock'):
189 self.make_mocks(node_num)
190 if start_time is None:
191 start_time = time.time()
192 self.node_actor = dispatch.ComputeNodeMonitorActor.start(
193 self.cloud_mock, start_time, self.shutdowns, self.timer,
194 self.updates, arv_node).proxy()
195 self.node_actor.subscribe(self.subscriber).get(self.TIMEOUT)
197 def node_state(self, *states):
198 return self.node_actor.in_state(*states).get(self.TIMEOUT)
200 def test_in_state_when_unpaired(self):
202 self.assertIsNone(self.node_state('idle', 'alloc'))
204 def test_in_state_when_pairing_stale(self):
205 self.make_actor(arv_node=testutil.arvados_node_mock(
206 job_uuid=None, age=90000))
207 self.assertIsNone(self.node_state('idle', 'alloc'))
209 def test_in_state_when_no_state_available(self):
210 self.make_actor(arv_node=testutil.arvados_node_mock(info={}))
211 self.assertIsNone(self.node_state('idle', 'alloc'))
213 def test_in_idle_state(self):
214 self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None))
215 self.assertTrue(self.node_state('idle'))
216 self.assertFalse(self.node_state('alloc'))
217 self.assertTrue(self.node_state('idle', 'alloc'))
219 def test_in_alloc_state(self):
220 self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True))
221 self.assertFalse(self.node_state('idle'))
222 self.assertTrue(self.node_state('alloc'))
223 self.assertTrue(self.node_state('idle', 'alloc'))
225 def test_init_shutdown_scheduling(self):
227 self.assertTrue(self.timer.schedule.called)
228 self.assertEqual(300, self.timer.schedule.call_args[0][0])
230 def test_shutdown_window_close_scheduling(self):
232 self.shutdowns._set_state(False, 600)
233 self.timer.schedule.reset_mock()
234 self.node_actor.consider_shutdown().get(self.TIMEOUT)
235 self.stop_proxy(self.node_actor)
236 self.assertTrue(self.timer.schedule.called)
237 self.assertEqual(600, self.timer.schedule.call_args[0][0])
238 self.assertFalse(self.subscriber.called)
240 def test_shutdown_subscription(self):
242 self.shutdowns._set_state(True, 600)
243 self.node_actor.consider_shutdown().get(self.TIMEOUT)
244 self.assertTrue(self.subscriber.called)
245 self.assertEqual(self.node_actor.actor_ref.actor_urn,
246 self.subscriber.call_args[0][0].actor_ref.actor_urn)
248 def test_shutdown_without_arvados_node(self):
250 self.shutdowns._set_state(True, 600)
251 self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
253 def test_no_shutdown_without_arvados_node_and_old_cloud_node(self):
254 self.make_actor(start_time=0)
255 self.shutdowns._set_state(True, 600)
256 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
258 def test_no_shutdown_when_window_closed(self):
259 self.make_actor(3, testutil.arvados_node_mock(3, job_uuid=None))
260 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
262 def test_no_shutdown_when_node_running_job(self):
263 self.make_actor(4, testutil.arvados_node_mock(4, job_uuid=True))
264 self.shutdowns._set_state(True, 600)
265 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
267 def test_no_shutdown_when_node_state_unknown(self):
268 self.make_actor(5, testutil.arvados_node_mock(5, info={}))
269 self.shutdowns._set_state(True, 600)
270 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
272 def test_no_shutdown_when_node_state_stale(self):
273 self.make_actor(6, testutil.arvados_node_mock(6, age=90000))
274 self.shutdowns._set_state(True, 600)
275 self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
277 def test_arvados_node_match(self):
279 arv_node = testutil.arvados_node_mock(
280 2, hostname='compute-two.zzzzz.arvadosapi.com')
281 pair_id = self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)
282 self.assertEqual(self.cloud_mock.id, pair_id)
283 self.stop_proxy(self.node_actor)
284 self.updates.sync_node.assert_called_with(self.cloud_mock, arv_node)
286 def test_arvados_node_mismatch(self):
288 arv_node = testutil.arvados_node_mock(1)
290 self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
292 def test_update_cloud_node(self):
295 self.cloud_mock.id = '1'
296 self.node_actor.update_cloud_node(self.cloud_mock)
297 current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
298 self.assertEqual([testutil.ip_address_mock(2)],
299 current_cloud.private_ips)
301 def test_missing_cloud_node_update(self):
303 self.node_actor.update_cloud_node(None)
304 current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
305 self.assertEqual([testutil.ip_address_mock(1)],
306 current_cloud.private_ips)
308 def test_update_arvados_node(self):
310 job_uuid = 'zzzzz-jjjjj-updatejobnode00'
311 new_arvados = testutil.arvados_node_mock(3, job_uuid)
312 self.node_actor.update_arvados_node(new_arvados)
313 current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
314 self.assertEqual(job_uuid, current_arvados['job_uuid'])
316 def test_missing_arvados_node_update(self):
317 self.make_actor(4, testutil.arvados_node_mock(4))
318 self.node_actor.update_arvados_node(None)
319 current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
320 self.assertEqual(testutil.ip_address_mock(4),
321 current_arvados['ip_address'])