Merge branch 'master' into 4523-search-index
[arvados.git] / services / nodemanager / tests / test_computenode_dispatch.py
1 #!/usr/bin/env python
2
3 from __future__ import absolute_import, print_function
4
5 import time
6 import unittest
7
8 import arvados.errors as arverror
9 import httplib2
10 import mock
11 import pykka
12
13 import arvnodeman.computenode.dispatch as dispatch
14 from . import testutil
15
16 class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
17     def make_mocks(self, arvados_effect=None):
18         if arvados_effect is None:
19             arvados_effect = [testutil.arvados_node_mock()]
20         self.arvados_effect = arvados_effect
21         self.timer = testutil.MockTimer()
22         self.api_client = mock.MagicMock(name='api_client')
23         self.api_client.nodes().create().execute.side_effect = arvados_effect
24         self.api_client.nodes().update().execute.side_effect = arvados_effect
25         self.cloud_client = mock.MagicMock(name='cloud_client')
26         self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1)
27
28     def make_actor(self, arv_node=None):
29         if not hasattr(self, 'timer'):
30             self.make_mocks(arvados_effect=[arv_node])
31         self.setup_actor = dispatch.ComputeNodeSetupActor.start(
32             self.timer, self.api_client, self.cloud_client,
33             testutil.MockSize(1), arv_node).proxy()
34
35     def test_creation_without_arvados_node(self):
36         self.make_actor()
37         self.assertEqual(self.arvados_effect[-1],
38                          self.setup_actor.arvados_node.get(self.TIMEOUT))
39         self.assertTrue(self.api_client.nodes().create().execute.called)
40         self.assertEqual(self.cloud_client.create_node(),
41                          self.setup_actor.cloud_node.get(self.TIMEOUT))
42
43     def test_creation_with_arvados_node(self):
44         self.make_actor(testutil.arvados_node_mock())
45         self.assertEqual(self.arvados_effect[-1],
46                          self.setup_actor.arvados_node.get(self.TIMEOUT))
47         self.assertTrue(self.api_client.nodes().update().execute.called)
48         self.assertEqual(self.cloud_client.create_node(),
49                          self.setup_actor.cloud_node.get(self.TIMEOUT))
50
51     def test_failed_arvados_calls_retried(self):
52         self.make_mocks([
53                 arverror.ApiError(httplib2.Response({'status': '500'}), ""),
54                 testutil.arvados_node_mock(),
55                 ])
56         self.make_actor()
57         self.wait_for_assignment(self.setup_actor, 'arvados_node')
58
59     def test_failed_cloud_calls_retried(self):
60         self.make_mocks()
61         self.cloud_client.create_node.side_effect = [
62             Exception("test cloud creation error"),
63             self.cloud_client.create_node.return_value,
64             ]
65         self.make_actor()
66         self.wait_for_assignment(self.setup_actor, 'cloud_node')
67
68     def test_failed_post_create_retried(self):
69         self.make_mocks()
70         self.cloud_client.post_create_node.side_effect = [
71             Exception("test cloud post-create error"), None]
72         self.make_actor()
73         done = self.FUTURE_CLASS()
74         self.setup_actor.subscribe(done.set)
75         done.get(self.TIMEOUT)
76         self.assertEqual(2, self.cloud_client.post_create_node.call_count)
77
78     def test_stop_when_no_cloud_node(self):
79         self.make_mocks(
80             arverror.ApiError(httplib2.Response({'status': '500'}), ""))
81         self.make_actor()
82         self.setup_actor.stop_if_no_cloud_node()
83         self.assertTrue(
84             self.setup_actor.actor_ref.actor_stopped.wait(self.TIMEOUT))
85
86     def test_no_stop_when_cloud_node(self):
87         self.make_actor()
88         self.wait_for_assignment(self.setup_actor, 'cloud_node')
89         self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT)
90         self.assertTrue(self.stop_proxy(self.setup_actor),
91                         "actor was stopped by stop_if_no_cloud_node")
92
93     def test_subscribe(self):
94         self.make_mocks(
95             arverror.ApiError(httplib2.Response({'status': '500'}), ""))
96         self.make_actor()
97         subscriber = mock.Mock(name='subscriber_mock')
98         self.setup_actor.subscribe(subscriber)
99         self.api_client.nodes().create().execute.side_effect = [
100             testutil.arvados_node_mock()]
101         self.wait_for_assignment(self.setup_actor, 'cloud_node')
102         self.assertEqual(self.setup_actor.actor_ref.actor_urn,
103                          subscriber.call_args[0][0].actor_ref.actor_urn)
104
105     def test_late_subscribe(self):
106         self.make_actor()
107         subscriber = mock.Mock(name='subscriber_mock')
108         self.wait_for_assignment(self.setup_actor, 'cloud_node')
109         self.setup_actor.subscribe(subscriber).get(self.TIMEOUT)
110         self.stop_proxy(self.setup_actor)
111         self.assertEqual(self.setup_actor.actor_ref.actor_urn,
112                          subscriber.call_args[0][0].actor_ref.actor_urn)
113
114
115 class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin):
116     def make_mocks(self, cloud_node=None, arvados_node=None,
117                    shutdown_open=True):
118         self.timer = testutil.MockTimer()
119         self.shutdowns = testutil.MockShutdownTimer()
120         self.shutdowns._set_state(shutdown_open, 300)
121         self.cloud_client = mock.MagicMock(name='cloud_client')
122         self.updates = mock.MagicMock(name='update_mock')
123         if cloud_node is None:
124             cloud_node = testutil.cloud_node_mock()
125         self.cloud_node = cloud_node
126         self.arvados_node = arvados_node
127
128     def make_actor(self, cancellable=True):
129         if not hasattr(self, 'timer'):
130             self.make_mocks()
131         monitor_actor = dispatch.ComputeNodeMonitorActor.start(
132             self.cloud_node, time.time(), self.shutdowns, self.timer,
133             self.updates, self.arvados_node)
134         self.shutdown_actor = self.ACTOR_CLASS.start(
135             self.timer, self.cloud_client, monitor_actor, cancellable).proxy()
136         self.monitor_actor = monitor_actor.proxy()
137
138     def check_success_flag(self, expected, allow_msg_count=1):
139         # allow_msg_count is the number of internal messages that may
140         # need to be handled for shutdown to finish.
141         for try_num in range(1 + allow_msg_count):
142             last_flag = self.shutdown_actor.success.get(self.TIMEOUT)
143             if last_flag is expected:
144                 break
145         else:
146             self.fail("success flag {} is not {}".format(last_flag, expected))
147
148     def test_uncancellable_shutdown(self, *mocks):
149         self.make_mocks(shutdown_open=False)
150         self.cloud_client.destroy_node.return_value = False
151         self.make_actor(cancellable=False)
152         self.check_success_flag(None, 0)
153         self.shutdowns._set_state(True, 600)
154         self.cloud_client.destroy_node.return_value = True
155         self.check_success_flag(True)
156
157
158 class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin,
159                                        unittest.TestCase):
160     ACTOR_CLASS = dispatch.ComputeNodeShutdownActor
161
162     def test_easy_shutdown(self):
163         self.make_actor()
164         self.check_success_flag(True)
165         self.assertTrue(self.cloud_client.destroy_node.called)
166
167     def test_shutdown_cancelled_when_window_closes(self):
168         self.make_mocks(shutdown_open=False)
169         self.make_actor()
170         self.check_success_flag(False, 2)
171         self.assertFalse(self.cloud_client.destroy_node.called)
172
173     def test_shutdown_retries_when_cloud_fails(self):
174         self.make_mocks()
175         self.cloud_client.destroy_node.return_value = False
176         self.make_actor()
177         self.assertIsNone(self.shutdown_actor.success.get(self.TIMEOUT))
178         self.cloud_client.destroy_node.return_value = True
179         self.check_success_flag(True)
180
181     def test_late_subscribe(self):
182         self.make_actor()
183         subscriber = mock.Mock(name='subscriber_mock')
184         self.shutdown_actor.subscribe(subscriber).get(self.TIMEOUT)
185         self.stop_proxy(self.shutdown_actor)
186         self.assertTrue(subscriber.called)
187         self.assertEqual(self.shutdown_actor.actor_ref.actor_urn,
188                          subscriber.call_args[0][0].actor_ref.actor_urn)
189
190
191 class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
192                                      unittest.TestCase):
193     def make_actor(self):
194         self.driver = mock.MagicMock(name='driver_mock')
195         self.updater = dispatch.ComputeNodeUpdateActor.start(self.driver).proxy()
196
197     def test_node_sync(self):
198         self.make_actor()
199         cloud_node = testutil.cloud_node_mock()
200         arv_node = testutil.arvados_node_mock()
201         self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
202         self.driver().sync_node.assert_called_with(cloud_node, arv_node)
203
204
205 class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin,
206                                       unittest.TestCase):
207     def make_mocks(self, node_num):
208         self.shutdowns = testutil.MockShutdownTimer()
209         self.shutdowns._set_state(False, 300)
210         self.timer = mock.MagicMock(name='timer_mock')
211         self.updates = mock.MagicMock(name='update_mock')
212         self.cloud_mock = testutil.cloud_node_mock(node_num)
213         self.subscriber = mock.Mock(name='subscriber_mock')
214
215     def make_actor(self, node_num=1, arv_node=None, start_time=None):
216         if not hasattr(self, 'cloud_mock'):
217             self.make_mocks(node_num)
218         if start_time is None:
219             start_time = time.time()
220         self.node_actor = dispatch.ComputeNodeMonitorActor.start(
221             self.cloud_mock, start_time, self.shutdowns, self.timer,
222             self.updates, arv_node).proxy()
223         self.node_actor.subscribe(self.subscriber).get(self.TIMEOUT)
224
225     def node_state(self, *states):
226         return self.node_actor.in_state(*states).get(self.TIMEOUT)
227
228     def test_in_state_when_unpaired(self):
229         self.make_actor()
230         self.assertIsNone(self.node_state('idle', 'alloc'))
231
232     def test_in_state_when_pairing_stale(self):
233         self.make_actor(arv_node=testutil.arvados_node_mock(
234                 job_uuid=None, age=90000))
235         self.assertIsNone(self.node_state('idle', 'alloc'))
236
237     def test_in_state_when_no_state_available(self):
238         self.make_actor(arv_node=testutil.arvados_node_mock(info={}))
239         self.assertIsNone(self.node_state('idle', 'alloc'))
240
241     def test_in_idle_state(self):
242         self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None))
243         self.assertTrue(self.node_state('idle'))
244         self.assertFalse(self.node_state('alloc'))
245         self.assertTrue(self.node_state('idle', 'alloc'))
246
247     def test_in_alloc_state(self):
248         self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True))
249         self.assertFalse(self.node_state('idle'))
250         self.assertTrue(self.node_state('alloc'))
251         self.assertTrue(self.node_state('idle', 'alloc'))
252
253     def test_init_shutdown_scheduling(self):
254         self.make_actor()
255         self.assertTrue(self.timer.schedule.called)
256         self.assertEqual(300, self.timer.schedule.call_args[0][0])
257
258     def test_shutdown_window_close_scheduling(self):
259         self.make_actor()
260         self.shutdowns._set_state(False, 600)
261         self.timer.schedule.reset_mock()
262         self.node_actor.consider_shutdown().get(self.TIMEOUT)
263         self.stop_proxy(self.node_actor)
264         self.assertTrue(self.timer.schedule.called)
265         self.assertEqual(600, self.timer.schedule.call_args[0][0])
266         self.assertFalse(self.subscriber.called)
267
268     def test_shutdown_subscription(self):
269         self.make_actor()
270         self.shutdowns._set_state(True, 600)
271         self.node_actor.consider_shutdown().get(self.TIMEOUT)
272         self.assertTrue(self.subscriber.called)
273         self.assertEqual(self.node_actor.actor_ref.actor_urn,
274                          self.subscriber.call_args[0][0].actor_ref.actor_urn)
275
276     def test_shutdown_without_arvados_node(self):
277         self.make_actor()
278         self.shutdowns._set_state(True, 600)
279         self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
280
281     def test_no_shutdown_without_arvados_node_and_old_cloud_node(self):
282         self.make_actor(start_time=0)
283         self.shutdowns._set_state(True, 600)
284         self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
285
286     def test_no_shutdown_when_window_closed(self):
287         self.make_actor(3, testutil.arvados_node_mock(3, job_uuid=None))
288         self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
289
290     def test_no_shutdown_when_node_running_job(self):
291         self.make_actor(4, testutil.arvados_node_mock(4, job_uuid=True))
292         self.shutdowns._set_state(True, 600)
293         self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
294
295     def test_no_shutdown_when_node_state_unknown(self):
296         self.make_actor(5, testutil.arvados_node_mock(5, info={}))
297         self.shutdowns._set_state(True, 600)
298         self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
299
300     def test_no_shutdown_when_node_state_stale(self):
301         self.make_actor(6, testutil.arvados_node_mock(6, age=90000))
302         self.shutdowns._set_state(True, 600)
303         self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
304
305     def test_arvados_node_match(self):
306         self.make_actor(2)
307         arv_node = testutil.arvados_node_mock(
308             2, hostname='compute-two.zzzzz.arvadosapi.com')
309         pair_id = self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)
310         self.assertEqual(self.cloud_mock.id, pair_id)
311         self.stop_proxy(self.node_actor)
312         self.updates.sync_node.assert_called_with(self.cloud_mock, arv_node)
313
314     def test_arvados_node_mismatch(self):
315         self.make_actor(3)
316         arv_node = testutil.arvados_node_mock(1)
317         self.assertIsNone(
318             self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
319
320     def test_update_cloud_node(self):
321         self.make_actor(1)
322         self.make_mocks(2)
323         self.cloud_mock.id = '1'
324         self.node_actor.update_cloud_node(self.cloud_mock)
325         current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
326         self.assertEqual([testutil.ip_address_mock(2)],
327                          current_cloud.private_ips)
328
329     def test_missing_cloud_node_update(self):
330         self.make_actor(1)
331         self.node_actor.update_cloud_node(None)
332         current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
333         self.assertEqual([testutil.ip_address_mock(1)],
334                          current_cloud.private_ips)
335
336     def test_update_arvados_node(self):
337         self.make_actor(3)
338         job_uuid = 'zzzzz-jjjjj-updatejobnode00'
339         new_arvados = testutil.arvados_node_mock(3, job_uuid)
340         self.node_actor.update_arvados_node(new_arvados)
341         current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
342         self.assertEqual(job_uuid, current_arvados['job_uuid'])
343
344     def test_missing_arvados_node_update(self):
345         self.make_actor(4, testutil.arvados_node_mock(4))
346         self.node_actor.update_arvados_node(None)
347         current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
348         self.assertEqual(testutil.ip_address_mock(4),
349                          current_arvados['ip_address'])