services/nodemanager/tests/test_computenode.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import absolute_import, print_function
   4
   5 import time
   6 import unittest
   7
   8 import arvados.errors as arverror
   9 import httplib2
  10 import mock
  11 import pykka
  12
  13 import arvnodeman.computenode as cnode
  14 from . import testutil
  15
  16 class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
  17     def make_mocks(self, arvados_effect=None, cloud_effect=None):
  18         if arvados_effect is None:
  19             arvados_effect = [testutil.arvados_node_mock()]
  20         self.arvados_effect = arvados_effect
  21         self.timer = testutil.MockTimer()
  22         self.api_client = mock.MagicMock(name='api_client')
  23         self.api_client.nodes().create().execute.side_effect = arvados_effect
  24         self.api_client.nodes().update().execute.side_effect = arvados_effect
  25         self.cloud_client = mock.MagicMock(name='cloud_client')
  26         self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1)
  27
  28     def make_actor(self, arv_node=None):
  29         if not hasattr(self, 'timer'):
  30             self.make_mocks(arvados_effect=[arv_node])
  31         self.setup_actor = cnode.ComputeNodeSetupActor.start(
  32             self.timer, self.api_client, self.cloud_client,
  33             testutil.MockSize(1), arv_node).proxy()
  34
  35     def test_creation_without_arvados_node(self):
  36         self.make_actor()
  37         self.assertEqual(self.arvados_effect[-1],
  38                          self.setup_actor.arvados_node.get(self.TIMEOUT))
  39         self.assertTrue(self.api_client.nodes().create().execute.called)
  40         self.assertEqual(self.cloud_client.create_node(),
  41                          self.setup_actor.cloud_node.get(self.TIMEOUT))
  42
  43     def test_creation_with_arvados_node(self):
  44         self.make_actor(testutil.arvados_node_mock())
  45         self.assertEqual(self.arvados_effect[-1],
  46                          self.setup_actor.arvados_node.get(self.TIMEOUT))
  47         self.assertTrue(self.api_client.nodes().update().execute.called)
  48         self.assertEqual(self.cloud_client.create_node(),
  49                          self.setup_actor.cloud_node.get(self.TIMEOUT))
  50
  51     def test_failed_calls_retried(self):
  52         self.make_mocks([
  53                 arverror.ApiError(httplib2.Response({'status': '500'}), ""),
  54                 testutil.arvados_node_mock(),
  55                 ])
  56         self.make_actor()
  57         self.wait_for_assignment(self.setup_actor, 'cloud_node')
  58
  59     def test_stop_when_no_cloud_node(self):
  60         self.make_mocks(
  61             arverror.ApiError(httplib2.Response({'status': '500'}), ""))
  62         self.make_actor()
  63         self.setup_actor.stop_if_no_cloud_node()
  64         self.assertTrue(
  65             self.setup_actor.actor_ref.actor_stopped.wait(self.TIMEOUT))
  66
  67     def test_no_stop_when_cloud_node(self):
  68         self.make_actor()
  69         self.wait_for_assignment(self.setup_actor, 'cloud_node')
  70         self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT)
  71         self.assertTrue(self.stop_proxy(self.setup_actor),
  72                         "actor was stopped by stop_if_no_cloud_node")
  73
  74     def test_subscribe(self):
  75         self.make_mocks(
  76             arverror.ApiError(httplib2.Response({'status': '500'}), ""))
  77         self.make_actor()
  78         subscriber = mock.Mock(name='subscriber_mock')
  79         self.setup_actor.subscribe(subscriber)
  80         self.api_client.nodes().create().execute.side_effect = [
  81             testutil.arvados_node_mock()]
  82         self.wait_for_assignment(self.setup_actor, 'cloud_node')
  83         self.assertEqual(self.setup_actor.actor_ref.actor_urn,
  84                          subscriber.call_args[0][0].actor_ref.actor_urn)
  85
  86     def test_late_subscribe(self):
  87         self.make_actor()
  88         subscriber = mock.Mock(name='subscriber_mock')
  89         self.wait_for_assignment(self.setup_actor, 'cloud_node')
  90         self.setup_actor.subscribe(subscriber).get(self.TIMEOUT)
  91         self.stop_proxy(self.setup_actor)
  92         self.assertEqual(self.setup_actor.actor_ref.actor_urn,
  93                          subscriber.call_args[0][0].actor_ref.actor_urn)
  94
  95
  96 class ComputeNodeShutdownActorTestCase(testutil.ActorTestMixin,
  97                                        unittest.TestCase):
  98     def make_mocks(self, cloud_node=None):
  99         self.timer = testutil.MockTimer()
 100         self.cloud_client = mock.MagicMock(name='cloud_client')
 101         if cloud_node is None:
 102             cloud_node = testutil.cloud_node_mock()
 103         self.cloud_node = cloud_node
 104
 105     def make_actor(self, arv_node=None):
 106         if not hasattr(self, 'timer'):
 107             self.make_mocks()
 108         self.shutdown_actor = cnode.ComputeNodeShutdownActor.start(
 109             self.timer, self.cloud_client, self.cloud_node).proxy()
 110
 111     def test_easy_shutdown(self):
 112         self.make_actor()
 113         self.shutdown_actor.cloud_node.get(self.TIMEOUT)
 114         self.stop_proxy(self.shutdown_actor)
 115         self.assertTrue(self.cloud_client.destroy_node.called)
 116
 117     def test_late_subscribe(self):
 118         self.make_actor()
 119         subscriber = mock.Mock(name='subscriber_mock')
 120         self.shutdown_actor.subscribe(subscriber).get(self.TIMEOUT)
 121         self.stop_proxy(self.shutdown_actor)
 122         self.assertEqual(self.shutdown_actor.actor_ref.actor_urn,
 123                          subscriber.call_args[0][0].actor_ref.actor_urn)
 124
 125
 126 class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
 127                                      unittest.TestCase):
 128     def make_actor(self):
 129         self.driver = mock.MagicMock(name='driver_mock')
 130         self.updater = cnode.ComputeNodeUpdateActor.start(self.driver).proxy()
 131
 132     def test_node_sync(self):
 133         self.make_actor()
 134         cloud_node = testutil.cloud_node_mock()
 135         arv_node = testutil.arvados_node_mock()
 136         self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
 137         self.driver().sync_node.assert_called_with(cloud_node, arv_node)
 138
 139
 140 @mock.patch('time.time', return_value=1)
 141 class ShutdownTimerTestCase(unittest.TestCase):
 142     def test_two_length_window(self, time_mock):
 143         timer = cnode.ShutdownTimer(time_mock.return_value, [8, 2])
 144         self.assertEqual(481, timer.next_opening())
 145         self.assertFalse(timer.window_open())
 146         time_mock.return_value += 500
 147         self.assertEqual(1081, timer.next_opening())
 148         self.assertTrue(timer.window_open())
 149         time_mock.return_value += 200
 150         self.assertEqual(1081, timer.next_opening())
 151         self.assertFalse(timer.window_open())
 152
 153     def test_three_length_window(self, time_mock):
 154         timer = cnode.ShutdownTimer(time_mock.return_value, [6, 3, 1])
 155         self.assertEqual(361, timer.next_opening())
 156         self.assertFalse(timer.window_open())
 157         time_mock.return_value += 400
 158         self.assertEqual(961, timer.next_opening())
 159         self.assertTrue(timer.window_open())
 160         time_mock.return_value += 200
 161         self.assertEqual(961, timer.next_opening())
 162         self.assertFalse(timer.window_open())
 163
 164
 165 class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin,
 166                                       unittest.TestCase):
 167     class MockShutdownTimer(object):
 168         def _set_state(self, is_open, next_opening):
 169             self.window_open = lambda: is_open
 170             self.next_opening = lambda: next_opening
 171
 172
 173     def make_mocks(self, node_num):
 174         self.shutdowns = self.MockShutdownTimer()
 175         self.shutdowns._set_state(False, 300)
 176         self.timer = mock.MagicMock(name='timer_mock')
 177         self.updates = mock.MagicMock(name='update_mock')
 178         self.cloud_mock = testutil.cloud_node_mock(node_num)
 179         self.subscriber = mock.Mock(name='subscriber_mock')
 180
 181     def make_actor(self, node_num=1, arv_node=None, start_time=None):
 182         if not hasattr(self, 'cloud_mock'):
 183             self.make_mocks(node_num)
 184         if start_time is None:
 185             start_time = time.time()
 186         self.node_actor = cnode.ComputeNodeMonitorActor.start(
 187             self.cloud_mock, start_time, self.shutdowns, self.timer,
 188             self.updates, arv_node).proxy()
 189         self.node_actor.subscribe(self.subscriber).get(self.TIMEOUT)
 190
 191     def node_state(self, *states):
 192         return self.node_actor.in_state(*states).get(self.TIMEOUT)
 193
 194     def test_in_state_when_unpaired(self):
 195         self.make_actor()
 196         self.assertIsNone(self.node_state('idle', 'alloc'))
 197
 198     def test_in_state_when_pairing_stale(self):
 199         self.make_actor(arv_node=testutil.arvados_node_mock(
 200                 job_uuid=None, age=90000))
 201         self.assertIsNone(self.node_state('idle', 'alloc'))
 202
 203     def test_in_state_when_no_state_available(self):
 204         self.make_actor(arv_node=testutil.arvados_node_mock(info={}))
 205         self.assertIsNone(self.node_state('idle', 'alloc'))
 206
 207     def test_in_idle_state(self):
 208         self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None))
 209         self.assertTrue(self.node_state('idle'))
 210         self.assertFalse(self.node_state('alloc'))
 211         self.assertTrue(self.node_state('idle', 'alloc'))
 212
 213     def test_in_alloc_state(self):
 214         self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True))
 215         self.assertFalse(self.node_state('idle'))
 216         self.assertTrue(self.node_state('alloc'))
 217         self.assertTrue(self.node_state('idle', 'alloc'))
 218
 219     def test_init_shutdown_scheduling(self):
 220         self.make_actor()
 221         self.assertTrue(self.timer.schedule.called)
 222         self.assertEqual(300, self.timer.schedule.call_args[0][0])
 223
 224     def test_shutdown_subscription(self):
 225         self.make_actor()
 226         self.shutdowns._set_state(True, 600)
 227         self.node_actor.consider_shutdown().get(self.TIMEOUT)
 228         self.assertTrue(self.subscriber.called)
 229         self.assertEqual(self.node_actor.actor_ref.actor_urn,
 230                          self.subscriber.call_args[0][0].actor_ref.actor_urn)
 231
 232     def test_shutdown_without_arvados_node(self):
 233         self.make_actor()
 234         self.shutdowns._set_state(True, 600)
 235         self.node_actor.consider_shutdown().get(self.TIMEOUT)
 236         self.assertTrue(self.subscriber.called)
 237
 238     def test_no_shutdown_without_arvados_node_and_old_cloud_node(self):
 239         self.make_actor(start_time=0)
 240         self.shutdowns._set_state(True, 600)
 241         self.node_actor.consider_shutdown().get(self.TIMEOUT)
 242         self.assertFalse(self.subscriber.called)
 243
 244     def check_shutdown_rescheduled(self, window_open, next_window,
 245                                    schedule_time=None):
 246         self.shutdowns._set_state(window_open, next_window)
 247         self.timer.schedule.reset_mock()
 248         self.node_actor.consider_shutdown().get(self.TIMEOUT)
 249         self.stop_proxy(self.node_actor)
 250         self.assertTrue(self.timer.schedule.called)
 251         if schedule_time is not None:
 252             self.assertEqual(schedule_time, self.timer.schedule.call_args[0][0])
 253         self.assertFalse(self.subscriber.called)
 254
 255     def test_shutdown_window_close_scheduling(self):
 256         self.make_actor()
 257         self.check_shutdown_rescheduled(False, 600, 600)
 258
 259     def test_no_shutdown_when_node_running_job(self):
 260         self.make_actor(4, testutil.arvados_node_mock(4, job_uuid=True))
 261         self.check_shutdown_rescheduled(True, 600)
 262
 263     def test_no_shutdown_when_node_state_unknown(self):
 264         self.make_actor(5, testutil.arvados_node_mock(5, info={}))
 265         self.check_shutdown_rescheduled(True, 600)
 266
 267     def test_no_shutdown_when_node_state_stale(self):
 268         self.make_actor(6, testutil.arvados_node_mock(6, age=90000))
 269         self.check_shutdown_rescheduled(True, 600)
 270
 271     def test_arvados_node_match(self):
 272         self.make_actor(2)
 273         arv_node = testutil.arvados_node_mock(
 274             2, hostname='compute-two.zzzzz.arvadosapi.com')
 275         pair_id = self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)
 276         self.assertEqual(self.cloud_mock.id, pair_id)
 277         self.stop_proxy(self.node_actor)
 278         self.updates.sync_node.assert_called_with(self.cloud_mock, arv_node)
 279
 280     def test_arvados_node_mismatch(self):
 281         self.make_actor(3)
 282         arv_node = testutil.arvados_node_mock(1)
 283         self.assertIsNone(
 284             self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
 285
 286     def test_update_cloud_node(self):
 287         self.make_actor(1)
 288         self.make_mocks(2)
 289         self.cloud_mock.id = '1'
 290         self.node_actor.update_cloud_node(self.cloud_mock)
 291         current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
 292         self.assertEqual([testutil.ip_address_mock(2)],
 293                          current_cloud.private_ips)
 294
 295     def test_missing_cloud_node_update(self):
 296         self.make_actor(1)
 297         self.node_actor.update_cloud_node(None)
 298         current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
 299         self.assertEqual([testutil.ip_address_mock(1)],
 300                          current_cloud.private_ips)
 301
 302     def test_update_arvados_node(self):
 303         self.make_actor(3)
 304         job_uuid = 'zzzzz-jjjjj-updatejobnode00'
 305         new_arvados = testutil.arvados_node_mock(3, job_uuid)
 306         self.node_actor.update_arvados_node(new_arvados)
 307         current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
 308         self.assertEqual(job_uuid, current_arvados['job_uuid'])
 309
 310     def test_missing_arvados_node_update(self):
 311         self.make_actor(4, testutil.arvados_node_mock(4))
 312         self.node_actor.update_arvados_node(None)
 313         current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
 314         self.assertEqual(testutil.ip_address_mock(4),
 315                          current_arvados['ip_address'])