services/nodemanager/tests/test_computenode_dispatch.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import absolute_import, print_function
   4
   5 import time
   6 import unittest
   7
   8 import arvados.errors as arverror
   9 import httplib2
  10 import mock
  11 import pykka
  12
  13 import arvnodeman.computenode.dispatch as dispatch
  14 from . import testutil
  15
  16 class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
  17     def make_mocks(self, arvados_effect=None):
  18         if arvados_effect is None:
  19             arvados_effect = [testutil.arvados_node_mock()]
  20         self.arvados_effect = arvados_effect
  21         self.timer = testutil.MockTimer()
  22         self.api_client = mock.MagicMock(name='api_client')
  23         self.api_client.nodes().create().execute.side_effect = arvados_effect
  24         self.api_client.nodes().update().execute.side_effect = arvados_effect
  25         self.cloud_client = mock.MagicMock(name='cloud_client')
  26         self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1)
  27
  28     def make_actor(self, arv_node=None):
  29         if not hasattr(self, 'timer'):
  30             self.make_mocks(arvados_effect=[arv_node])
  31         self.setup_actor = dispatch.ComputeNodeSetupActor.start(
  32             self.timer, self.api_client, self.cloud_client,
  33             testutil.MockSize(1), arv_node).proxy()
  34
  35     def test_creation_without_arvados_node(self):
  36         self.make_actor()
  37         self.assertEqual(self.arvados_effect[-1],
  38                          self.setup_actor.arvados_node.get(self.TIMEOUT))
  39         self.assertTrue(self.api_client.nodes().create().execute.called)
  40         self.assertEqual(self.cloud_client.create_node(),
  41                          self.setup_actor.cloud_node.get(self.TIMEOUT))
  42
  43     def test_creation_with_arvados_node(self):
  44         self.make_actor(testutil.arvados_node_mock())
  45         self.assertEqual(self.arvados_effect[-1],
  46                          self.setup_actor.arvados_node.get(self.TIMEOUT))
  47         self.assertTrue(self.api_client.nodes().update().execute.called)
  48         self.assertEqual(self.cloud_client.create_node(),
  49                          self.setup_actor.cloud_node.get(self.TIMEOUT))
  50
  51     def test_failed_arvados_calls_retried(self):
  52         self.make_mocks([
  53                 arverror.ApiError(httplib2.Response({'status': '500'}), ""),
  54                 testutil.arvados_node_mock(),
  55                 ])
  56         self.make_actor()
  57         self.wait_for_assignment(self.setup_actor, 'arvados_node')
  58
  59     def test_failed_cloud_calls_retried(self):
  60         self.make_mocks()
  61         self.cloud_client.create_node.side_effect = [
  62             Exception("test cloud creation error"),
  63             self.cloud_client.create_node.return_value,
  64             ]
  65         self.make_actor()
  66         self.wait_for_assignment(self.setup_actor, 'cloud_node')
  67
  68     def test_failed_post_create_retried(self):
  69         self.make_mocks()
  70         self.cloud_client.post_create_node.side_effect = [
  71             Exception("test cloud post-create error"), None]
  72         self.make_actor()
  73         done = self.FUTURE_CLASS()
  74         self.setup_actor.subscribe(done.set)
  75         done.get(self.TIMEOUT)
  76         self.assertEqual(2, self.cloud_client.post_create_node.call_count)
  77
  78     def test_stop_when_no_cloud_node(self):
  79         self.make_mocks(
  80             arverror.ApiError(httplib2.Response({'status': '500'}), ""))
  81         self.make_actor()
  82         self.assertTrue(
  83             self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT))
  84         self.assertTrue(
  85             self.setup_actor.actor_ref.actor_stopped.wait(self.TIMEOUT))
  86
  87     def test_no_stop_when_cloud_node(self):
  88         self.make_actor()
  89         self.wait_for_assignment(self.setup_actor, 'cloud_node')
  90         self.assertFalse(
  91             self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT))
  92         self.assertTrue(self.stop_proxy(self.setup_actor),
  93                         "actor was stopped by stop_if_no_cloud_node")
  94
  95     def test_subscribe(self):
  96         self.make_mocks(
  97             arverror.ApiError(httplib2.Response({'status': '500'}), ""))
  98         self.make_actor()
  99         subscriber = mock.Mock(name='subscriber_mock')
 100         self.setup_actor.subscribe(subscriber)
 101         self.api_client.nodes().create().execute.side_effect = [
 102             testutil.arvados_node_mock()]
 103         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 104         self.assertEqual(self.setup_actor.actor_ref.actor_urn,
 105                          subscriber.call_args[0][0].actor_ref.actor_urn)
 106
 107     def test_late_subscribe(self):
 108         self.make_actor()
 109         subscriber = mock.Mock(name='subscriber_mock')
 110         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 111         self.setup_actor.subscribe(subscriber).get(self.TIMEOUT)
 112         self.stop_proxy(self.setup_actor)
 113         self.assertEqual(self.setup_actor.actor_ref.actor_urn,
 114                          subscriber.call_args[0][0].actor_ref.actor_urn)
 115
 116
 117 class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin):
 118     def make_mocks(self, cloud_node=None, arvados_node=None,
 119                    shutdown_open=True):
 120         self.timer = testutil.MockTimer()
 121         self.shutdowns = testutil.MockShutdownTimer()
 122         self.shutdowns._set_state(shutdown_open, 300)
 123         self.cloud_client = mock.MagicMock(name='cloud_client')
 124         self.arvados_client = mock.MagicMock(name='arvados_client')
 125         self.updates = mock.MagicMock(name='update_mock')
 126         if cloud_node is None:
 127             cloud_node = testutil.cloud_node_mock()
 128         self.cloud_node = cloud_node
 129         self.arvados_node = arvados_node
 130
 131     def make_actor(self, cancellable=True):
 132         if not hasattr(self, 'timer'):
 133             self.make_mocks()
 134         monitor_actor = dispatch.ComputeNodeMonitorActor.start(
 135             self.cloud_node, time.time(), self.shutdowns,
 136             testutil.cloud_node_fqdn, self.timer, self.updates,
 137             self.arvados_node)
 138         self.shutdown_actor = self.ACTOR_CLASS.start(
 139             self.timer, self.cloud_client, self.arvados_client, monitor_actor,
 140             cancellable).proxy()
 141         self.monitor_actor = monitor_actor.proxy()
 142
 143     def check_success_flag(self, expected, allow_msg_count=1):
 144         # allow_msg_count is the number of internal messages that may
 145         # need to be handled for shutdown to finish.
 146         for try_num in range(1 + allow_msg_count):
 147             last_flag = self.shutdown_actor.success.get(self.TIMEOUT)
 148             if last_flag is expected:
 149                 break
 150         else:
 151             self.fail("success flag {} is not {}".format(last_flag, expected))
 152
 153     def test_uncancellable_shutdown(self, *mocks):
 154         self.make_mocks(shutdown_open=False)
 155         self.cloud_client.destroy_node.return_value = False
 156         self.make_actor(cancellable=False)
 157         self.check_success_flag(None, 0)
 158         self.shutdowns._set_state(True, 600)
 159         self.cloud_client.destroy_node.return_value = True
 160         self.check_success_flag(True)
 161
 162     def test_arvados_node_cleaned_after_shutdown(self, *mocks):
 163         cloud_node = testutil.cloud_node_mock(62)
 164         arv_node = testutil.arvados_node_mock(62)
 165         self.make_mocks(cloud_node, arv_node)
 166         self.make_actor()
 167         self.check_success_flag(True, 3)
 168         update_mock = self.arvados_client.nodes().update
 169         self.assertTrue(update_mock.called)
 170         update_kwargs = update_mock.call_args_list[0][1]
 171         self.assertEqual(arv_node['uuid'], update_kwargs.get('uuid'))
 172         self.assertIn('body', update_kwargs)
 173         for clear_key in ['slot_number', 'hostname', 'ip_address',
 174                           'first_ping_at', 'last_ping_at']:
 175             self.assertIn(clear_key, update_kwargs['body'])
 176             self.assertIsNone(update_kwargs['body'][clear_key])
 177         self.assertTrue(update_mock().execute.called)
 178
 179     def test_arvados_node_not_cleaned_after_shutdown_cancelled(self, *mocks):
 180         cloud_node = testutil.cloud_node_mock(61)
 181         arv_node = testutil.arvados_node_mock(61)
 182         self.make_mocks(cloud_node, arv_node, shutdown_open=False)
 183         self.make_actor(cancellable=True)
 184         self.check_success_flag(False, 2)
 185         self.assertFalse(self.arvados_client.nodes().update.called)
 186
 187
 188 class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin,
 189                                        unittest.TestCase):
 190     ACTOR_CLASS = dispatch.ComputeNodeShutdownActor
 191
 192     def test_easy_shutdown(self):
 193         self.make_actor()
 194         self.check_success_flag(True)
 195         self.assertTrue(self.cloud_client.destroy_node.called)
 196
 197     def test_shutdown_cancelled_when_window_closes(self):
 198         self.make_mocks(shutdown_open=False)
 199         self.make_actor()
 200         self.check_success_flag(False, 2)
 201         self.assertFalse(self.cloud_client.destroy_node.called)
 202
 203     def test_shutdown_retries_when_cloud_fails(self):
 204         self.make_mocks()
 205         self.cloud_client.destroy_node.return_value = False
 206         self.make_actor()
 207         self.assertIsNone(self.shutdown_actor.success.get(self.TIMEOUT))
 208         self.cloud_client.destroy_node.return_value = True
 209         self.check_success_flag(True)
 210
 211     def test_late_subscribe(self):
 212         self.make_actor()
 213         subscriber = mock.Mock(name='subscriber_mock')
 214         self.shutdown_actor.subscribe(subscriber).get(self.TIMEOUT)
 215         self.stop_proxy(self.shutdown_actor)
 216         self.assertTrue(subscriber.called)
 217         self.assertEqual(self.shutdown_actor.actor_ref.actor_urn,
 218                          subscriber.call_args[0][0].actor_ref.actor_urn)
 219
 220
 221 class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
 222                                      unittest.TestCase):
 223     def make_actor(self):
 224         self.driver = mock.MagicMock(name='driver_mock')
 225         self.updater = dispatch.ComputeNodeUpdateActor.start(self.driver).proxy()
 226
 227     def test_node_sync(self):
 228         self.make_actor()
 229         cloud_node = testutil.cloud_node_mock()
 230         arv_node = testutil.arvados_node_mock()
 231         self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
 232         self.driver().sync_node.assert_called_with(cloud_node, arv_node)
 233
 234
 235 class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin,
 236                                       unittest.TestCase):
 237     def make_mocks(self, node_num):
 238         self.shutdowns = testutil.MockShutdownTimer()
 239         self.shutdowns._set_state(False, 300)
 240         self.timer = mock.MagicMock(name='timer_mock')
 241         self.updates = mock.MagicMock(name='update_mock')
 242         self.cloud_mock = testutil.cloud_node_mock(node_num)
 243         self.subscriber = mock.Mock(name='subscriber_mock')
 244
 245     def make_actor(self, node_num=1, arv_node=None, start_time=None):
 246         if not hasattr(self, 'cloud_mock'):
 247             self.make_mocks(node_num)
 248         if start_time is None:
 249             start_time = time.time()
 250         self.node_actor = dispatch.ComputeNodeMonitorActor.start(
 251             self.cloud_mock, start_time, self.shutdowns,
 252             testutil.cloud_node_fqdn, self.timer, self.updates,
 253             arv_node).proxy()
 254         self.node_actor.subscribe(self.subscriber).get(self.TIMEOUT)
 255
 256     def node_state(self, *states):
 257         return self.node_actor.in_state(*states).get(self.TIMEOUT)
 258
 259     def test_in_state_when_unpaired(self):
 260         self.make_actor()
 261         self.assertIsNone(self.node_state('idle', 'busy'))
 262
 263     def test_in_state_when_pairing_stale(self):
 264         self.make_actor(arv_node=testutil.arvados_node_mock(
 265                 job_uuid=None, age=90000))
 266         self.assertIsNone(self.node_state('idle', 'busy'))
 267
 268     def test_in_state_when_no_state_available(self):
 269         self.make_actor(arv_node=testutil.arvados_node_mock(
 270                 crunch_worker_state=None))
 271         self.assertIsNone(self.node_state('idle', 'busy'))
 272
 273     def test_in_idle_state(self):
 274         self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None))
 275         self.assertTrue(self.node_state('idle'))
 276         self.assertFalse(self.node_state('busy'))
 277         self.assertTrue(self.node_state('idle', 'busy'))
 278
 279     def test_in_busy_state(self):
 280         self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True))
 281         self.assertFalse(self.node_state('idle'))
 282         self.assertTrue(self.node_state('busy'))
 283         self.assertTrue(self.node_state('idle', 'busy'))
 284
 285     def test_init_shutdown_scheduling(self):
 286         self.make_actor()
 287         self.assertTrue(self.timer.schedule.called)
 288         self.assertEqual(300, self.timer.schedule.call_args[0][0])
 289
 290     def test_shutdown_window_close_scheduling(self):
 291         self.make_actor()
 292         self.shutdowns._set_state(False, 600)
 293         self.timer.schedule.reset_mock()
 294         self.node_actor.consider_shutdown().get(self.TIMEOUT)
 295         self.stop_proxy(self.node_actor)
 296         self.assertTrue(self.timer.schedule.called)
 297         self.assertEqual(600, self.timer.schedule.call_args[0][0])
 298         self.assertFalse(self.subscriber.called)
 299
 300     def test_shutdown_subscription(self):
 301         self.make_actor()
 302         self.shutdowns._set_state(True, 600)
 303         self.node_actor.consider_shutdown().get(self.TIMEOUT)
 304         self.assertTrue(self.subscriber.called)
 305         self.assertEqual(self.node_actor.actor_ref.actor_urn,
 306                          self.subscriber.call_args[0][0].actor_ref.actor_urn)
 307
 308     def test_shutdown_without_arvados_node(self):
 309         self.make_actor()
 310         self.shutdowns._set_state(True, 600)
 311         self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 312
 313     def test_no_shutdown_without_arvados_node_and_old_cloud_node(self):
 314         self.make_actor(start_time=0)
 315         self.shutdowns._set_state(True, 600)
 316         self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 317
 318     def test_no_shutdown_when_window_closed(self):
 319         self.make_actor(3, testutil.arvados_node_mock(3, job_uuid=None))
 320         self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 321
 322     def test_no_shutdown_when_node_running_job(self):
 323         self.make_actor(4, testutil.arvados_node_mock(4, job_uuid=True))
 324         self.shutdowns._set_state(True, 600)
 325         self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 326
 327     def test_no_shutdown_when_node_state_unknown(self):
 328         self.make_actor(5, testutil.arvados_node_mock(
 329             5, crunch_worker_state=None))
 330         self.shutdowns._set_state(True, 600)
 331         self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 332
 333     def test_no_shutdown_when_node_state_stale(self):
 334         self.make_actor(6, testutil.arvados_node_mock(6, age=90000))
 335         self.shutdowns._set_state(True, 600)
 336         self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 337
 338     def test_arvados_node_match(self):
 339         self.make_actor(2)
 340         arv_node = testutil.arvados_node_mock(
 341             2, hostname='compute-two.zzzzz.arvadosapi.com')
 342         pair_id = self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)
 343         self.assertEqual(self.cloud_mock.id, pair_id)
 344         self.stop_proxy(self.node_actor)
 345         self.updates.sync_node.assert_called_with(self.cloud_mock, arv_node)
 346
 347     def test_arvados_node_mismatch(self):
 348         self.make_actor(3)
 349         arv_node = testutil.arvados_node_mock(1)
 350         self.assertIsNone(
 351             self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
 352
 353     def test_arvados_node_mismatch_first_ping_too_early(self):
 354         self.make_actor(4)
 355         arv_node = testutil.arvados_node_mock(
 356             4, first_ping_at='1971-03-02T14:15:16.1717282Z')
 357         self.assertIsNone(
 358             self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
 359
 360     def test_update_cloud_node(self):
 361         self.make_actor(1)
 362         self.make_mocks(2)
 363         self.cloud_mock.id = '1'
 364         self.node_actor.update_cloud_node(self.cloud_mock)
 365         current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
 366         self.assertEqual([testutil.ip_address_mock(2)],
 367                          current_cloud.private_ips)
 368
 369     def test_missing_cloud_node_update(self):
 370         self.make_actor(1)
 371         self.node_actor.update_cloud_node(None)
 372         current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
 373         self.assertEqual([testutil.ip_address_mock(1)],
 374                          current_cloud.private_ips)
 375
 376     def test_update_arvados_node(self):
 377         self.make_actor(3)
 378         job_uuid = 'zzzzz-jjjjj-updatejobnode00'
 379         new_arvados = testutil.arvados_node_mock(3, job_uuid)
 380         self.node_actor.update_arvados_node(new_arvados)
 381         current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
 382         self.assertEqual(job_uuid, current_arvados['job_uuid'])
 383
 384     def test_missing_arvados_node_update(self):
 385         self.make_actor(4, testutil.arvados_node_mock(4))
 386         self.node_actor.update_arvados_node(None)
 387         current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
 388         self.assertEqual(testutil.ip_address_mock(4),
 389                          current_arvados['ip_address'])
 390
 391     def test_update_arvados_node_syncs_when_fqdn_mismatch(self):
 392         self.make_mocks(5)
 393         self.cloud_mock.extra['testname'] = 'cloudfqdn.zzzzz.arvadosapi.com'
 394         self.make_actor()
 395         arv_node = testutil.arvados_node_mock(5)
 396         self.node_actor.update_arvados_node(arv_node).get(self.TIMEOUT)
 397         self.assertEqual(1, self.updates.sync_node.call_count)
 398
 399     def test_update_arvados_node_skips_sync_when_fqdn_match(self):
 400         self.make_mocks(6)
 401         arv_node = testutil.arvados_node_mock(6)
 402         self.cloud_mock.extra['testname'] ='{n[hostname]}.{n[domain]}'.format(
 403             n=arv_node)
 404         self.make_actor()
 405         self.node_actor.update_arvados_node(arv_node).get(self.TIMEOUT)
 406         self.assertEqual(0, self.updates.sync_node.call_count)