X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c59093180fd92f0d7c6607a49458446212ebd058..1d1c6de3c842a33a57b7d469fdaaaa1b873433dc:/services/nodemanager/tests/test_computenode_dispatch.py diff --git a/services/nodemanager/tests/test_computenode_dispatch.py b/services/nodemanager/tests/test_computenode_dispatch.py index 96a70c6c96..e718fc134b 100644 --- a/services/nodemanager/tests/test_computenode_dispatch.py +++ b/services/nodemanager/tests/test_computenode_dispatch.py @@ -121,21 +121,25 @@ class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin): self.shutdowns = testutil.MockShutdownTimer() self.shutdowns._set_state(shutdown_open, 300) self.cloud_client = mock.MagicMock(name='cloud_client') + self.arvados_client = mock.MagicMock(name='arvados_client') self.updates = mock.MagicMock(name='update_mock') if cloud_node is None: cloud_node = testutil.cloud_node_mock() self.cloud_node = cloud_node self.arvados_node = arvados_node - def make_actor(self, cancellable=True): + def make_actor(self, cancellable=True, start_time=None): if not hasattr(self, 'timer'): self.make_mocks() + if start_time is None: + start_time = time.time() monitor_actor = dispatch.ComputeNodeMonitorActor.start( - self.cloud_node, time.time(), self.shutdowns, - testutil.cloud_node_fqdn, self.timer, self.updates, + self.cloud_node, start_time, self.shutdowns, + testutil.cloud_node_fqdn, self.timer, self.updates, self.cloud_client, self.arvados_node) self.shutdown_actor = self.ACTOR_CLASS.start( - self.timer, self.cloud_client, monitor_actor, cancellable).proxy() + self.timer, self.cloud_client, self.arvados_client, monitor_actor, + cancellable).proxy() self.monitor_actor = monitor_actor.proxy() def check_success_flag(self, expected, allow_msg_count=1): @@ -157,13 +161,38 @@ class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin): self.cloud_client.destroy_node.return_value = True self.check_success_flag(True) + def test_arvados_node_cleaned_after_shutdown(self, *mocks): + cloud_node = testutil.cloud_node_mock(62) + arv_node = testutil.arvados_node_mock(62) + self.make_mocks(cloud_node, arv_node) + self.make_actor() + self.check_success_flag(True, 3) + update_mock = self.arvados_client.nodes().update + self.assertTrue(update_mock.called) + update_kwargs = update_mock.call_args_list[0][1] + self.assertEqual(arv_node['uuid'], update_kwargs.get('uuid')) + self.assertIn('body', update_kwargs) + for clear_key in ['slot_number', 'hostname', 'ip_address', + 'first_ping_at', 'last_ping_at']: + self.assertIn(clear_key, update_kwargs['body']) + self.assertIsNone(update_kwargs['body'][clear_key]) + self.assertTrue(update_mock().execute.called) + + def test_arvados_node_not_cleaned_after_shutdown_cancelled(self, *mocks): + cloud_node = testutil.cloud_node_mock(61) + arv_node = testutil.arvados_node_mock(61) + self.make_mocks(cloud_node, arv_node, shutdown_open=False) + self.make_actor(cancellable=True) + self.check_success_flag(False, 2) + self.assertFalse(self.arvados_client.nodes().update.called) + class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin, unittest.TestCase): ACTOR_CLASS = dispatch.ComputeNodeShutdownActor def test_easy_shutdown(self): - self.make_actor() + self.make_actor(start_time=0) self.check_success_flag(True) self.assertTrue(self.cloud_client.destroy_node.called) @@ -176,7 +205,7 @@ class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin, def test_shutdown_retries_when_cloud_fails(self): self.make_mocks() self.cloud_client.destroy_node.return_value = False - self.make_actor() + self.make_actor(start_time=0) self.assertIsNone(self.shutdown_actor.success.get(self.TIMEOUT)) self.cloud_client.destroy_node.return_value = True self.check_success_flag(True) @@ -214,6 +243,8 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.updates = mock.MagicMock(name='update_mock') self.cloud_mock = testutil.cloud_node_mock(node_num) self.subscriber = mock.Mock(name='subscriber_mock') + self.cloud_client = mock.MagicMock(name='cloud_client') + self.cloud_client.broken.return_value = False def make_actor(self, node_num=1, arv_node=None, start_time=None): if not hasattr(self, 'cloud_mock'): @@ -222,8 +253,8 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, start_time = time.time() self.node_actor = dispatch.ComputeNodeMonitorActor.start( self.cloud_mock, start_time, self.shutdowns, - testutil.cloud_node_fqdn, self.timer, self.updates, - arv_node).proxy() + testutil.cloud_node_fqdn, self.timer, self.updates, self.cloud_client, + arv_node, boot_fail_after=300).proxy() self.node_actor.subscribe(self.subscriber).get(self.TIMEOUT) def node_state(self, *states): @@ -271,23 +302,48 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.assertFalse(self.subscriber.called) def test_shutdown_subscription(self): - self.make_actor() + self.make_actor(start_time=0) self.shutdowns._set_state(True, 600) self.node_actor.consider_shutdown().get(self.TIMEOUT) self.assertTrue(self.subscriber.called) self.assertEqual(self.node_actor.actor_ref.actor_urn, self.subscriber.call_args[0][0].actor_ref.actor_urn) - def test_shutdown_without_arvados_node(self): + def test_no_shutdown_booting(self): self.make_actor() self.shutdowns._set_state(True, 600) - self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) - def test_no_shutdown_without_arvados_node_and_old_cloud_node(self): + def test_shutdown_without_arvados_node(self): self.make_actor(start_time=0) self.shutdowns._set_state(True, 600) + self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + + def test_no_shutdown_missing(self): + arv_node = testutil.arvados_node_mock(10, job_uuid=None, + crunch_worker_state="down", + last_ping_at='1970-01-01T01:02:03.04050607Z') + self.make_actor(10, arv_node) + self.shutdowns._set_state(True, 600) self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + def test_no_shutdown_running_broken(self): + arv_node = testutil.arvados_node_mock(12, job_uuid=None, + crunch_worker_state="down") + self.make_actor(12, arv_node) + self.shutdowns._set_state(True, 600) + self.cloud_client.broken.return_value = True + self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + + def test_shutdown_missing_broken(self): + arv_node = testutil.arvados_node_mock(11, job_uuid=None, + crunch_worker_state="down", + last_ping_at='1970-01-01T01:02:03.04050607Z') + self.make_actor(11, arv_node) + self.shutdowns._set_state(True, 600) + self.cloud_client.broken.return_value = True + self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + def test_no_shutdown_when_window_closed(self): self.make_actor(3, testutil.arvados_node_mock(3, job_uuid=None)) self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT))