X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/5120666e073018f6821bab07f0bbb788098f97b1..59a972c831bc8f7cd4e896ed8e1c71277b97f04e:/services/nodemanager/tests/test_computenode_dispatch.py diff --git a/services/nodemanager/tests/test_computenode_dispatch.py b/services/nodemanager/tests/test_computenode_dispatch.py index 1102bf72ec..aee3cbdac8 100644 --- a/services/nodemanager/tests/test_computenode_dispatch.py +++ b/services/nodemanager/tests/test_computenode_dispatch.py @@ -17,6 +17,7 @@ import threading from libcloud.common.exceptions import BaseHTTPError import arvnodeman.computenode.dispatch as dispatch +import arvnodeman.status as status from arvnodeman.computenode.driver import BaseComputeNodeDriver from . import testutil @@ -63,6 +64,7 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase): self.assertEqual(self.arvados_effect[-1], self.setup_actor.arvados_node.get(self.TIMEOUT)) assert(finished.wait(self.TIMEOUT)) + self.api_client.nodes().create.called_with(body={}, assign_slot=True) self.assertEqual(1, self.api_client.nodes().create().execute.call_count) self.assertEqual(1, self.api_client.nodes().update().execute.call_count) self.assert_node_properties_updated() @@ -78,7 +80,8 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase): self.setup_actor.arvados_node.get(self.TIMEOUT)) assert(finished.wait(self.TIMEOUT)) self.assert_node_properties_updated() - self.assertEqual(2, self.api_client.nodes().update().execute.call_count) + self.api_client.nodes().create.called_with(body={}, assign_slot=True) + self.assertEqual(3, self.api_client.nodes().update().execute.call_count) self.assertEqual(self.cloud_client.create_node(), self.setup_actor.cloud_node.get(self.TIMEOUT)) @@ -195,7 +198,7 @@ class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin): start_time = time.time() monitor_actor = dispatch.ComputeNodeMonitorActor.start( self.cloud_node, start_time, self.shutdowns, - testutil.cloud_node_fqdn, self.timer, self.updates, self.cloud_client, + self.timer, self.updates, self.cloud_client, self.arvados_node) self.shutdown_actor = self.ACTOR_CLASS.start( self.timer, self.cloud_client, self.arvados_client, monitor_actor, @@ -205,13 +208,23 @@ class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin): def check_success_flag(self, expected, allow_msg_count=1): # allow_msg_count is the number of internal messages that may # need to be handled for shutdown to finish. - for try_num in range(1 + allow_msg_count): + for _ in range(1 + allow_msg_count): last_flag = self.shutdown_actor.success.get(self.TIMEOUT) if last_flag is expected: break else: self.fail("success flag {} is not {}".format(last_flag, expected)) + def test_boot_failure_counting(self, *mocks): + # A boot failure happens when a node transitions from unpaired to shutdown + status.tracker.update({'boot_failures': 0}) + self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="unpaired")) + self.cloud_client.destroy_node.return_value = True + self.make_actor(cancellable=False) + self.check_success_flag(True, 2) + self.assertTrue(self.cloud_client.destroy_node.called) + self.assertEqual(1, status.tracker.get('boot_failures')) + def test_cancellable_shutdown(self, *mocks): self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy")) self.cloud_client.destroy_node.return_value = True @@ -220,11 +233,14 @@ class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin): self.assertFalse(self.cloud_client.destroy_node.called) def test_uncancellable_shutdown(self, *mocks): + status.tracker.update({'boot_failures': 0}) self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy")) self.cloud_client.destroy_node.return_value = True self.make_actor(cancellable=False) self.check_success_flag(True, 4) self.assertTrue(self.cloud_client.destroy_node.called) + # A normal shutdown shouldn't be counted as boot failure + self.assertEqual(0, status.tracker.get('boot_failures')) def test_arvados_node_cleaned_after_shutdown(self, *mocks): if len(mocks) == 1: @@ -333,7 +349,7 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, start_time = time.time() self.node_actor = dispatch.ComputeNodeMonitorActor.start( self.cloud_mock, start_time, self.shutdowns, - testutil.cloud_node_fqdn, self.timer, self.updates, self.cloud_client, + self.timer, self.updates, self.cloud_client, arv_node, boot_fail_after=300).proxy() self.node_actor.subscribe(self.subscriber).get(self.TIMEOUT) @@ -360,16 +376,26 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.assertTrue(self.node_state('down')) def test_in_idle_state(self): + idle_nodes_before = status.tracker._idle_nodes.keys() self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None)) self.assertTrue(self.node_state('idle')) self.assertFalse(self.node_state('busy')) self.assertTrue(self.node_state('idle', 'busy')) + idle_nodes_after = status.tracker._idle_nodes.keys() + new_idle_nodes = [n for n in idle_nodes_after if n not in idle_nodes_before] + # There should be 1 additional idle node + self.assertEqual(1, len(new_idle_nodes)) def test_in_busy_state(self): + idle_nodes_before = status.tracker._idle_nodes.keys() self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True)) self.assertFalse(self.node_state('idle')) self.assertTrue(self.node_state('busy')) self.assertTrue(self.node_state('idle', 'busy')) + idle_nodes_after = status.tracker._idle_nodes.keys() + new_idle_nodes = [n for n in idle_nodes_after if n not in idle_nodes_before] + # There shouldn't be any additional idle node + self.assertEqual(0, len(new_idle_nodes)) def test_init_shutdown_scheduling(self): self.make_actor() @@ -398,12 +424,21 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.make_actor() self.shutdowns._set_state(True, 600) self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT), - (False, "node state is ('unpaired', 'open', 'boot wait', 'idle exceeded')")) + (False, "node state is ('unpaired', 'open', 'boot wait', 'not idle')")) + + def test_shutdown_when_invalid_cloud_node_size(self): + self.make_mocks(1) + self.cloud_mock.size.id = 'invalid' + self.cloud_mock.extra['arvados_node_size'] = 'stale.type' + self.make_actor() + self.shutdowns._set_state(True, 600) + self.assertEquals((True, "node's size tag 'stale.type' not recognizable"), + self.node_actor.shutdown_eligible().get(self.TIMEOUT)) def test_shutdown_without_arvados_node(self): self.make_actor(start_time=0) self.shutdowns._set_state(True, 600) - self.assertEquals((True, "node state is ('down', 'open', 'boot exceeded', 'idle exceeded')"), + self.assertEquals((True, "node state is ('down', 'open', 'boot exceeded', 'not idle')"), self.node_actor.shutdown_eligible().get(self.TIMEOUT)) def test_shutdown_missing(self): @@ -412,7 +447,7 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, last_ping_at='1970-01-01T01:02:03.04050607Z') self.make_actor(10, arv_node) self.shutdowns._set_state(True, 600) - self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"), + self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'not idle')"), self.node_actor.shutdown_eligible().get(self.TIMEOUT)) def test_shutdown_running_broken(self): @@ -421,7 +456,7 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.make_actor(12, arv_node) self.shutdowns._set_state(True, 600) self.cloud_client.broken.return_value = True - self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"), + self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'not idle')"), self.node_actor.shutdown_eligible().get(self.TIMEOUT)) def test_shutdown_missing_broken(self): @@ -431,7 +466,7 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.make_actor(11, arv_node) self.shutdowns._set_state(True, 600) self.cloud_client.broken.return_value = True - self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT), (True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')")) + self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT), (True, "node state is ('down', 'open', 'boot wait', 'not idle')")) def test_no_shutdown_when_window_closed(self): self.make_actor(3, testutil.arvados_node_mock(3, job_uuid=None)) @@ -441,7 +476,7 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, def test_no_shutdown_when_node_running_job(self): self.make_actor(4, testutil.arvados_node_mock(4, job_uuid=True)) self.shutdowns._set_state(True, 600) - self.assertEquals((False, "node state is ('busy', 'open', 'boot wait', 'idle exceeded')"), + self.assertEquals((False, "node state is ('busy', 'open', 'boot wait', 'not idle')"), self.node_actor.shutdown_eligible().get(self.TIMEOUT)) def test_shutdown_when_node_state_unknown(self): @@ -455,7 +490,7 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.make_actor(5, testutil.arvados_node_mock( 5, crunch_worker_state='fail')) self.shutdowns._set_state(True, 600) - self.assertEquals((True, "node state is ('fail', 'open', 'boot wait', 'idle exceeded')"), + self.assertEquals((True, "node state is ('fail', 'open', 'boot wait', 'not idle')"), self.node_actor.shutdown_eligible().get(self.TIMEOUT)) def test_no_shutdown_when_node_state_stale(self): @@ -518,19 +553,10 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.assertEqual(testutil.ip_address_mock(4), current_arvados['ip_address']) - def test_update_arvados_node_syncs_when_fqdn_mismatch(self): + def test_update_arvados_node_calls_sync_node(self): self.make_mocks(5) self.cloud_mock.extra['testname'] = 'cloudfqdn.zzzzz.arvadosapi.com' self.make_actor() arv_node = testutil.arvados_node_mock(5) self.node_actor.update_arvados_node(arv_node).get(self.TIMEOUT) self.assertEqual(1, self.updates.sync_node.call_count) - - def test_update_arvados_node_skips_sync_when_fqdn_match(self): - self.make_mocks(6) - arv_node = testutil.arvados_node_mock(6) - self.cloud_mock.extra['testname'] ='{n[hostname]}.{n[domain]}'.format( - n=arv_node) - self.make_actor() - self.node_actor.update_arvados_node(arv_node).get(self.TIMEOUT) - self.assertEqual(0, self.updates.sync_node.call_count)