X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0662b235357dd40b5d27efd06b60044ddcec06f6..ba534230bcc23aaea69e5f256693b9e143200b63:/services/nodemanager/tests/test_computenode_dispatch.py diff --git a/services/nodemanager/tests/test_computenode_dispatch.py b/services/nodemanager/tests/test_computenode_dispatch.py index ecf83c693a..14cd85e414 100644 --- a/services/nodemanager/tests/test_computenode_dispatch.py +++ b/services/nodemanager/tests/test_computenode_dispatch.py @@ -9,6 +9,7 @@ import arvados.errors as arverror import httplib2 import mock import pykka +import threading import arvnodeman.computenode.dispatch as dispatch from . import testutil @@ -44,8 +45,11 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase): def test_creation_without_arvados_node(self): self.make_actor() + finished = threading.Event() + self.setup_actor.subscribe(lambda _: finished.set()) self.assertEqual(self.arvados_effect[-1], self.setup_actor.arvados_node.get(self.TIMEOUT)) + assert(finished.wait(self.TIMEOUT)) self.assertEqual(1, self.api_client.nodes().create().execute.call_count) self.assertEqual(1, self.api_client.nodes().update().execute.call_count) self.assert_node_properties_updated() @@ -55,8 +59,11 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase): def test_creation_with_arvados_node(self): self.make_mocks(arvados_effect=[testutil.arvados_node_mock()]*2) self.make_actor(testutil.arvados_node_mock()) + finished = threading.Event() + self.setup_actor.subscribe(lambda _: finished.set()) self.assertEqual(self.arvados_effect[-1], self.setup_actor.arvados_node.get(self.TIMEOUT)) + assert(finished.wait(self.TIMEOUT)) self.assert_node_properties_updated() self.assertEqual(2, self.api_client.nodes().update().execute.call_count) self.assertEqual(self.cloud_client.create_node(), @@ -260,6 +267,16 @@ class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin, self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT) self.driver().sync_node.assert_called_with(cloud_node, arv_node) + @testutil.no_sleep + def test_node_sync_error(self): + self.make_actor() + cloud_node = testutil.cloud_node_mock() + arv_node = testutil.arvados_node_mock() + self.driver().sync_node.side_effect = (IOError, Exception, True) + self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT) + self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT) + self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT) + self.driver().sync_node.assert_called_with(cloud_node, arv_node) class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, unittest.TestCase): @@ -339,12 +356,12 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, def test_no_shutdown_booting(self): self.make_actor() self.shutdowns._set_state(True, 600) - self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT).startswith("node is still booting")) def test_shutdown_without_arvados_node(self): self.make_actor(start_time=0) self.shutdowns._set_state(True, 600) - self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + self.assertIs(True, self.node_actor.shutdown_eligible().get(self.TIMEOUT)) def test_no_shutdown_missing(self): arv_node = testutil.arvados_node_mock(10, job_uuid=None, @@ -352,7 +369,7 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, last_ping_at='1970-01-01T01:02:03.04050607Z') self.make_actor(10, arv_node) self.shutdowns._set_state(True, 600) - self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT).startswith("node is not idle.")) def test_no_shutdown_running_broken(self): arv_node = testutil.arvados_node_mock(12, job_uuid=None, @@ -360,7 +377,7 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.make_actor(12, arv_node) self.shutdowns._set_state(True, 600) self.cloud_client.broken.return_value = True - self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT).startswith("node is not idle.")) def test_shutdown_missing_broken(self): arv_node = testutil.arvados_node_mock(11, job_uuid=None, @@ -369,32 +386,33 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.make_actor(11, arv_node) self.shutdowns._set_state(True, 600) self.cloud_client.broken.return_value = True - self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + self.assertIs(True, self.node_actor.shutdown_eligible().get(self.TIMEOUT)) def test_no_shutdown_when_window_closed(self): self.make_actor(3, testutil.arvados_node_mock(3, job_uuid=None)) - self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT).startswith("shutdown window is not open.")) def test_no_shutdown_when_node_running_job(self): self.make_actor(4, testutil.arvados_node_mock(4, job_uuid=True)) self.shutdowns._set_state(True, 600) - self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT).startswith("node is not idle.")) def test_no_shutdown_when_node_state_unknown(self): self.make_actor(5, testutil.arvados_node_mock( 5, crunch_worker_state=None)) self.shutdowns._set_state(True, 600) - self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT).startswith("node is not idle.")) def test_no_shutdown_when_node_state_stale(self): self.make_actor(6, testutil.arvados_node_mock(6, age=90000)) self.shutdowns._set_state(True, 600) - self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) + self.assertTrue(self.node_actor.shutdown_eligible().get(self.TIMEOUT).startswith("node is not idle.")) def test_arvados_node_match(self): self.make_actor(2) arv_node = testutil.arvados_node_mock( 2, hostname='compute-two.zzzzz.arvadosapi.com') + self.cloud_client.node_id.return_value = '2' pair_id = self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT) self.assertEqual(self.cloud_mock.id, pair_id) self.stop_proxy(self.node_actor)