X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/5624fec61db977d386ce03ca333241c74ca251b5..b51d376ed64efc68f7ee27fd061323da43faabd5:/services/nodemanager/tests/test_computenode_dispatch.py diff --git a/services/nodemanager/tests/test_computenode_dispatch.py b/services/nodemanager/tests/test_computenode_dispatch.py index ec1573409a..e4037d11a1 100644 --- a/services/nodemanager/tests/test_computenode_dispatch.py +++ b/services/nodemanager/tests/test_computenode_dispatch.py @@ -1,4 +1,7 @@ #!/usr/bin/env python +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 from __future__ import absolute_import, print_function @@ -28,7 +31,6 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase): self.api_client.nodes().update().execute.side_effect = arvados_effect self.cloud_client = mock.MagicMock(name='cloud_client') self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1) - self.cloud_client.is_cloud_exception = BaseComputeNodeDriver.is_cloud_exception def make_actor(self, arv_node=None): if not hasattr(self, 'timer'): @@ -90,27 +92,28 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase): self.make_actor() self.wait_for_assignment(self.setup_actor, 'cloud_node') - def test_unknown_basehttperror_not_retried(self): + def test_basehttperror_retried(self): self.make_mocks() self.cloud_client.create_node.side_effect = [ - BaseHTTPError(400, "Unknown"), + BaseHTTPError(500, "Try again"), self.cloud_client.create_node.return_value, ] self.make_actor() - finished = threading.Event() - self.setup_actor.subscribe(lambda _: finished.set()) - assert(finished.wait(self.TIMEOUT)) - self.assertEqual(0, self.cloud_client.post_create_node.call_count) + self.wait_for_assignment(self.setup_actor, 'cloud_node') + self.setup_actor.ping().get(self.TIMEOUT) + self.assertEqual(1, self.cloud_client.post_create_node.call_count) - def test_known_basehttperror_retried(self): + def test_instance_exceeded_not_retried(self): self.make_mocks() self.cloud_client.create_node.side_effect = [ BaseHTTPError(400, "InstanceLimitExceeded"), self.cloud_client.create_node.return_value, ] self.make_actor() - self.wait_for_assignment(self.setup_actor, 'cloud_node') - self.assertEqual(1, self.cloud_client.post_create_node.call_count) + done = self.FUTURE_CLASS() + self.setup_actor.subscribe(done.set) + done.get(self.TIMEOUT) + self.assertEqual(0, self.cloud_client.post_create_node.call_count) def test_failed_post_create_retried(self): self.make_mocks() @@ -149,6 +152,7 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase): self.api_client.nodes().create().execute.side_effect = retry_resp self.api_client.nodes().update().execute.side_effect = retry_resp self.wait_for_assignment(self.setup_actor, 'cloud_node') + self.setup_actor.ping().get(self.TIMEOUT) self.assertEqual(self.setup_actor.actor_ref.actor_urn, subscriber.call_args[0][0].actor_ref.actor_urn) @@ -205,17 +209,19 @@ class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin): self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy")) self.cloud_client.destroy_node.return_value = True self.make_actor(cancellable=True) - self.check_success_flag(False) + self.check_success_flag(False, 2) self.assertFalse(self.cloud_client.destroy_node.called) def test_uncancellable_shutdown(self, *mocks): self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy")) self.cloud_client.destroy_node.return_value = True self.make_actor(cancellable=False) - self.check_success_flag(True, 2) + self.check_success_flag(True, 4) self.assertTrue(self.cloud_client.destroy_node.called) def test_arvados_node_cleaned_after_shutdown(self, *mocks): + if len(mocks) == 1: + mocks[0].return_value = "drain\n" cloud_node = testutil.cloud_node_mock(62) arv_node = testutil.arvados_node_mock(62) self.make_mocks(cloud_node, arv_node) @@ -233,12 +239,15 @@ class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin): self.assertTrue(update_mock().execute.called) def test_arvados_node_not_cleaned_after_shutdown_cancelled(self, *mocks): + if len(mocks) == 1: + mocks[0].return_value = "idle\n" cloud_node = testutil.cloud_node_mock(61) arv_node = testutil.arvados_node_mock(61) self.make_mocks(cloud_node, arv_node, shutdown_open=False) self.cloud_client.destroy_node.return_value = False self.make_actor(cancellable=True) self.shutdown_actor.cancel_shutdown("test") + self.shutdown_actor.ping().get(self.TIMEOUT) self.check_success_flag(False, 2) self.assertFalse(self.arvados_client.nodes().update.called) @@ -273,11 +282,14 @@ class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin, class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin, unittest.TestCase): + ACTOR_CLASS = dispatch.ComputeNodeUpdateActor + def make_actor(self): self.driver = mock.MagicMock(name='driver_mock') - self.updater = dispatch.ComputeNodeUpdateActor.start(self.driver).proxy() + self.timer = mock.MagicMock(name='timer_mock') + self.updater = self.ACTOR_CLASS.start(self.driver, self.timer).proxy() - def test_node_sync(self): + def test_node_sync(self, *args): self.make_actor() cloud_node = testutil.cloud_node_mock() arv_node = testutil.arvados_node_mock() @@ -285,7 +297,7 @@ class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin, self.driver().sync_node.assert_called_with(cloud_node, arv_node) @testutil.no_sleep - def test_node_sync_error(self): + def test_node_sync_error(self, *args): self.make_actor() cloud_node = testutil.cloud_node_mock() arv_node = testutil.arvados_node_mock() @@ -333,13 +345,11 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, def test_in_state_when_no_state_available(self): self.make_actor(arv_node=testutil.arvados_node_mock( crunch_worker_state=None)) - print(self.node_actor.get_state().get()) self.assertTrue(self.node_state('idle')) def test_in_state_when_no_state_available_old(self): self.make_actor(arv_node=testutil.arvados_node_mock( crunch_worker_state=None, age=90000)) - print(self.node_actor.get_state().get()) self.assertTrue(self.node_state('down')) def test_in_idle_state(self): @@ -386,7 +396,7 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, def test_shutdown_without_arvados_node(self): self.make_actor(start_time=0) self.shutdowns._set_state(True, 600) - self.assertEquals((True, "node state is ('unpaired', 'open', 'boot exceeded', 'idle exceeded')"), + self.assertEquals((True, "node state is ('down', 'open', 'boot exceeded', 'idle exceeded')"), self.node_actor.shutdown_eligible().get(self.TIMEOUT)) def test_shutdown_missing(self):