X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ffc7c1b20530dda13c1adf3aa48122ac881bbc29..b887df487c6cb465c3212e8ef99278350e58ff0a:/services/nodemanager/tests/test_computenode_dispatch.py diff --git a/services/nodemanager/tests/test_computenode_dispatch.py b/services/nodemanager/tests/test_computenode_dispatch.py index 7f6988dbe9..8d69ea958c 100644 --- a/services/nodemanager/tests/test_computenode_dispatch.py +++ b/services/nodemanager/tests/test_computenode_dispatch.py @@ -14,7 +14,7 @@ import arvnodeman.computenode.dispatch as dispatch from . import testutil class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase): - def make_mocks(self, arvados_effect=None, cloud_effect=None): + def make_mocks(self, arvados_effect=None): if arvados_effect is None: arvados_effect = [testutil.arvados_node_mock()] self.arvados_effect = arvados_effect @@ -48,14 +48,33 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase): self.assertEqual(self.cloud_client.create_node(), self.setup_actor.cloud_node.get(self.TIMEOUT)) - def test_failed_calls_retried(self): + def test_failed_arvados_calls_retried(self): self.make_mocks([ arverror.ApiError(httplib2.Response({'status': '500'}), ""), testutil.arvados_node_mock(), ]) self.make_actor() + self.wait_for_assignment(self.setup_actor, 'arvados_node') + + def test_failed_cloud_calls_retried(self): + self.make_mocks() + self.cloud_client.create_node.side_effect = [ + Exception("test cloud creation error"), + self.cloud_client.create_node.return_value, + ] + self.make_actor() self.wait_for_assignment(self.setup_actor, 'cloud_node') + def test_failed_post_create_retried(self): + self.make_mocks() + self.cloud_client.post_create_node.side_effect = [ + Exception("test cloud post-create error"), None] + self.make_actor() + done = self.FUTURE_CLASS() + self.setup_actor.subscribe(done.set) + done.get(self.TIMEOUT) + self.assertEqual(2, self.cloud_client.post_create_node.call_count) + def test_stop_when_no_cloud_node(self): self.make_mocks( arverror.ApiError(httplib2.Response({'status': '500'}), "")) @@ -106,14 +125,14 @@ class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin): self.cloud_node = cloud_node self.arvados_node = arvados_node - def make_actor(self): + def make_actor(self, cancellable=True): if not hasattr(self, 'timer'): self.make_mocks() monitor_actor = dispatch.ComputeNodeMonitorActor.start( self.cloud_node, time.time(), self.shutdowns, self.timer, self.updates, self.arvados_node) self.shutdown_actor = self.ACTOR_CLASS.start( - self.timer, self.cloud_client, monitor_actor).proxy() + self.timer, self.cloud_client, monitor_actor, cancellable).proxy() self.monitor_actor = monitor_actor.proxy() def check_success_flag(self, expected, allow_msg_count=1): @@ -126,6 +145,15 @@ class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin): else: self.fail("success flag {} is not {}".format(last_flag, expected)) + def test_uncancellable_shutdown(self, *mocks): + self.make_mocks(shutdown_open=False) + self.cloud_client.destroy_node.return_value = False + self.make_actor(cancellable=False) + self.check_success_flag(None, 0) + self.shutdowns._set_state(True, 600) + self.cloud_client.destroy_node.return_value = True + self.check_success_flag(True) + class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin, unittest.TestCase): @@ -199,28 +227,29 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, def test_in_state_when_unpaired(self): self.make_actor() - self.assertIsNone(self.node_state('idle', 'alloc')) + self.assertIsNone(self.node_state('idle', 'busy')) def test_in_state_when_pairing_stale(self): self.make_actor(arv_node=testutil.arvados_node_mock( job_uuid=None, age=90000)) - self.assertIsNone(self.node_state('idle', 'alloc')) + self.assertIsNone(self.node_state('idle', 'busy')) def test_in_state_when_no_state_available(self): - self.make_actor(arv_node=testutil.arvados_node_mock(info={})) - self.assertIsNone(self.node_state('idle', 'alloc')) + self.make_actor(arv_node=testutil.arvados_node_mock( + crunch_worker_state=None)) + self.assertIsNone(self.node_state('idle', 'busy')) def test_in_idle_state(self): self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None)) self.assertTrue(self.node_state('idle')) - self.assertFalse(self.node_state('alloc')) - self.assertTrue(self.node_state('idle', 'alloc')) + self.assertFalse(self.node_state('busy')) + self.assertTrue(self.node_state('idle', 'busy')) - def test_in_alloc_state(self): + def test_in_busy_state(self): self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True)) self.assertFalse(self.node_state('idle')) - self.assertTrue(self.node_state('alloc')) - self.assertTrue(self.node_state('idle', 'alloc')) + self.assertTrue(self.node_state('busy')) + self.assertTrue(self.node_state('idle', 'busy')) def test_init_shutdown_scheduling(self): self.make_actor() @@ -265,7 +294,8 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) def test_no_shutdown_when_node_state_unknown(self): - self.make_actor(5, testutil.arvados_node_mock(5, info={})) + self.make_actor(5, testutil.arvados_node_mock( + 5, crunch_worker_state=None)) self.shutdowns._set_state(True, 600) self.assertFalse(self.node_actor.shutdown_eligible().get(self.TIMEOUT)) @@ -289,6 +319,13 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin, self.assertIsNone( self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)) + def test_arvados_node_mismatch_first_ping_too_early(self): + self.make_actor(4) + arv_node = testutil.arvados_node_mock( + 4, first_ping_at='1971-03-02T14:15:16.1717282Z') + self.assertIsNone( + self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)) + def test_update_cloud_node(self): self.make_actor(1) self.make_mocks(2)