#!/usr/bin/env python
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
from __future__ import absolute_import, print_function
self.api_client.nodes().update().execute.side_effect = arvados_effect
self.cloud_client = mock.MagicMock(name='cloud_client')
self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1)
- self.cloud_client.is_cloud_exception = BaseComputeNodeDriver.is_cloud_exception
def make_actor(self, arv_node=None):
if not hasattr(self, 'timer'):
self.make_actor()
self.wait_for_assignment(self.setup_actor, 'cloud_node')
- def test_unknown_basehttperror_not_retried(self):
+ def test_basehttperror_retried(self):
self.make_mocks()
self.cloud_client.create_node.side_effect = [
- BaseHTTPError(400, "Unknown"),
+ BaseHTTPError(500, "Try again"),
self.cloud_client.create_node.return_value,
]
self.make_actor()
- finished = threading.Event()
- self.setup_actor.subscribe(lambda _: finished.set())
- assert(finished.wait(self.TIMEOUT))
- self.assertEqual(0, self.cloud_client.post_create_node.call_count)
+ self.wait_for_assignment(self.setup_actor, 'cloud_node')
+ self.setup_actor.ping().get(self.TIMEOUT)
+ self.assertEqual(1, self.cloud_client.post_create_node.call_count)
- def test_known_basehttperror_retried(self):
+ def test_instance_exceeded_not_retried(self):
self.make_mocks()
self.cloud_client.create_node.side_effect = [
BaseHTTPError(400, "InstanceLimitExceeded"),
self.cloud_client.create_node.return_value,
]
self.make_actor()
- self.wait_for_assignment(self.setup_actor, 'cloud_node')
- self.assertEqual(1, self.cloud_client.post_create_node.call_count)
+ done = self.FUTURE_CLASS()
+ self.setup_actor.subscribe(done.set)
+ done.get(self.TIMEOUT)
+ self.assertEqual(0, self.cloud_client.post_create_node.call_count)
def test_failed_post_create_retried(self):
self.make_mocks()
self.api_client.nodes().create().execute.side_effect = retry_resp
self.api_client.nodes().update().execute.side_effect = retry_resp
self.wait_for_assignment(self.setup_actor, 'cloud_node')
+ self.setup_actor.ping().get(self.TIMEOUT)
self.assertEqual(self.setup_actor.actor_ref.actor_urn,
subscriber.call_args[0][0].actor_ref.actor_urn)
self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
self.cloud_client.destroy_node.return_value = True
self.make_actor(cancellable=True)
- self.check_success_flag(False)
+ self.check_success_flag(False, 2)
self.assertFalse(self.cloud_client.destroy_node.called)
def test_uncancellable_shutdown(self, *mocks):
self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
self.cloud_client.destroy_node.return_value = True
self.make_actor(cancellable=False)
- self.check_success_flag(True, 2)
+ self.check_success_flag(True, 4)
self.assertTrue(self.cloud_client.destroy_node.called)
def test_arvados_node_cleaned_after_shutdown(self, *mocks):
+ if len(mocks) == 1:
+ mocks[0].return_value = "drain\n"
cloud_node = testutil.cloud_node_mock(62)
arv_node = testutil.arvados_node_mock(62)
self.make_mocks(cloud_node, arv_node)
self.assertTrue(update_mock().execute.called)
def test_arvados_node_not_cleaned_after_shutdown_cancelled(self, *mocks):
+ if len(mocks) == 1:
+ mocks[0].return_value = "idle\n"
cloud_node = testutil.cloud_node_mock(61)
arv_node = testutil.arvados_node_mock(61)
self.make_mocks(cloud_node, arv_node, shutdown_open=False)
self.cloud_client.destroy_node.return_value = False
self.make_actor(cancellable=True)
self.shutdown_actor.cancel_shutdown("test")
+ self.shutdown_actor.ping().get(self.TIMEOUT)
self.check_success_flag(False, 2)
self.assertFalse(self.arvados_client.nodes().update.called)
class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
unittest.TestCase):
+ ACTOR_CLASS = dispatch.ComputeNodeUpdateActor
+
def make_actor(self):
self.driver = mock.MagicMock(name='driver_mock')
- self.updater = dispatch.ComputeNodeUpdateActor.start(self.driver).proxy()
+ self.timer = mock.MagicMock(name='timer_mock')
+ self.updater = self.ACTOR_CLASS.start(self.driver, self.timer).proxy()
- def test_node_sync(self):
+ def test_node_sync(self, *args):
self.make_actor()
cloud_node = testutil.cloud_node_mock()
arv_node = testutil.arvados_node_mock()
self.driver().sync_node.assert_called_with(cloud_node, arv_node)
@testutil.no_sleep
- def test_node_sync_error(self):
+ def test_node_sync_error(self, *args):
self.make_actor()
cloud_node = testutil.cloud_node_mock()
arv_node = testutil.arvados_node_mock()
def test_in_state_when_no_state_available(self):
self.make_actor(arv_node=testutil.arvados_node_mock(
crunch_worker_state=None))
- print(self.node_actor.get_state().get())
self.assertTrue(self.node_state('idle'))
def test_in_state_when_no_state_available_old(self):
self.make_actor(arv_node=testutil.arvados_node_mock(
crunch_worker_state=None, age=90000))
- print(self.node_actor.get_state().get())
self.assertTrue(self.node_state('down'))
def test_in_idle_state(self):
def test_shutdown_without_arvados_node(self):
self.make_actor(start_time=0)
self.shutdowns._set_state(True, 600)
- self.assertEquals((True, "node state is ('unpaired', 'open', 'boot exceeded', 'idle exceeded')"),
+ self.assertEquals((True, "node state is ('down', 'open', 'boot exceeded', 'idle exceeded')"),
self.node_actor.shutdown_eligible().get(self.TIMEOUT))
def test_shutdown_missing(self):