import pykka
import threading
+from libcloud.common.exceptions import BaseHTTPError
+
import arvnodeman.computenode.dispatch as dispatch
+from arvnodeman.computenode.driver import BaseComputeNodeDriver
from . import testutil
class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
self.api_client.nodes().update().execute.side_effect = arvados_effect
self.cloud_client = mock.MagicMock(name='cloud_client')
self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1)
+ self.cloud_client.is_cloud_exception = BaseComputeNodeDriver.is_cloud_exception
def make_actor(self, arv_node=None):
if not hasattr(self, 'timer'):
self.make_actor()
self.wait_for_assignment(self.setup_actor, 'cloud_node')
+ def test_unknown_basehttperror_not_retried(self):
+ self.make_mocks()
+ self.cloud_client.create_node.side_effect = [
+ BaseHTTPError(400, "Unknown"),
+ self.cloud_client.create_node.return_value,
+ ]
+ self.make_actor()
+ finished = threading.Event()
+ self.setup_actor.subscribe(lambda _: finished.set())
+ assert(finished.wait(self.TIMEOUT))
+ self.assertEqual(0, self.cloud_client.post_create_node.call_count)
+
+ def test_known_basehttperror_retried(self):
+ self.make_mocks()
+ self.cloud_client.create_node.side_effect = [
+ BaseHTTPError(400, "InstanceLimitExceeded"),
+ self.cloud_client.create_node.return_value,
+ ]
+ self.make_actor()
+ self.wait_for_assignment(self.setup_actor, 'cloud_node')
+ self.assertEqual(1, self.cloud_client.post_create_node.call_count)
+
def test_failed_post_create_retried(self):
self.make_mocks()
self.cloud_client.post_create_node.side_effect = [
else:
self.fail("success flag {} is not {}".format(last_flag, expected))
+ def test_cancellable_shutdown(self, *mocks):
+ self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
+ self.cloud_client.destroy_node.return_value = True
+ self.make_actor(cancellable=True)
+ self.check_success_flag(False)
+ self.assertFalse(self.cloud_client.destroy_node.called)
+
def test_uncancellable_shutdown(self, *mocks):
- self.make_mocks(shutdown_open=False)
- self.cloud_client.destroy_node.return_value = False
- self.make_actor(cancellable=False)
- self.check_success_flag(None, 0)
- self.shutdowns._set_state(True, 600)
+ self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
self.cloud_client.destroy_node.return_value = True
- self.check_success_flag(True)
+ self.make_actor(cancellable=False)
+ self.check_success_flag(True, 2)
+ self.assertTrue(self.cloud_client.destroy_node.called)
def test_arvados_node_cleaned_after_shutdown(self, *mocks):
cloud_node = testutil.cloud_node_mock(62)
self.check_success_flag(True)
self.assertTrue(self.cloud_client.destroy_node.called)
- def test_shutdown_retries_when_cloud_fails(self):
- self.make_mocks()
- self.cloud_client.destroy_node.return_value = False
- self.make_actor(start_time=0)
- self.assertIsNone(self.shutdown_actor.success.get(self.TIMEOUT))
- self.cloud_client.destroy_node.return_value = True
- self.check_success_flag(True)
-
- def test_shutdown_cancelled_when_cloud_fails_on_broken_node(self):
+ def test_shutdown_cancelled_when_destroy_node_fails(self):
self.make_mocks(node_broken=True)
self.cloud_client.destroy_node.return_value = False
self.make_actor(start_time=0)
self.check_success_flag(False, 2)
self.assertEqual(1, self.cloud_client.destroy_node.call_count)
- self.assertEqual(self.ACTOR_CLASS.NODE_BROKEN,
+ self.assertEqual(self.ACTOR_CLASS.DESTROY_FAILED,
self.shutdown_actor.cancel_reason.get(self.TIMEOUT))
def test_late_subscribe(self):
class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
unittest.TestCase):
+ ACTOR_CLASS = dispatch.ComputeNodeUpdateActor
+
def make_actor(self):
self.driver = mock.MagicMock(name='driver_mock')
- self.updater = dispatch.ComputeNodeUpdateActor.start(self.driver).proxy()
+ self.updater = self.ACTOR_CLASS.start(self.driver).proxy()
- def test_node_sync(self):
+ def test_node_sync(self, *args):
self.make_actor()
cloud_node = testutil.cloud_node_mock()
arv_node = testutil.arvados_node_mock()
self.driver().sync_node.assert_called_with(cloud_node, arv_node)
@testutil.no_sleep
- def test_node_sync_error(self):
+ def test_node_sync_error(self, *args):
self.make_actor()
cloud_node = testutil.cloud_node_mock()
arv_node = testutil.arvados_node_mock()