* Remove node blacklisting. Because of arvados node record reuse and assigning
compute node ids based on record uuid, it was possible to boot a new node
with the same id as a previously blacklisted node. Previously blacklisted
'broken' nodes are now considered 'down' when determining if it is necessary
to bring up new nodes.
* Failure to destroy a node is no longer retried by the shutdown actor. A
failure cancels the shutdown. The daemon is expected to schedule a new
shutdown attempt.
* Restored the concept of "cancellable" shutdown, which checks if the node is
still shutdown eligible before actually making the call to shut it down.
* Adjusted mocking behavior to fix tests which were producing suppressed
errors (visible in debug logging but not failing the test) when node sizes
were inconsistent between the wishlist, cloud_node objects, and
ServerCalculator.
"""
# Reasons for a shutdown to be cancelled.
WINDOW_CLOSED = "shutdown window closed"
"""
# Reasons for a shutdown to be cancelled.
WINDOW_CLOSED = "shutdown window closed"
- NODE_BROKEN = "cloud failed to shut down broken node"
+ DESTROY_FAILED = "destroy_node failed"
def __init__(self, timer_actor, cloud_client, arvados_client, node_monitor,
cancellable=True, retry_wait=1, max_retry_wait=180):
def __init__(self, timer_actor, cloud_client, arvados_client, node_monitor,
cancellable=True, retry_wait=1, max_retry_wait=180):
try:
return orig_func(self, *args, **kwargs)
except Exception as error:
try:
return orig_func(self, *args, **kwargs)
except Exception as error:
- self._logger.error("Actor error %s", error)
+ self._logger.error("Actor error %s", error, exc_info=True)
+ self._logger.debug("", exc_info=True)
self._later.cancel_shutdown("Unhandled exception %s" % error)
return finish_wrapper
@_cancel_on_exception
self._later.cancel_shutdown("Unhandled exception %s" % error)
return finish_wrapper
@_cancel_on_exception
+ if self.cancellable:
+ self._logger.info("Checking that node is still eligible for shutdown")
+ # Check that we still want to shut down the node.
+ eligible, reason = self._monitor.shutdown_eligible().get()
+ if not eligible:
+ self.cancel_shutdown("No longer eligible for shut down because %s" % reason)
+ return
+
self._logger.info("Starting shutdown")
arv_node = self._arvados_node()
self._logger.info("Starting shutdown")
arv_node = self._arvados_node()
- if not self._cloud.destroy_node(self.cloud_node):
- if self._cloud.broken(self.cloud_node):
- self._later.cancel_shutdown(self.NODE_BROKEN)
- return
+ if self._cloud.destroy_node(self.cloud_node):
+ self._logger.info("Shutdown success")
+ if arv_node:
+ self._later.clean_arvados_node(arv_node)
- # Force a retry.
- raise cloud_types.LibcloudError("destroy_node failed")
- self._logger.info("Shutdown success")
- if arv_node is None:
- self._finished(success_flag=True)
+ self._finished(success_flag=True)
- self._later.clean_arvados_node(arv_node)
+ self.cancel_shutdown(self.DESTROY_FAILED)
@ComputeNodeStateChangeBase._finish_on_exception
@RetryMixin._retry(config.ARVADOS_ERRORS)
@ComputeNodeStateChangeBase._finish_on_exception
@RetryMixin._retry(config.ARVADOS_ERRORS)
if self.arvados_node is None:
return 'unpaired'
if self.arvados_node is None:
return 'unpaired'
+ # This node is indicated as non-functioning by the cloud
+ if self._cloud.broken(self.cloud_node):
+ return 'down'
+
state = self.arvados_node['crunch_worker_state']
# If state information is not available because it is missing or the
state = self.arvados_node['crunch_worker_state']
# If state information is not available because it is missing or the
def __init__(self):
self.nodes = {}
self.orphans = {}
def __init__(self):
self.nodes = {}
self.orphans = {}
- self._blacklist = set()
# Proxy the methods listed below to self.nodes.
def _proxy_method(name):
# Proxy the methods listed below to self.nodes.
def _proxy_method(name):
def add(self, record):
self.nodes[self.record_key(record)] = record
def add(self, record):
self.nodes[self.record_key(record)] = record
- def blacklist(self, key):
- self._blacklist.add(key)
-
def update_record(self, key, item):
setattr(self.nodes[key], self.RECORD_ATTR, item)
def update_record(self, key, item):
setattr(self.nodes[key], self.RECORD_ATTR, item)
unseen = set(self.nodes.iterkeys())
for item in response:
key = self.item_key(item)
unseen = set(self.nodes.iterkeys())
for item in response:
key = self.item_key(item)
- if key in self._blacklist:
- continue
- elif key in unseen:
unseen.remove(key)
self.update_record(key, item)
else:
unseen.remove(key)
self.update_record(key, item)
else:
if hasattr(record.cloud_node, "_nodemanager_recently_booted"):
self.cloud_nodes.add(record)
else:
if hasattr(record.cloud_node, "_nodemanager_recently_booted"):
self.cloud_nodes.add(record)
else:
+ # Node disappeared from the cloud node list. Stop the monitor
+ # actor if necessary and forget about the node.
+ if record.actor:
+ try:
+ record.actor.stop()
+ except pykka.ActorDeadError:
+ pass
+ record.actor = None
record.cloud_node = None
def _register_arvados_node(self, key, arv_node):
record.cloud_node = None
def _register_arvados_node(self, key, arv_node):
def node_finished_shutdown(self, shutdown_actor):
try:
def node_finished_shutdown(self, shutdown_actor):
try:
- cloud_node, success, cancel_reason = self._get_actor_attrs(
- shutdown_actor, 'cloud_node', 'success', 'cancel_reason')
+ cloud_node, success = self._get_actor_attrs(
+ shutdown_actor, 'cloud_node', 'success')
except pykka.ActorDeadError:
return
cloud_node_id = cloud_node.id
record = self.cloud_nodes[cloud_node_id]
shutdown_actor.stop()
except pykka.ActorDeadError:
return
cloud_node_id = cloud_node.id
record = self.cloud_nodes[cloud_node_id]
shutdown_actor.stop()
+ record.shutdown_actor = None
+
- if cancel_reason == self._node_shutdown.NODE_BROKEN:
- self.cloud_nodes.blacklist(cloud_node_id)
- record.shutdown_actor = None
- else:
- # If the node went from being booted to being shut down without ever
- # appearing in the cloud node list, it will have the
- # _nodemanager_recently_booted tag, so get rid of it so that the node
- # can be forgotten completely.
- if hasattr(self.cloud_nodes[cloud_node_id].cloud_node, "_nodemanager_recently_booted"):
- del self.cloud_nodes[cloud_node_id].cloud_node._nodemanager_recently_booted
+ return
+
+ # Shutdown was successful, so stop the monitor actor, otherwise it
+ # will keep offering the node as a candidate for shutdown.
+ record.actor.stop()
+ record.actor = None
+
+ # If the node went from being booted to being shut down without ever
+ # appearing in the cloud node list, it will have the
+ # _nodemanager_recently_booted tag, so get rid of it so that the node
+ # can be forgotten completely.
+ if hasattr(record.cloud_node, "_nodemanager_recently_booted"):
+ del record.cloud_node._nodemanager_recently_booted
def shutdown(self):
self._logger.info("Shutting down after signal.")
def shutdown(self):
self._logger.info("Shutting down after signal.")
else:
self.fail("success flag {} is not {}".format(last_flag, expected))
else:
self.fail("success flag {} is not {}".format(last_flag, expected))
+ def test_cancellable_shutdown(self, *mocks):
+ self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
+ self.cloud_client.destroy_node.return_value = True
+ self.make_actor(cancellable=True)
+ self.check_success_flag(False)
+ self.assertFalse(self.cloud_client.destroy_node.called)
+
def test_uncancellable_shutdown(self, *mocks):
def test_uncancellable_shutdown(self, *mocks):
- self.make_mocks(shutdown_open=False)
- self.cloud_client.destroy_node.return_value = False
- self.make_actor(cancellable=False)
- self.check_success_flag(None, 0)
- self.shutdowns._set_state(True, 600)
+ self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
self.cloud_client.destroy_node.return_value = True
self.cloud_client.destroy_node.return_value = True
- self.check_success_flag(True)
+ self.make_actor(cancellable=False)
+ self.check_success_flag(True, 2)
+ self.assertTrue(self.cloud_client.destroy_node.called)
def test_arvados_node_cleaned_after_shutdown(self, *mocks):
cloud_node = testutil.cloud_node_mock(62)
def test_arvados_node_cleaned_after_shutdown(self, *mocks):
cloud_node = testutil.cloud_node_mock(62)
self.check_success_flag(True)
self.assertTrue(self.cloud_client.destroy_node.called)
self.check_success_flag(True)
self.assertTrue(self.cloud_client.destroy_node.called)
- def test_shutdown_retries_when_cloud_fails(self):
- self.make_mocks()
- self.cloud_client.destroy_node.return_value = False
- self.make_actor(start_time=0)
- self.assertIsNone(self.shutdown_actor.success.get(self.TIMEOUT))
- self.cloud_client.destroy_node.return_value = True
- self.check_success_flag(True)
-
- def test_shutdown_cancelled_when_cloud_fails_on_broken_node(self):
+ def test_shutdown_cancelled_when_destroy_node_fails(self):
self.make_mocks(node_broken=True)
self.cloud_client.destroy_node.return_value = False
self.make_actor(start_time=0)
self.check_success_flag(False, 2)
self.assertEqual(1, self.cloud_client.destroy_node.call_count)
self.make_mocks(node_broken=True)
self.cloud_client.destroy_node.return_value = False
self.make_actor(start_time=0)
self.check_success_flag(False, 2)
self.assertEqual(1, self.cloud_client.destroy_node.call_count)
- self.assertEqual(self.ACTOR_CLASS.NODE_BROKEN,
+ self.assertEqual(self.ACTOR_CLASS.DESTROY_FAILED,
self.shutdown_actor.cancel_reason.get(self.TIMEOUT))
def test_late_subscribe(self):
self.shutdown_actor.cancel_reason.get(self.TIMEOUT))
def test_late_subscribe(self):
proc_mock.return_value = 'drain\n'
super(SLURMComputeNodeShutdownActorTestCase,
self).test_arvados_node_cleaned_after_shutdown()
proc_mock.return_value = 'drain\n'
super(SLURMComputeNodeShutdownActorTestCase,
self).test_arvados_node_cleaned_after_shutdown()
+
+ def test_cancellable_shutdown(self, proc_mock):
+ proc_mock.return_value = 'other\n'
+ super(SLURMComputeNodeShutdownActorTestCase,
+ self).test_cancellable_shutdown()
+
+ def test_uncancellable_shutdown(self, proc_mock):
+ proc_mock.return_value = 'other\n'
+ super(SLURMComputeNodeShutdownActorTestCase,
+ self).test_uncancellable_shutdown()
def test_node_found_after_timeout_has_fixed_size(self):
size = testutil.MockSize(4)
node_props = {'hardwareProfile': {'vmSize': size.id}}
def test_node_found_after_timeout_has_fixed_size(self):
size = testutil.MockSize(4)
node_props = {'hardwareProfile': {'vmSize': size.id}}
- cloud_node = testutil.cloud_node_mock(
- size=None, tags={'arvados-class': 'test'}, properties=node_props)
+ cloud_node = testutil.cloud_node_mock(tags={'arvados-class': 'test'}, properties=node_props)
+ cloud_node.size = None
self.check_node_found_after_timeout_has_fixed_size(
size, cloud_node, {'tag_arvados-class': 'test'})
self.check_node_found_after_timeout_has_fixed_size(
size, cloud_node, {'tag_arvados-class': 'test'})
return mock_actor
def make_daemon(self, cloud_nodes=[], arvados_nodes=[], want_sizes=[],
return mock_actor
def make_daemon(self, cloud_nodes=[], arvados_nodes=[], want_sizes=[],
- avail_sizes=[(testutil.MockSize(1), {"cores": 1})],
min_nodes=0, max_nodes=8,
shutdown_windows=[54, 5, 1],
max_total_price=None):
for name in ['cloud_nodes', 'arvados_nodes', 'server_wishlist']:
setattr(self, name + '_poller', mock.MagicMock(name=name + '_mock'))
min_nodes=0, max_nodes=8,
shutdown_windows=[54, 5, 1],
max_total_price=None):
for name in ['cloud_nodes', 'arvados_nodes', 'server_wishlist']:
setattr(self, name + '_poller', mock.MagicMock(name=name + '_mock'))
+
+ if not avail_sizes:
+ if cloud_nodes or want_sizes:
+ avail_sizes=[(c.size, {"cores": int(c.id)}) for c in cloud_nodes] + [(s, {"cores": 1}) for s in want_sizes]
+ else:
+ avail_sizes=[(testutil.MockSize(1), {"cores": 1})]
+
self.arv_factory = mock.MagicMock(name='arvados_mock')
api_client = mock.MagicMock(name='api_client')
api_client.nodes().create().execute.side_effect = [testutil.arvados_node_mock(1),
self.arv_factory = mock.MagicMock(name='arvados_mock')
api_client = mock.MagicMock(name='api_client')
api_client.nodes().create().execute.side_effect = [testutil.arvados_node_mock(1),
self.cloud_updates = mock.MagicMock(name='updates_mock')
self.timer = testutil.MockTimer(deliver_immediately=False)
self.cloud_factory().node_id.side_effect = lambda node: node.id
self.cloud_updates = mock.MagicMock(name='updates_mock')
self.timer = testutil.MockTimer(deliver_immediately=False)
self.cloud_factory().node_id.side_effect = lambda node: node.id
+ self.cloud_factory().broken.return_value = False
self.node_setup = mock.MagicMock(name='setup_mock')
self.node_setup.start.side_effect = self.mock_node_start
self.node_setup = mock.MagicMock(name='setup_mock')
self.node_setup.start.side_effect = self.mock_node_start
def test_arvados_node_un_and_re_paired(self):
# We need to create the Arvados node mock after spinning up the daemon
# to make sure it's new enough to pair with the cloud node.
def test_arvados_node_un_and_re_paired(self):
# We need to create the Arvados node mock after spinning up the daemon
# to make sure it's new enough to pair with the cloud node.
- self.make_daemon([testutil.cloud_node_mock(3)], arvados_nodes=None)
+ self.make_daemon(cloud_nodes=[testutil.cloud_node_mock(3)],
+ arvados_nodes=None)
arv_node = testutil.arvados_node_mock(3)
self.daemon.update_arvados_nodes([arv_node]).get(self.TIMEOUT)
self.check_monitors_arvados_nodes(arv_node)
arv_node = testutil.arvados_node_mock(3)
self.daemon.update_arvados_nodes([arv_node]).get(self.TIMEOUT)
self.check_monitors_arvados_nodes(arv_node)
def test_old_arvados_node_not_double_assigned(self):
arv_node = testutil.arvados_node_mock(3, age=9000)
size = testutil.MockSize(3)
def test_old_arvados_node_not_double_assigned(self):
arv_node = testutil.arvados_node_mock(3, age=9000)
size = testutil.MockSize(3)
- self.make_daemon(arvados_nodes=[arv_node], avail_sizes=[(size, {"cores":1})])
+ self.make_daemon(arvados_nodes=[arv_node],
+ avail_sizes=[(size, {"cores":1})])
self.daemon.update_server_wishlist([size]).get(self.TIMEOUT)
self.daemon.update_server_wishlist([size, size]).get(self.TIMEOUT)
self.stop_proxy(self.daemon)
self.daemon.update_server_wishlist([size]).get(self.TIMEOUT)
self.daemon.update_server_wishlist([size, size]).get(self.TIMEOUT)
self.stop_proxy(self.daemon)
self.assertIn(None, used_nodes)
def test_node_count_satisfied(self):
self.assertIn(None, used_nodes)
def test_node_count_satisfied(self):
- self.make_daemon([testutil.cloud_node_mock()],
+ self.make_daemon(cloud_nodes=[testutil.cloud_node_mock(1)],
want_sizes=[testutil.MockSize(1)])
self.stop_proxy(self.daemon)
self.assertFalse(self.node_setup.start.called)
def test_dont_count_missing_as_busy(self):
size = testutil.MockSize(1)
want_sizes=[testutil.MockSize(1)])
self.stop_proxy(self.daemon)
self.assertFalse(self.node_setup.start.called)
def test_dont_count_missing_as_busy(self):
size = testutil.MockSize(1)
- self.make_daemon(cloud_nodes=[testutil.cloud_node_mock(1),
- testutil.cloud_node_mock(2)],
+ self.make_daemon(cloud_nodes=[testutil.cloud_node_mock(1, size=size),
+ testutil.cloud_node_mock(2, size=size)],
arvados_nodes=[testutil.arvados_node_mock(1),
arvados_nodes=[testutil.arvados_node_mock(1),
- testutil.arvados_node_mock(2,
- last_ping_at='1970-01-01T01:02:03.04050607Z')],
+ testutil.arvados_node_mock(
+ 2,
+ last_ping_at='1970-01-01T01:02:03.04050607Z')],
want_sizes=[size, size])
self.stop_proxy(self.daemon)
self.assertTrue(self.node_setup.start.called)
def test_missing_counts_towards_max(self):
size = testutil.MockSize(1)
want_sizes=[size, size])
self.stop_proxy(self.daemon)
self.assertTrue(self.node_setup.start.called)
def test_missing_counts_towards_max(self):
size = testutil.MockSize(1)
- self.make_daemon(cloud_nodes=[testutil.cloud_node_mock(1),
- testutil.cloud_node_mock(2)],
+ self.make_daemon(cloud_nodes=[testutil.cloud_node_mock(1, size=size),
+ testutil.cloud_node_mock(2, size=size)],
arvados_nodes=[testutil.arvados_node_mock(1),
testutil.arvados_node_mock(2, last_ping_at='1970-01-01T01:02:03.04050607Z')],
want_sizes=[size, size],
arvados_nodes=[testutil.arvados_node_mock(1),
testutil.arvados_node_mock(2, last_ping_at='1970-01-01T01:02:03.04050607Z')],
want_sizes=[size, size],
def test_excess_counts_missing(self):
size = testutil.MockSize(1)
def test_excess_counts_missing(self):
size = testutil.MockSize(1)
- cloud_nodes = [testutil.cloud_node_mock(1), testutil.cloud_node_mock(2)]
+ cloud_nodes = [testutil.cloud_node_mock(1, size=size), testutil.cloud_node_mock(2, size=size)]
self.make_daemon(cloud_nodes=cloud_nodes,
arvados_nodes=[testutil.arvados_node_mock(1),
testutil.arvados_node_mock(2, last_ping_at='1970-01-01T01:02:03.04050607Z')],
self.make_daemon(cloud_nodes=cloud_nodes,
arvados_nodes=[testutil.arvados_node_mock(1),
testutil.arvados_node_mock(2, last_ping_at='1970-01-01T01:02:03.04050607Z')],
def test_missing_shutdown_not_excess(self):
size = testutil.MockSize(1)
def test_missing_shutdown_not_excess(self):
size = testutil.MockSize(1)
- cloud_nodes = [testutil.cloud_node_mock(1), testutil.cloud_node_mock(2)]
+ cloud_nodes = [testutil.cloud_node_mock(1, size=size), testutil.cloud_node_mock(2, size=size)]
self.make_daemon(cloud_nodes=cloud_nodes,
arvados_nodes=[testutil.arvados_node_mock(1),
testutil.arvados_node_mock(2, last_ping_at='1970-01-01T01:02:03.04050607Z')],
self.make_daemon(cloud_nodes=cloud_nodes,
arvados_nodes=[testutil.arvados_node_mock(1),
testutil.arvados_node_mock(2, last_ping_at='1970-01-01T01:02:03.04050607Z')],
for call in self.node_setup.start.call_args_list])
def test_no_new_node_when_ge_min_nodes_busy(self):
for call in self.node_setup.start.call_args_list])
def test_no_new_node_when_ge_min_nodes_busy(self):
- cloud_nodes = [testutil.cloud_node_mock(n) for n in range(1, 4)]
+ size = testutil.MockSize(2)
+ cloud_nodes = [testutil.cloud_node_mock(n, size=size) for n in range(1, 4)]
arv_nodes = [testutil.arvados_node_mock(n, job_uuid=True)
for n in range(1, 4)]
self.make_daemon(cloud_nodes, arv_nodes, [], min_nodes=2)
arv_nodes = [testutil.arvados_node_mock(n, job_uuid=True)
for n in range(1, 4)]
self.make_daemon(cloud_nodes, arv_nodes, [], min_nodes=2)
self.assertEqual(0, self.node_setup.start.call_count)
def test_no_new_node_when_max_nodes_busy(self):
self.assertEqual(0, self.node_setup.start.call_count)
def test_no_new_node_when_max_nodes_busy(self):
- self.make_daemon([testutil.cloud_node_mock(3)],
- [testutil.arvados_node_mock(3, job_uuid=True)],
- [testutil.MockSize(3)],
+ size = testutil.MockSize(3)
+ self.make_daemon(cloud_nodes=[testutil.cloud_node_mock(3)],
+ arvados_nodes=[testutil.arvados_node_mock(3, job_uuid=True)],
+ want_sizes=[size],
max_nodes=1)
self.stop_proxy(self.daemon)
self.assertFalse(self.node_setup.start.called)
max_nodes=1)
self.stop_proxy(self.daemon)
self.assertFalse(self.node_setup.start.called)
def start_node_boot(self, cloud_node=None, arv_node=None, id_num=1):
if cloud_node is None:
cloud_node = testutil.cloud_node_mock(id_num)
def start_node_boot(self, cloud_node=None, arv_node=None, id_num=1):
if cloud_node is None:
cloud_node = testutil.cloud_node_mock(id_num)
+ id_num = int(cloud_node.id)
if arv_node is None:
arv_node = testutil.arvados_node_mock(id_num)
self.make_daemon(want_sizes=[testutil.MockSize(id_num)],
if arv_node is None:
arv_node = testutil.arvados_node_mock(id_num)
self.make_daemon(want_sizes=[testutil.MockSize(id_num)],
self.daemon.update_cloud_nodes([cloud_node])
self.monitor_list()[0].proxy().cloud_node_start_time = time.time()-1801
self.daemon.update_server_wishlist(
self.daemon.update_cloud_nodes([cloud_node])
self.monitor_list()[0].proxy().cloud_node_start_time = time.time()-1801
self.daemon.update_server_wishlist(
- [testutil.MockSize(1)]).get(self.TIMEOUT)
+ [testutil.MockSize(4)]).get(self.TIMEOUT)
self.stop_proxy(self.daemon)
self.assertEqual(2, self.node_setup.start.call_count)
self.stop_proxy(self.daemon)
self.assertEqual(2, self.node_setup.start.call_count)
self.assertTrue(self.node_shutdown.start.called)
def test_shutdown_declined_when_idle_and_job_queued(self):
self.assertTrue(self.node_shutdown.start.called)
def test_shutdown_declined_when_idle_and_job_queued(self):
- cloud_nodes = [testutil.cloud_node_mock(n) for n in [3, 4]]
+ size = testutil.MockSize(1)
+ cloud_nodes = [testutil.cloud_node_mock(n, size=size) for n in [3, 4]]
arv_nodes = [testutil.arvados_node_mock(3, job_uuid=True),
testutil.arvados_node_mock(4, job_uuid=None)]
arv_nodes = [testutil.arvados_node_mock(3, job_uuid=True),
testutil.arvados_node_mock(4, job_uuid=None)]
- self.make_daemon(cloud_nodes, arv_nodes, [testutil.MockSize(1)])
+ self.make_daemon(cloud_nodes, arv_nodes, [size])
self.assertEqual(2, self.alive_monitor_count())
for mon_ref in self.monitor_list():
monitor = mon_ref.proxy()
self.assertEqual(2, self.alive_monitor_count())
for mon_ref in self.monitor_list():
monitor = mon_ref.proxy()
self.daemon.node_finished_shutdown(self.last_shutdown).get(self.TIMEOUT)
self.assertEqual(0, self.alive_monitor_count())
self.daemon.node_finished_shutdown(self.last_shutdown).get(self.TIMEOUT)
self.assertEqual(0, self.alive_monitor_count())
- def test_broken_node_blackholed_after_cancelled_shutdown(self):
+ def test_broken_node_not_counted(self):
size = testutil.MockSize(8)
cloud_node = testutil.cloud_node_mock(8, size=size)
wishlist = [size]
size = testutil.MockSize(8)
cloud_node = testutil.cloud_node_mock(8, size=size)
wishlist = [size]
shutdown_proxy = self.node_shutdown.start().proxy
shutdown_proxy().cloud_node.get.return_value = cloud_node
shutdown_proxy().success.get.return_value = False
shutdown_proxy = self.node_shutdown.start().proxy
shutdown_proxy().cloud_node.get.return_value = cloud_node
shutdown_proxy().success.get.return_value = False
- shutdown_proxy().cancel_reason.get.return_value = self.node_shutdown.NODE_BROKEN
+ self.cloud_factory().broken.return_value = True
self.daemon.update_server_wishlist([]).get(self.TIMEOUT)
self.daemon.node_can_shutdown(monitor).get(self.TIMEOUT)
self.daemon.node_finished_shutdown(shutdown_proxy()).get(self.TIMEOUT)
self.daemon.update_server_wishlist([]).get(self.TIMEOUT)
self.daemon.node_can_shutdown(monitor).get(self.TIMEOUT)
self.daemon.node_finished_shutdown(shutdown_proxy()).get(self.TIMEOUT)
time.sleep(2)
return True
time.sleep(2)
return True
-class ActorUnhandledExceptionTest(unittest.TestCase):
+class ActorUnhandledExceptionTest(testutil.ActorTestMixin, unittest.TestCase):
def test_fatal_error(self):
for e in (MemoryError(), threading.ThreadError(), OSError(errno.ENOMEM, "")):
with mock.patch('os.kill') as kill_mock:
def test_fatal_error(self):
for e in (MemoryError(), threading.ThreadError(), OSError(errno.ENOMEM, "")):
with mock.patch('os.kill') as kill_mock:
act.actor_ref.stop(block=True)
self.assertFalse(kill_mock.called)
act.actor_ref.stop(block=True)
self.assertFalse(kill_mock.called)
-class WatchdogActorTest(unittest.TestCase):
+class WatchdogActorTest(testutil.ActorTestMixin, unittest.TestCase):
@mock.patch('os.kill')
def test_time_timout(self, kill_mock):
act = BogusActor.start(OSError(errno.ENOENT, ""))
@mock.patch('os.kill')
def test_time_timout(self, kill_mock):
act = BogusActor.start(OSError(errno.ENOENT, ""))
self.monitor = self.TEST_CLASS.start(
self.client, self.timer, *args, **kwargs).proxy()
self.monitor = self.TEST_CLASS.start(
self.client, self.timer, *args, **kwargs).proxy()
-def cloud_node_mock(node_num=99, size=MockSize(1), **extra):
+def cloud_node_mock(node_num=99, size=None, **extra):
+ if size is None:
+ size = MockSize(node_num)
node = mock.NonCallableMagicMock(
['id', 'name', 'state', 'public_ips', 'private_ips', 'driver', 'size',
'image', 'extra'],
node = mock.NonCallableMagicMock(
['id', 'name', 'state', 'public_ips', 'private_ips', 'driver', 'size',
'image', 'extra'],