X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f116df2ef0439328044ccc46a90d7fc86541ace8..055ac1cc91121d3f27405d4f45455a8de5a0e57e:/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py index 6d5c223fac..1016fda20e 100644 --- a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py +++ b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py @@ -10,7 +10,7 @@ import libcloud.common.types as cloud_types import pykka from .. import \ - arvados_node_fqdn, arvados_node_mtime, arvados_timestamp, timestamp_fresh + arvados_node_fqdn, arvados_node_mtime, arvados_timestamp, timestamp_fresh, arvados_node_missing from ...clientactor import _notify_subscribers from ... import config @@ -88,6 +88,7 @@ class ComputeNodeStateChangeBase(config.actor_class): 'slot_number': None, 'first_ping_at': None, 'last_ping_at': None, + 'properties': {}, 'info': {'ec2_instance_id': None, 'last_action': explanation}}, ).execute() @@ -134,6 +135,30 @@ class ComputeNodeSetupActor(ComputeNodeStateChangeBase): self.cloud_node = self._cloud.create_node(self.cloud_size, self.arvados_node) self._logger.info("Cloud node %s created.", self.cloud_node.id) + self._later.update_arvados_node_properties() + + @ComputeNodeStateChangeBase._retry(config.ARVADOS_ERRORS) + def update_arvados_node_properties(self): + """Tell Arvados some details about the cloud node. + + Currently we only include size/price from our request, which + we already knew before create_cloud_node(), but doing it here + gives us an opportunity to provide more detail from + self.cloud_node, too. + """ + self.arvados_node['properties']['cloud_node'] = { + # Note this 'size' is the node size we asked the cloud + # driver to create -- not necessarily equal to the size + # reported by the cloud driver for the node that was + # created. + 'size': self.cloud_size.id, + 'price': self.cloud_size.price, + } + self.arvados_node = self._arvados.nodes().update( + uuid=self.arvados_node['uuid'], + body={'properties': self.arvados_node['properties']}, + ).execute() + self._logger.info("%s updated properties.", self.arvados_node['uuid']) self._later.post_create() @ComputeNodeStateChangeBase._retry() @@ -154,6 +179,10 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): This actor simply destroys a cloud node, retrying as needed. """ + # Reasons for a shutdown to be cancelled. + WINDOW_CLOSED = "shutdown window closed" + NODE_BROKEN = "cloud failed to shut down broken node" + def __init__(self, timer_actor, cloud_client, arvados_client, node_monitor, cancellable=True, retry_wait=1, max_retry_wait=180): # If a ShutdownActor is cancellable, it will ask the @@ -167,6 +196,7 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): self._monitor = node_monitor.proxy() self.cloud_node = self._monitor.cloud_node.get() self.cancellable = cancellable + self.cancel_reason = None self.success = None def on_start(self): @@ -180,7 +210,10 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): self.success = success_flag return super(ComputeNodeShutdownActor, self)._finished() - def cancel_shutdown(self): + def cancel_shutdown(self, reason): + self.cancel_reason = reason + self._logger.info("Cloud node %s shutdown cancelled: %s.", + self.cloud_node.id, reason) self._finished(success_flag=False) def _stop_if_window_closed(orig_func): @@ -188,10 +221,7 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): def stop_wrapper(self, *args, **kwargs): if (self.cancellable and (not self._monitor.shutdown_eligible().get())): - self._logger.info( - "Cloud node %s shutdown cancelled - no longer eligible.", - self.cloud_node.id) - self._later.cancel_shutdown() + self._later.cancel_shutdown(self.WINDOW_CLOSED) return None else: return orig_func(self, *args, **kwargs) @@ -201,8 +231,11 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): @ComputeNodeStateChangeBase._retry() def shutdown_node(self): if not self._cloud.destroy_node(self.cloud_node): - # Force a retry. - raise cloud_types.LibcloudError("destroy_node failed") + if self._cloud.broken(self.cloud_node): + self._later.cancel_shutdown(self.NODE_BROKEN) + else: + # Force a retry. + raise cloud_types.LibcloudError("destroy_node failed") self._logger.info("Cloud node %s shut down.", self.cloud_node.id) arv_node = self._arvados_node() if arv_node is None: @@ -273,8 +306,10 @@ class ComputeNodeMonitorActor(config.actor_class): for shutdown. """ def __init__(self, cloud_node, cloud_node_start_time, shutdown_timer, - cloud_fqdn_func, timer_actor, update_actor, arvados_node=None, - poll_stale_after=600, node_stale_after=3600): + cloud_fqdn_func, timer_actor, update_actor, cloud_client, + arvados_node=None, poll_stale_after=600, node_stale_after=3600, + boot_fail_after=1800 + ): super(ComputeNodeMonitorActor, self).__init__() self._later = self.actor_ref.proxy() self._logger = logging.getLogger('arvnodeman.computenode') @@ -283,10 +318,12 @@ class ComputeNodeMonitorActor(config.actor_class): self._cloud_node_fqdn = cloud_fqdn_func self._timer = timer_actor self._update = update_actor + self._cloud = cloud_client self.cloud_node = cloud_node self.cloud_node_start_time = cloud_node_start_time self.poll_stale_after = poll_stale_after self.node_stale_after = node_stale_after + self.boot_fail_after = boot_fail_after self.subscribers = set() self.arvados_node = None self._later.update_arvados_node(arvados_node) @@ -321,13 +358,21 @@ class ComputeNodeMonitorActor(config.actor_class): def shutdown_eligible(self): if not self._shutdowns.window_open(): return False - elif self.arvados_node is None: - # If this is a new, unpaired node, it's eligible for - # shutdown--we figure there was an error during bootstrap. - return timestamp_fresh(self.cloud_node_start_time, - self.node_stale_after) - else: - return self.in_state('idle') + if self.arvados_node is None: + # Node is unpaired. + # If it hasn't pinged Arvados after boot_fail seconds, shut it down + return not timestamp_fresh(self.cloud_node_start_time, self.boot_fail_after) + missing = arvados_node_missing(self.arvados_node, self.node_stale_after) + if missing and self._cloud.broken(self.cloud_node): + # Node is paired, but Arvados says it is missing and the cloud says the node + # is in an error state, so shut it down. + return True + if missing is None and self._cloud.broken(self.cloud_node): + self._logger.warning( + "cloud reports broken node, but paired node %s never pinged " + "(bug?) -- skipped check for node_stale_after", + self.arvados_node['uuid']) + return self.in_state('idle') def consider_shutdown(self): next_opening = self._shutdowns.next_opening()