X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/caacfc031998dc73cd2f4c767e1a746b7783d379..ea47e67af0e7528f0bcb23f3b34019b308eaa68a:/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py index 8674f168f7..63dac3f0ed 100644 --- a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py +++ b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, print_function import functools import logging import time +import re import libcloud.common.types as cloud_types import pykka @@ -16,6 +17,8 @@ from ...clientactor import _notify_subscribers from ... import config from .transitions import transitions +QuotaExceeded = "QuotaExceeded" + class ComputeNodeStateChangeBase(config.actor_class, RetryMixin): """Base class for actors that change a compute node's state. @@ -96,6 +99,7 @@ class ComputeNodeSetupActor(ComputeNodeStateChangeBase): self.cloud_size = cloud_size self.arvados_node = None self.cloud_node = None + self.error = None if arvados_node is None: self._later.create_arvados_node() else: @@ -119,10 +123,28 @@ class ComputeNodeSetupActor(ComputeNodeStateChangeBase): def create_cloud_node(self): self._logger.info("Sending create_node request for node size %s.", self.cloud_size.name) - self.cloud_node = self._cloud.create_node(self.cloud_size, - self.arvados_node) - if not self.cloud_node.size: - self.cloud_node.size = self.cloud_size + try: + self.cloud_node = self._cloud.create_node(self.cloud_size, + self.arvados_node) + except Exception as e: + # The set of possible error codes / messages isn't documented for + # all clouds, so use a keyword heuristic to determine if the + # failure is likely due to a quota. + if re.search(r'(exceed|quota|limit)', e.message, re.I): + self.error = QuotaExceeded + self._logger.warning("Quota exceeded: %s", e) + self._finished() + return + else: + raise + + # The information included in the node size object we get from libcloud + # is inconsistent between cloud drivers. Replace libcloud NodeSize + # object with compatible CloudSizeWrapper object which merges the size + # info reported from the cloud with size information from the + # configuration file. + self.cloud_node.size = self.cloud_size + self._logger.info("Cloud node %s created.", self.cloud_node.id) self._later.update_arvados_node_properties() @@ -171,7 +193,7 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): """ # Reasons for a shutdown to be cancelled. WINDOW_CLOSED = "shutdown window closed" - NODE_BROKEN = "cloud failed to shut down broken node" + DESTROY_FAILED = "destroy_node failed" def __init__(self, timer_actor, cloud_client, arvados_client, node_monitor, cancellable=True, retry_wait=1, max_retry_wait=180): @@ -204,7 +226,7 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): self.success = success_flag return super(ComputeNodeShutdownActor, self)._finished() - def cancel_shutdown(self, reason): + def cancel_shutdown(self, reason, **kwargs): self.cancel_reason = reason self._logger.info("Shutdown cancelled: %s.", reason) self._finished(success_flag=False) @@ -216,26 +238,32 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): return orig_func(self, *args, **kwargs) except Exception as error: self._logger.error("Actor error %s", error) - self._later.cancel_shutdown("Unhandled exception %s" % error) + self._logger.debug("", exc_info=True) + self._later.cancel_shutdown("Unhandled exception %s" % error, try_resume=False) return finish_wrapper @_cancel_on_exception - @RetryMixin._retry() def shutdown_node(self): - self._logger.info("Starting shutdown") - if not self._cloud.destroy_node(self.cloud_node): - if self._cloud.broken(self.cloud_node): - self._later.cancel_shutdown(self.NODE_BROKEN) + if self.cancellable: + self._logger.info("Checking that node is still eligible for shutdown") + eligible, reason = self._monitor.shutdown_eligible().get() + if not eligible: + self.cancel_shutdown("No longer eligible for shut down because %s" % reason, + try_resume=True) return - else: - # Force a retry. - raise cloud_types.LibcloudError("destroy_node failed") - self._logger.info("Shutdown success") + self._destroy_node() + + def _destroy_node(self): + self._logger.info("Starting shutdown") arv_node = self._arvados_node() - if arv_node is None: - self._finished(success_flag=True) + if self._cloud.destroy_node(self.cloud_node): + self._logger.info("Shutdown success") + if arv_node: + self._later.clean_arvados_node(arv_node) + else: + self._finished(success_flag=True) else: - self._later.clean_arvados_node(arv_node) + self.cancel_shutdown(self.DESTROY_FAILED, try_resume=False) @ComputeNodeStateChangeBase._finish_on_exception @RetryMixin._retry(config.ARVADOS_ERRORS) @@ -306,7 +334,6 @@ class ComputeNodeMonitorActor(config.actor_class): ): super(ComputeNodeMonitorActor, self).__init__() self._later = self.actor_ref.tell_proxy() - self._last_log = None self._shutdowns = shutdown_timer self._cloud_node_fqdn = cloud_fqdn_func self._timer = timer_actor @@ -334,22 +361,22 @@ class ComputeNodeMonitorActor(config.actor_class): self.subscribers.add(subscriber) def _debug(self, msg, *args): - if msg == self._last_log: - return - self._last_log = msg self._logger.debug(msg, *args) - def in_state(self, *states): - # Return a boolean to say whether or not our Arvados node record is in - # one of the given states. If state information is not - # available--because this node has no Arvados record, the record is - # stale, or the record has no state information--return None. - if (self.arvados_node is None) or not timestamp_fresh( - arvados_node_mtime(self.arvados_node), self.node_stale_after): - return None + def get_state(self): + """Get node state, one of ['unpaired', 'busy', 'idle', 'down'].""" + + # If this node is not associated with an Arvados node, return 'unpaired'. + if self.arvados_node is None: + return 'unpaired' + state = self.arvados_node['crunch_worker_state'] - if not state: - return None + + # If state information is not available because it is missing or the + # record is stale, return 'down'. + if not state or not timestamp_fresh(arvados_node_mtime(self.arvados_node), + self.node_stale_after): + state = 'down' # There's a window between when a node pings for the first time and the # value of 'slurm_state' is synchronized by crunch-dispatch. In this @@ -368,11 +395,18 @@ class ComputeNodeMonitorActor(config.actor_class): if arvados_node_missing(self.arvados_node, self.node_stale_after): state = 'down' - result = state in states - if state == 'idle': - result = result and not self.arvados_node['job_uuid'] + # Turns out using 'job_uuid' this way is a bad idea. The node record + # is assigned the job_uuid before the job is locked (which removes it + # from the queue) which means the job will be double-counted as both in + # the wishlist and but also keeping a node busy. This end result is + # excess nodes being booted. + #if state == 'idle' and self.arvados_node['job_uuid']: + # state = 'busy' + + return state - return result + def in_state(self, *states): + return self.get_state() in states def shutdown_eligible(self): """Determine if node is candidate for shut down. @@ -389,18 +423,10 @@ class ComputeNodeMonitorActor(config.actor_class): # boot_grace = ["boot wait", "boot exceeded"] # idle_grace = ["not idle", "idle wait", "idle exceeded"] - if self.arvados_node is None: - crunch_worker_state = 'unpaired' - elif not timestamp_fresh(arvados_node_mtime(self.arvados_node), self.node_stale_after): + if self.arvados_node and not timestamp_fresh(arvados_node_mtime(self.arvados_node), self.node_stale_after): return (False, "node state is stale") - elif self.in_state('down'): - crunch_worker_state = 'down' - elif self.in_state('idle'): - crunch_worker_state = 'idle' - elif self.in_state('busy'): - crunch_worker_state = 'busy' - else: - return (False, "node is paired but crunch_worker_state is '%s'" % self.arvados_node['crunch_worker_state']) + + crunch_worker_state = self.get_state() window = "open" if self._shutdowns.window_open() else "closed"