X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d3229c7f727f40436cad66fba5f3345e0b3eede5..eff37344f3cecd5aed259c9852aca4bdcdfb6922:/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py index d613ef1371..1608b529fb 100644 --- a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py +++ b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py @@ -6,6 +6,7 @@ import functools import logging import time +import libcloud.common.types as cloud_types import pykka from .. import arvados_node_fqdn, arvados_node_mtime, timestamp_fresh @@ -18,35 +19,42 @@ class ComputeNodeStateChangeBase(config.actor_class): This base class takes care of retrying changes and notifying subscribers when the change is finished. """ - def __init__(self, logger_name, timer_actor, retry_wait, max_retry_wait): + def __init__(self, logger_name, cloud_client, timer_actor, + retry_wait, max_retry_wait): super(ComputeNodeStateChangeBase, self).__init__() self._later = self.actor_ref.proxy() - self._timer = timer_actor self._logger = logging.getLogger(logger_name) + self._cloud = cloud_client + self._timer = timer_actor self.min_retry_wait = retry_wait self.max_retry_wait = max_retry_wait self.retry_wait = retry_wait self.subscribers = set() @staticmethod - def _retry(errors): + def _retry(errors=()): """Retry decorator for an actor method that makes remote requests. Use this function to decorator an actor method, and pass in a tuple of exceptions to catch. This decorator will schedule retries of that method with exponential backoff if the - original method raises any of the given errors. + original method raises a known cloud driver error, or any of the + given exception types. """ def decorator(orig_func): @functools.wraps(orig_func) - def wrapper(self, *args, **kwargs): + def retry_wrapper(self, *args, **kwargs): + start_time = time.time() try: orig_func(self, *args, **kwargs) - except errors as error: + except Exception as error: + if not (isinstance(error, errors) or + self._cloud.is_cloud_exception(error)): + raise self._logger.warning( "Client error: %s - waiting %s seconds", error, self.retry_wait) - self._timer.schedule(self.retry_wait, + self._timer.schedule(start_time + self.retry_wait, getattr(self._later, orig_func.__name__), *args, **kwargs) @@ -54,7 +62,7 @@ class ComputeNodeStateChangeBase(config.actor_class): self.max_retry_wait) else: self.retry_wait = self.min_retry_wait - return wrapper + return retry_wrapper return decorator def _finished(self): @@ -84,9 +92,9 @@ class ComputeNodeSetupActor(ComputeNodeStateChangeBase): cloud_size, arvados_node=None, retry_wait=1, max_retry_wait=180): super(ComputeNodeSetupActor, self).__init__( - 'arvnodeman.nodeup', timer_actor, retry_wait, max_retry_wait) + 'arvnodeman.nodeup', cloud_client, timer_actor, + retry_wait, max_retry_wait) self._arvados = arvados_client - self._cloud = cloud_client self.cloud_size = cloud_size self.arvados_node = None self.cloud_node = None @@ -95,12 +103,12 @@ class ComputeNodeSetupActor(ComputeNodeStateChangeBase): else: self._later.prepare_arvados_node(arvados_node) - @ComputeNodeStateChangeBase._retry(config.ARVADOS_ERRORS) + @ComputeNodeStateChangeBase._retry() def create_arvados_node(self): self.arvados_node = self._arvados.nodes().create(body={}).execute() self._later.create_cloud_node() - @ComputeNodeStateChangeBase._retry(config.ARVADOS_ERRORS) + @ComputeNodeStateChangeBase._retry() def prepare_arvados_node(self, node): self.arvados_node = self._arvados.nodes().update( uuid=node['uuid'], @@ -114,13 +122,19 @@ class ComputeNodeSetupActor(ComputeNodeStateChangeBase): ).execute() self._later.create_cloud_node() - @ComputeNodeStateChangeBase._retry(config.CLOUD_ERRORS) + @ComputeNodeStateChangeBase._retry() def create_cloud_node(self): self._logger.info("Creating cloud node with size %s.", self.cloud_size.name) self.cloud_node = self._cloud.create_node(self.cloud_size, self.arvados_node) self._logger.info("Cloud node %s created.", self.cloud_node.id) + self._later.post_create() + + @ComputeNodeStateChangeBase._retry() + def post_create(self): + self._cloud.post_create_node(self.cloud_node) + self._logger.info("%s post-create work done.", self.cloud_node.id) self._finished() def stop_if_no_cloud_node(self): @@ -133,20 +147,56 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): This actor simply destroys a cloud node, retrying as needed. """ - def __init__(self, timer_actor, cloud_client, cloud_node, - retry_wait=1, max_retry_wait=180): + def __init__(self, timer_actor, cloud_client, node_monitor, + cancellable=True, retry_wait=1, max_retry_wait=180): + # If a ShutdownActor is cancellable, it will ask the + # ComputeNodeMonitorActor if it's still eligible before taking each + # action, and stop the shutdown process if the node is no longer + # eligible. Normal shutdowns based on job demand should be + # cancellable; shutdowns based on node misbehavior should not. super(ComputeNodeShutdownActor, self).__init__( - 'arvnodeman.nodedown', timer_actor, retry_wait, max_retry_wait) - self._cloud = cloud_client - self.cloud_node = cloud_node + 'arvnodeman.nodedown', cloud_client, timer_actor, + retry_wait, max_retry_wait) + self._monitor = node_monitor.proxy() + self.cloud_node = self._monitor.cloud_node.get() + self.cancellable = cancellable + self.success = None + + def on_start(self): self._later.shutdown_node() - @ComputeNodeStateChangeBase._retry(config.CLOUD_ERRORS) - def shutdown_node(self): - self._cloud.destroy_node(self.cloud_node) - self._logger.info("Cloud node %s shut down.", self.cloud_node.id) + def cancel_shutdown(self): + self.success = False self._finished() + def _stop_if_window_closed(orig_func): + @functools.wraps(orig_func) + def stop_wrapper(self, *args, **kwargs): + if (self.cancellable and + (not self._monitor.shutdown_eligible().get())): + self._logger.info( + "Cloud node %s shutdown cancelled - no longer eligible.", + self.cloud_node.id) + self._later.cancel_shutdown() + return None + else: + return orig_func(self, *args, **kwargs) + return stop_wrapper + + @_stop_if_window_closed + @ComputeNodeStateChangeBase._retry() + def shutdown_node(self): + if self._cloud.destroy_node(self.cloud_node): + self._logger.info("Cloud node %s shut down.", self.cloud_node.id) + self.success = True + self._finished() + else: + # Force a retry. + raise cloud_types.LibcloudError("destroy_node failed") + + # Make the decorator available to subclasses. + _stop_if_window_closed = staticmethod(_stop_if_window_closed) + class ComputeNodeUpdateActor(config.actor_class): """Actor to dispatch one-off cloud management requests. @@ -172,14 +222,14 @@ class ComputeNodeUpdateActor(config.actor_class): def _throttle_errors(orig_func): @functools.wraps(orig_func) - def wrapper(self, *args, **kwargs): + def throttle_wrapper(self, *args, **kwargs): throttle_time = self.next_request_time - time.time() if throttle_time > 0: time.sleep(throttle_time) self.next_request_time = time.time() try: result = orig_func(self, *args, **kwargs) - except config.CLOUD_ERRORS: + except Exception as error: self.error_streak += 1 self.next_request_time += min(2 ** self.error_streak, self.max_retry_wait) @@ -187,7 +237,7 @@ class ComputeNodeUpdateActor(config.actor_class): else: self.error_streak = 0 return result - return wrapper + return throttle_wrapper @_throttle_errors def sync_node(self, cloud_node, arvados_node): @@ -238,7 +288,7 @@ class ComputeNodeMonitorActor(config.actor_class): if (self.arvados_node is None) or not timestamp_fresh( arvados_node_mtime(self.arvados_node), self.node_stale_after): return None - state = self.arvados_node['info'].get('slurm_state') + state = self.arvados_node['crunch_worker_state'] if not state: return None result = state in states @@ -246,8 +296,10 @@ class ComputeNodeMonitorActor(config.actor_class): result = result and not self.arvados_node['job_uuid'] return result - def _shutdown_eligible(self): - if self.arvados_node is None: + def shutdown_eligible(self): + if not self._shutdowns.window_open(): + return False + elif self.arvados_node is None: # If this is a new, unpaired node, it's eligible for # shutdown--we figure there was an error during bootstrap. return timestamp_fresh(self.cloud_node_start_time, @@ -257,17 +309,15 @@ class ComputeNodeMonitorActor(config.actor_class): def consider_shutdown(self): next_opening = self._shutdowns.next_opening() - if self._shutdowns.window_open(): - if self._shutdown_eligible(): - self._debug("Node %s suggesting shutdown.", self.cloud_node.id) - _notify_subscribers(self._later, self.subscribers) - else: - self._debug("Node %s shutdown window open but node busy.", - self.cloud_node.id) - else: + if self.shutdown_eligible(): + self._debug("Node %s suggesting shutdown.", self.cloud_node.id) + _notify_subscribers(self._later, self.subscribers) + elif self._shutdowns.window_open(): + self._debug("Node %s shutdown window open but node busy.", + self.cloud_node.id) + elif self.last_shutdown_opening != next_opening: self._debug("Node %s shutdown window closed. Next at %s.", self.cloud_node.id, time.ctime(next_opening)) - if self.last_shutdown_opening != next_opening: self._timer.schedule(next_opening, self._later.consider_shutdown) self.last_shutdown_opening = next_opening