X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/dd47cf79c71bb4cc5b90f3752d0b79110278e197..f6aa7c0c8c84b85b550d73117c6fdbd663a38c4c:/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py index c79d8f9588..455719832d 100644 --- a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py +++ b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py @@ -9,7 +9,8 @@ import time import libcloud.common.types as cloud_types import pykka -from .. import arvados_node_fqdn, arvados_node_mtime, timestamp_fresh +from .. import \ + arvados_node_fqdn, arvados_node_mtime, arvados_timestamp, timestamp_fresh from ...clientactor import _notify_subscribers from ... import config @@ -19,32 +20,39 @@ class ComputeNodeStateChangeBase(config.actor_class): This base class takes care of retrying changes and notifying subscribers when the change is finished. """ - def __init__(self, logger_name, timer_actor, retry_wait, max_retry_wait): + def __init__(self, logger_name, cloud_client, arvados_client, timer_actor, + retry_wait, max_retry_wait): super(ComputeNodeStateChangeBase, self).__init__() self._later = self.actor_ref.proxy() - self._timer = timer_actor self._logger = logging.getLogger(logger_name) + self._cloud = cloud_client + self._arvados = arvados_client + self._timer = timer_actor self.min_retry_wait = retry_wait self.max_retry_wait = max_retry_wait self.retry_wait = retry_wait self.subscribers = set() @staticmethod - def _retry(errors): + def _retry(errors=()): """Retry decorator for an actor method that makes remote requests. Use this function to decorator an actor method, and pass in a tuple of exceptions to catch. This decorator will schedule retries of that method with exponential backoff if the - original method raises any of the given errors. + original method raises a known cloud driver error, or any of the + given exception types. """ def decorator(orig_func): @functools.wraps(orig_func) - def wrapper(self, *args, **kwargs): + def retry_wrapper(self, *args, **kwargs): start_time = time.time() try: orig_func(self, *args, **kwargs) - except errors as error: + except Exception as error: + if not (isinstance(error, errors) or + self._cloud.is_cloud_exception(error)): + raise self._logger.warning( "Client error: %s - waiting %s seconds", error, self.retry_wait) @@ -56,7 +64,7 @@ class ComputeNodeStateChangeBase(config.actor_class): self.max_retry_wait) else: self.retry_wait = self.min_retry_wait - return wrapper + return retry_wrapper return decorator def _finished(self): @@ -72,6 +80,18 @@ class ComputeNodeStateChangeBase(config.actor_class): else: self.subscribers.add(subscriber) + def _clean_arvados_node(self, arvados_node, explanation): + return self._arvados.nodes().update( + uuid=arvados_node['uuid'], + body={'hostname': None, + 'ip_address': None, + 'slot_number': None, + 'first_ping_at': None, + 'last_ping_at': None, + 'info': {'ec2_instance_id': None, + 'last_action': explanation}}, + ).execute() + class ComputeNodeSetupActor(ComputeNodeStateChangeBase): """Actor to create and set up a cloud compute node. @@ -86,9 +106,8 @@ class ComputeNodeSetupActor(ComputeNodeStateChangeBase): cloud_size, arvados_node=None, retry_wait=1, max_retry_wait=180): super(ComputeNodeSetupActor, self).__init__( - 'arvnodeman.nodeup', timer_actor, retry_wait, max_retry_wait) - self._arvados = arvados_client - self._cloud = cloud_client + 'arvnodeman.nodeup', cloud_client, arvados_client, timer_actor, + retry_wait, max_retry_wait) self.cloud_size = cloud_size self.arvados_node = None self.cloud_node = None @@ -104,30 +123,30 @@ class ComputeNodeSetupActor(ComputeNodeStateChangeBase): @ComputeNodeStateChangeBase._retry(config.ARVADOS_ERRORS) def prepare_arvados_node(self, node): - self.arvados_node = self._arvados.nodes().update( - uuid=node['uuid'], - body={'hostname': None, - 'ip_address': None, - 'slot_number': None, - 'first_ping_at': None, - 'last_ping_at': None, - 'info': {'ec2_instance_id': None, - 'last_action': "Prepared by Node Manager"}} - ).execute() + self.arvados_node = self._clean_arvados_node( + node, "Prepared by Node Manager") self._later.create_cloud_node() - @ComputeNodeStateChangeBase._retry(config.CLOUD_ERRORS) + @ComputeNodeStateChangeBase._retry() def create_cloud_node(self): self._logger.info("Creating cloud node with size %s.", self.cloud_size.name) self.cloud_node = self._cloud.create_node(self.cloud_size, self.arvados_node) self._logger.info("Cloud node %s created.", self.cloud_node.id) + self._later.post_create() + + @ComputeNodeStateChangeBase._retry() + def post_create(self): + self._cloud.post_create_node(self.cloud_node) + self._logger.info("%s post-create work done.", self.cloud_node.id) self._finished() def stop_if_no_cloud_node(self): - if self.cloud_node is None: - self.stop() + if self.cloud_node is not None: + return False + self.stop() + return True class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): @@ -135,7 +154,7 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): This actor simply destroys a cloud node, retrying as needed. """ - def __init__(self, timer_actor, cloud_client, node_monitor, + def __init__(self, timer_actor, cloud_client, arvados_client, node_monitor, cancellable=True, retry_wait=1, max_retry_wait=180): # If a ShutdownActor is cancellable, it will ask the # ComputeNodeMonitorActor if it's still eligible before taking each @@ -143,8 +162,8 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): # eligible. Normal shutdowns based on job demand should be # cancellable; shutdowns based on node misbehavior should not. super(ComputeNodeShutdownActor, self).__init__( - 'arvnodeman.nodedown', timer_actor, retry_wait, max_retry_wait) - self._cloud = cloud_client + 'arvnodeman.nodedown', cloud_client, arvados_client, timer_actor, + retry_wait, max_retry_wait) self._monitor = node_monitor.proxy() self.cloud_node = self._monitor.cloud_node.get() self.cancellable = cancellable @@ -153,13 +172,20 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): def on_start(self): self._later.shutdown_node() + def _arvados_node(self): + return self._monitor.arvados_node.get() + + def _finished(self, success_flag=None): + if success_flag is not None: + self.success = success_flag + return super(ComputeNodeShutdownActor, self)._finished() + def cancel_shutdown(self): - self.success = False - self._finished() + self._finished(success_flag=False) def _stop_if_window_closed(orig_func): @functools.wraps(orig_func) - def wrapper(self, *args, **kwargs): + def stop_wrapper(self, *args, **kwargs): if (self.cancellable and (not self._monitor.shutdown_eligible().get())): self._logger.info( @@ -169,18 +195,25 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): return None else: return orig_func(self, *args, **kwargs) - return wrapper + return stop_wrapper @_stop_if_window_closed - @ComputeNodeStateChangeBase._retry(config.CLOUD_ERRORS) + @ComputeNodeStateChangeBase._retry() def shutdown_node(self): - if self._cloud.destroy_node(self.cloud_node): - self._logger.info("Cloud node %s shut down.", self.cloud_node.id) - self.success = True - self._finished() - else: + if not self._cloud.destroy_node(self.cloud_node): # Force a retry. raise cloud_types.LibcloudError("destroy_node failed") + self._logger.info("Cloud node %s shut down.", self.cloud_node.id) + arv_node = self._arvados_node() + if arv_node is None: + self._finished(success_flag=True) + else: + self._later.clean_arvados_node(arv_node) + + @ComputeNodeStateChangeBase._retry(config.ARVADOS_ERRORS) + def clean_arvados_node(self, arvados_node): + self._clean_arvados_node(arvados_node, "Shut down by Node Manager") + self._finished(success_flag=True) # Make the decorator available to subclasses. _stop_if_window_closed = staticmethod(_stop_if_window_closed) @@ -210,14 +243,14 @@ class ComputeNodeUpdateActor(config.actor_class): def _throttle_errors(orig_func): @functools.wraps(orig_func) - def wrapper(self, *args, **kwargs): + def throttle_wrapper(self, *args, **kwargs): throttle_time = self.next_request_time - time.time() if throttle_time > 0: time.sleep(throttle_time) self.next_request_time = time.time() try: result = orig_func(self, *args, **kwargs) - except config.CLOUD_ERRORS: + except Exception as error: self.error_streak += 1 self.next_request_time += min(2 ** self.error_streak, self.max_retry_wait) @@ -225,7 +258,7 @@ class ComputeNodeUpdateActor(config.actor_class): else: self.error_streak = 0 return result - return wrapper + return throttle_wrapper @_throttle_errors def sync_node(self, cloud_node, arvados_node): @@ -240,19 +273,24 @@ class ComputeNodeMonitorActor(config.actor_class): for shutdown. """ def __init__(self, cloud_node, cloud_node_start_time, shutdown_timer, - timer_actor, update_actor, arvados_node=None, - poll_stale_after=600, node_stale_after=3600): + cloud_fqdn_func, timer_actor, update_actor, cloud_client, + arvados_node=None, poll_stale_after=600, node_stale_after=3600, + boot_fail_after=1800 + ): super(ComputeNodeMonitorActor, self).__init__() self._later = self.actor_ref.proxy() self._logger = logging.getLogger('arvnodeman.computenode') self._last_log = None self._shutdowns = shutdown_timer + self._cloud_node_fqdn = cloud_fqdn_func self._timer = timer_actor self._update = update_actor + self._cloud = cloud_client self.cloud_node = cloud_node self.cloud_node_start_time = cloud_node_start_time self.poll_stale_after = poll_stale_after self.node_stale_after = node_stale_after + self.boot_fail_after = boot_fail_after self.subscribers = set() self.arvados_node = None self._later.update_arvados_node(arvados_node) @@ -276,7 +314,7 @@ class ComputeNodeMonitorActor(config.actor_class): if (self.arvados_node is None) or not timestamp_fresh( arvados_node_mtime(self.arvados_node), self.node_stale_after): return None - state = self.arvados_node['info'].get('slurm_state') + state = self.arvados_node['crunch_worker_state'] if not state: return None result = state in states @@ -288,10 +326,13 @@ class ComputeNodeMonitorActor(config.actor_class): if not self._shutdowns.window_open(): return False elif self.arvados_node is None: - # If this is a new, unpaired node, it's eligible for - # shutdown--we figure there was an error during bootstrap. - return timestamp_fresh(self.cloud_node_start_time, - self.node_stale_after) + # Node is unpaired. + # If it hasn't pinged Arvados after boot_fail seconds, shut it down + return not timestamp_fresh(self.cloud_node_start_time, self.boot_fail_after) + elif self.arvados_node.get('status') == "missing" and self._cloud.broken(self.cloud_node): + # Node is paired, but Arvados says it is missing and the cloud says the node + # is in an error state, so shut it down. + return True else: return self.in_state('idle') @@ -310,9 +351,11 @@ class ComputeNodeMonitorActor(config.actor_class): self.last_shutdown_opening = next_opening def offer_arvados_pair(self, arvados_node): - if self.arvados_node is not None: + first_ping_s = arvados_node.get('first_ping_at') + if (self.arvados_node is not None) or (not first_ping_s): return None - elif arvados_node['ip_address'] in self.cloud_node.private_ips: + elif ((arvados_node['ip_address'] in self.cloud_node.private_ips) and + (arvados_timestamp(first_ping_s) >= self.cloud_node_start_time)): self._later.update_arvados_node(arvados_node) return self.cloud_node.id else: @@ -324,9 +367,17 @@ class ComputeNodeMonitorActor(config.actor_class): self._later.consider_shutdown() def update_arvados_node(self, arvados_node): + # If the cloud node's FQDN doesn't match what's in the Arvados node + # record, make them match. + # This method is a little unusual in the way it just fires off the + # request without checking the result or retrying errors. That's + # because this update happens every time we reload the Arvados node + # list: if a previous sync attempt failed, we'll see that the names + # are out of sync and just try again. ComputeNodeUpdateActor has + # the logic to throttle those effective retries when there's trouble. if arvados_node is not None: self.arvados_node = arvados_node - new_hostname = arvados_node_fqdn(self.arvados_node) - if new_hostname != self.cloud_node.name: + if (self._cloud_node_fqdn(self.cloud_node) != + arvados_node_fqdn(self.arvados_node)): self._update.sync_node(self.cloud_node, self.arvados_node) self._later.consider_shutdown()