X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ae8aaa4c55762222c837fcce8e9ad6800ff8b128..8a41cc44ee196c9347785baa476a370abe77c75c:/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py index 71f9083c01..77c515d565 100644 --- a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py +++ b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py @@ -1,12 +1,18 @@ #!/usr/bin/env python +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 from __future__ import absolute_import, print_function import functools import logging import time +import re import libcloud.common.types as cloud_types +from libcloud.common.exceptions import BaseHTTPError + import pykka from .. import \ @@ -14,8 +20,11 @@ from .. import \ arvados_node_missing, RetryMixin from ...clientactor import _notify_subscribers from ... import config +from ... import status from .transitions import transitions +QuotaExceeded = "QuotaExceeded" + class ComputeNodeStateChangeBase(config.actor_class, RetryMixin): """Base class for actors that change a compute node's state. @@ -96,6 +105,7 @@ class ComputeNodeSetupActor(ComputeNodeStateChangeBase): self.cloud_size = cloud_size self.arvados_node = None self.cloud_node = None + self.error = None if arvados_node is None: self._later.create_arvados_node() else: @@ -104,25 +114,53 @@ class ComputeNodeSetupActor(ComputeNodeStateChangeBase): @ComputeNodeStateChangeBase._finish_on_exception @RetryMixin._retry(config.ARVADOS_ERRORS) def create_arvados_node(self): - self.arvados_node = self._arvados.nodes().create(body={}).execute() + self.arvados_node = self._arvados.nodes().create( + body={}, assign_slot=True).execute() self._later.create_cloud_node() @ComputeNodeStateChangeBase._finish_on_exception @RetryMixin._retry(config.ARVADOS_ERRORS) def prepare_arvados_node(self, node): - self.arvados_node = self._clean_arvados_node( - node, "Prepared by Node Manager") + self._clean_arvados_node(node, "Prepared by Node Manager") + self.arvados_node = self._arvados.nodes().update( + uuid=node['uuid'], body={}, assign_slot=True).execute() self._later.create_cloud_node() @ComputeNodeStateChangeBase._finish_on_exception @RetryMixin._retry() def create_cloud_node(self): self._logger.info("Sending create_node request for node size %s.", - self.cloud_size.name) - self.cloud_node = self._cloud.create_node(self.cloud_size, - self.arvados_node) - if not self.cloud_node.size: - self.cloud_node.size = self.cloud_size + self.cloud_size.id) + try: + self.cloud_node = self._cloud.create_node(self.cloud_size, + self.arvados_node) + except BaseHTTPError as e: + if e.code == 429 or "RequestLimitExceeded" in e.message: + # Don't consider API rate limits to be quota errors. + # re-raise so the Retry logic applies. + raise + + # The set of possible error codes / messages isn't documented for + # all clouds, so use a keyword heuristic to determine if the + # failure is likely due to a quota. + if re.search(r'(exceed|quota|limit)', e.message, re.I): + self.error = QuotaExceeded + self._logger.warning("Quota exceeded: %s", e) + self._finished() + return + else: + # Something else happened, re-raise so the Retry logic applies. + raise + except Exception as e: + raise + + # The information included in the node size object we get from libcloud + # is inconsistent between cloud drivers. Replace libcloud NodeSize + # object with compatible CloudSizeWrapper object which merges the size + # info reported from the cloud with size information from the + # configuration file. + self.cloud_node.size = self.cloud_size + self._logger.info("Cloud node %s created.", self.cloud_node.id) self._later.update_arvados_node_properties() @@ -205,9 +243,15 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): return super(ComputeNodeShutdownActor, self)._finished() def cancel_shutdown(self, reason, **kwargs): + if not self.cancellable: + return False + if self.cancel_reason is not None: + # already cancelled + return False self.cancel_reason = reason self._logger.info("Shutdown cancelled: %s.", reason) self._finished(success_flag=False) + return True def _cancel_on_exception(orig_func): @functools.wraps(orig_func) @@ -222,6 +266,9 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): @_cancel_on_exception def shutdown_node(self): + if self.cancel_reason is not None: + # already cancelled + return if self.cancellable: self._logger.info("Checking that node is still eligible for shutdown") eligible, reason = self._monitor.shutdown_eligible().get() @@ -229,12 +276,16 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): self.cancel_shutdown("No longer eligible for shut down because %s" % reason, try_resume=True) return + # If boot failed, count the event + if self._monitor.get_state().get() == 'unpaired': + status.tracker.counter_add('boot_failures') self._destroy_node() def _destroy_node(self): self._logger.info("Starting shutdown") arv_node = self._arvados_node() if self._cloud.destroy_node(self.cloud_node): + self.cancellable = False self._logger.info("Shutdown success") if arv_node: self._later.clean_arvados_node(arv_node) @@ -250,7 +301,7 @@ class ComputeNodeShutdownActor(ComputeNodeStateChangeBase): self._finished(success_flag=True) -class ComputeNodeUpdateActor(config.actor_class): +class ComputeNodeUpdateActor(config.actor_class, RetryMixin): """Actor to dispatch one-off cloud management requests. This actor receives requests for small cloud updates, and @@ -259,12 +310,12 @@ class ComputeNodeUpdateActor(config.actor_class): dedicated actor for this gives us the opportunity to control the flow of requests; e.g., by backing off when errors occur. """ - def __init__(self, cloud_factory, max_retry_wait=180): + def __init__(self, cloud_factory, timer_actor, max_retry_wait=180): super(ComputeNodeUpdateActor, self).__init__() + RetryMixin.__init__(self, 1, max_retry_wait, + None, cloud_factory(), timer_actor) self._cloud = cloud_factory() - self.max_retry_wait = max_retry_wait - self.error_streak = 0 - self.next_request_time = time.time() + self._later = self.actor_ref.tell_proxy() def _set_logger(self): self._logger = logging.getLogger("%s.%s" % (self.__class__.__name__, self.actor_urn[33:])) @@ -272,30 +323,10 @@ class ComputeNodeUpdateActor(config.actor_class): def on_start(self): self._set_logger() - def _throttle_errors(orig_func): - @functools.wraps(orig_func) - def throttle_wrapper(self, *args, **kwargs): - throttle_time = self.next_request_time - time.time() - if throttle_time > 0: - time.sleep(throttle_time) - self.next_request_time = time.time() - try: - result = orig_func(self, *args, **kwargs) - except Exception as error: - if self._cloud.is_cloud_exception(error): - self.error_streak += 1 - self.next_request_time += min(2 ** self.error_streak, - self.max_retry_wait) - self._logger.warn( - "Unhandled exception: %s", error, exc_info=error) - else: - self.error_streak = 0 - return result - return throttle_wrapper - - @_throttle_errors + @RetryMixin._retry() def sync_node(self, cloud_node, arvados_node): - return self._cloud.sync_node(cloud_node, arvados_node) + if self._cloud.node_fqdn(cloud_node) != arvados_node_fqdn(arvados_node): + return self._cloud.sync_node(cloud_node, arvados_node) class ComputeNodeMonitorActor(config.actor_class): @@ -306,14 +337,13 @@ class ComputeNodeMonitorActor(config.actor_class): for shutdown. """ def __init__(self, cloud_node, cloud_node_start_time, shutdown_timer, - cloud_fqdn_func, timer_actor, update_actor, cloud_client, + timer_actor, update_actor, cloud_client, arvados_node=None, poll_stale_after=600, node_stale_after=3600, - boot_fail_after=1800 + boot_fail_after=1800, consecutive_idle_count=0 ): super(ComputeNodeMonitorActor, self).__init__() self._later = self.actor_ref.tell_proxy() self._shutdowns = shutdown_timer - self._cloud_node_fqdn = cloud_fqdn_func self._timer = timer_actor self._update = update_actor self._cloud = cloud_client @@ -324,6 +354,8 @@ class ComputeNodeMonitorActor(config.actor_class): self.boot_fail_after = boot_fail_after self.subscribers = set() self.arvados_node = None + self.consecutive_idle_count = consecutive_idle_count + self.consecutive_idle = 0 self._later.update_arvados_node(arvados_node) self.last_shutdown_opening = None self._later.consider_shutdown() @@ -344,9 +376,15 @@ class ComputeNodeMonitorActor(config.actor_class): def get_state(self): """Get node state, one of ['unpaired', 'busy', 'idle', 'down'].""" - # If this node is not associated with an Arvados node, return 'unpaired'. + # If this node is not associated with an Arvados node, return + # 'unpaired' if we're in the boot grace period, and 'down' if not, + # so it isn't counted towards usable nodes. if self.arvados_node is None: - return 'unpaired' + if timestamp_fresh(self.cloud_node_start_time, + self.boot_fail_after): + return 'unpaired' + else: + return 'down' state = self.arvados_node['crunch_worker_state'] @@ -381,6 +419,12 @@ class ComputeNodeMonitorActor(config.actor_class): #if state == 'idle' and self.arvados_node['job_uuid']: # state = 'busy' + # Update idle node times tracker + if state == 'idle': + status.tracker.idle_in(self.arvados_node['hostname']) + else: + status.tracker.idle_out(self.arvados_node['hostname']) + return state def in_state(self, *states): @@ -394,6 +438,11 @@ class ComputeNodeMonitorActor(config.actor_class): reason for the decision. """ + # If this node's size is invalid (because it has a stale arvados_node_size + # tag), return True so that it's properly shut down. + if self.cloud_node.size.id == 'invalid': + return (True, "node's size tag '%s' not recognizable" % (self.cloud_node.extra['arvados_node_size'],)) + # Collect states and then consult state transition table whether we # should shut down. Possible states are: # crunch_worker_state = ['unpaired', 'busy', 'idle', 'down'] @@ -413,8 +462,14 @@ class ComputeNodeMonitorActor(config.actor_class): else: boot_grace = "boot exceeded" - # API server side not implemented yet. - idle_grace = 'idle exceeded' + if crunch_worker_state == "idle": + # Must report as "idle" at least "consecutive_idle_count" times + if self.consecutive_idle < self.consecutive_idle_count: + idle_grace = 'idle wait' + else: + idle_grace = 'idle exceeded' + else: + idle_grace = 'not idle' node_state = (crunch_worker_state, window, boot_grace, idle_grace) t = transitions[node_state] @@ -460,8 +515,11 @@ class ComputeNodeMonitorActor(config.actor_class): self._later.consider_shutdown() def update_arvados_node(self, arvados_node): - # If the cloud node's FQDN doesn't match what's in the Arvados node - # record, make them match. + """Called when the latest Arvados node record is retrieved. + + Calls the updater's sync_node() method. + + """ # This method is a little unusual in the way it just fires off the # request without checking the result or retrying errors. That's # because this update happens every time we reload the Arvados node @@ -470,7 +528,9 @@ class ComputeNodeMonitorActor(config.actor_class): # the logic to throttle those effective retries when there's trouble. if arvados_node is not None: self.arvados_node = arvados_node - if (self._cloud_node_fqdn(self.cloud_node) != - arvados_node_fqdn(self.arvados_node)): - self._update.sync_node(self.cloud_node, self.arvados_node) + self._update.sync_node(self.cloud_node, self.arvados_node) + if self.arvados_node['crunch_worker_state'] == "idle": + self.consecutive_idle += 1 + else: + self.consecutive_idle = 0 self._later.consider_shutdown()