X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/242d3272391baeb95eb0a5e4e51627c2d54e7bc6..3a2802166901386eba4c3bc9bf877a18346ba03c:/services/nodemanager/arvnodeman/computenode/dispatch/slurm.py diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/slurm.py b/services/nodemanager/arvnodeman/computenode/dispatch/slurm.py index 919b57f42c..4d70436801 100644 --- a/services/nodemanager/arvnodeman/computenode/dispatch/slurm.py +++ b/services/nodemanager/arvnodeman/computenode/dispatch/slurm.py @@ -8,6 +8,7 @@ import time from . import \ ComputeNodeSetupActor, ComputeNodeUpdateActor, ComputeNodeMonitorActor from . import ComputeNodeShutdownActor as ShutdownActorBase +from .. import RetryMixin class ComputeNodeShutdownActor(ShutdownActorBase): SLURM_END_STATES = frozenset(['down\n', 'down*\n', @@ -21,6 +22,7 @@ class ComputeNodeShutdownActor(ShutdownActorBase): self._nodename = None return super(ComputeNodeShutdownActor, self).on_start() else: + self._set_logger() self._nodename = arv_node['hostname'] self._logger.info("Draining SLURM node %s", self._nodename) self._later.issue_slurm_drain() @@ -42,7 +44,7 @@ class ComputeNodeShutdownActor(ShutdownActorBase): # of the excessive memory usage that result in the "Cannot allocate memory" # error are still being investigated. - @ShutdownActorBase._retry((subprocess.CalledProcessError, OSError)) + @RetryMixin._retry((subprocess.CalledProcessError, OSError)) def cancel_shutdown(self, reason): if self._nodename: if self._get_slurm_state() in self.SLURM_DRAIN_STATES: @@ -54,14 +56,14 @@ class ComputeNodeShutdownActor(ShutdownActorBase): pass return super(ComputeNodeShutdownActor, self).cancel_shutdown(reason) - @ShutdownActorBase._retry((subprocess.CalledProcessError, OSError)) + @RetryMixin._retry((subprocess.CalledProcessError, OSError)) @ShutdownActorBase._stop_if_window_closed def issue_slurm_drain(self): self._set_node_state('DRAIN', 'Reason=Node Manager shutdown') self._logger.info("Waiting for SLURM node %s to drain", self._nodename) self._later.await_slurm_drain() - @ShutdownActorBase._retry((subprocess.CalledProcessError, OSError)) + @RetryMixin._retry((subprocess.CalledProcessError, OSError)) @ShutdownActorBase._stop_if_window_closed def await_slurm_drain(self): output = self._get_slurm_state()