3 from __future__ import absolute_import, print_function
9 ComputeNodeSetupActor, ComputeNodeUpdateActor, ComputeNodeMonitorActor
10 from . import ComputeNodeShutdownActor as ShutdownActorBase
12 class ComputeNodeShutdownActor(ShutdownActorBase):
13 SLURM_END_STATES = frozenset(['down\n', 'down*\n', 'drain\n', 'fail\n'])
16 arv_node = self._arvados_node()
19 return super(ComputeNodeShutdownActor, self).on_start()
21 self._nodename = arv_node['hostname']
22 self._logger.info("Draining SLURM node %s", self._nodename)
23 self._later.issue_slurm_drain()
25 def _set_node_state(self, state, *args):
26 cmd = ['scontrol', 'update', 'NodeName=' + self._nodename,
29 subprocess.check_output(cmd)
31 @ShutdownActorBase._retry((subprocess.CalledProcessError,))
32 def cancel_shutdown(self):
34 self._set_node_state('RESUME')
35 return super(ComputeNodeShutdownActor, self).cancel_shutdown()
37 @ShutdownActorBase._stop_if_window_closed
38 @ShutdownActorBase._retry((subprocess.CalledProcessError,))
39 def issue_slurm_drain(self):
40 self._set_node_state('DRAIN', 'Reason=Node Manager shutdown')
41 self._logger.info("Waiting for SLURM node %s to drain", self._nodename)
42 self._later.await_slurm_drain()
44 @ShutdownActorBase._stop_if_window_closed
45 @ShutdownActorBase._retry((subprocess.CalledProcessError,))
46 def await_slurm_drain(self):
47 output = subprocess.check_output(
48 ['sinfo', '--noheader', '-o', '%t', '-n', self._nodename])
49 if output in self.SLURM_END_STATES:
50 self._later.shutdown_node()
52 self._timer.schedule(time.time() + 10,
53 self._later.await_slurm_drain)