3 from __future__ import absolute_import, print_function
9 ComputeNodeSetupActor, ComputeNodeUpdateActor, ComputeNodeMonitorActor
10 from . import ComputeNodeShutdownActor as ShutdownActorBase
12 class ComputeNodeShutdownActor(ShutdownActorBase):
13 SLURM_END_STATES = frozenset(['down\n', 'down*\n',
14 'drain\n', 'drain*\n',
18 arv_node = self._arvados_node()
21 return super(ComputeNodeShutdownActor, self).on_start()
23 self._nodename = arv_node['hostname']
24 self._logger.info("Draining SLURM node %s", self._nodename)
25 self._later.issue_slurm_drain()
27 def _set_node_state(self, state, *args):
28 cmd = ['scontrol', 'update', 'NodeName=' + self._nodename,
31 subprocess.check_output(cmd)
33 @ShutdownActorBase._retry((subprocess.CalledProcessError,))
34 def cancel_shutdown(self):
36 self._set_node_state('RESUME')
37 return super(ComputeNodeShutdownActor, self).cancel_shutdown()
39 @ShutdownActorBase._stop_if_window_closed
40 @ShutdownActorBase._retry((subprocess.CalledProcessError,))
41 def issue_slurm_drain(self):
42 self._set_node_state('DRAIN', 'Reason=Node Manager shutdown')
43 self._logger.info("Waiting for SLURM node %s to drain", self._nodename)
44 self._later.await_slurm_drain()
46 @ShutdownActorBase._stop_if_window_closed
47 @ShutdownActorBase._retry((subprocess.CalledProcessError,))
48 def await_slurm_drain(self):
49 output = subprocess.check_output(
50 ['sinfo', '--noheader', '-o', '%t', '-n', self._nodename])
51 if output in self.SLURM_END_STATES:
52 self._later.shutdown_node()
54 self._timer.schedule(time.time() + 10,
55 self._later.await_slurm_drain)