3 from __future__ import absolute_import, print_function
9 ComputeNodeSetupActor, ComputeNodeUpdateActor, ComputeNodeMonitorActor
10 from . import ComputeNodeShutdownActor as ShutdownActorBase
12 class ComputeNodeShutdownActor(ShutdownActorBase):
14 arv_node = self._monitor.arvados_node.get()
16 return super(ComputeNodeShutdownActor, self).on_start()
18 self._nodename = arv_node['hostname']
19 self._logger.info("Draining SLURM node %s", self._nodename)
20 self._later.issue_slurm_drain()
22 def _set_node_state(self, state, *args):
23 cmd = ['scontrol', 'update', 'NodeName=' + self._nodename,
26 subprocess.check_output(cmd)
28 @ShutdownActorBase._retry((subprocess.CalledProcessError,))
29 def cancel_shutdown(self):
30 self._set_node_state('RESUME')
31 return super(ComputeNodeShutdownActor, self).cancel_shutdown()
33 @ShutdownActorBase._stop_if_window_closed
34 @ShutdownActorBase._retry((subprocess.CalledProcessError,))
35 def issue_slurm_drain(self):
36 self._set_node_state('DRAIN', 'Reason=Node Manager shutdown')
37 self._logger.info("Waiting for SLURM node %s to drain", self._nodename)
38 self._later.await_slurm_drain()
40 @ShutdownActorBase._stop_if_window_closed
41 @ShutdownActorBase._retry((subprocess.CalledProcessError,))
42 def await_slurm_drain(self):
43 output = subprocess.check_output(
44 ['sinfo', '--noheader', '-o', '%t', '-n', self._nodename])
45 if output == 'drain\n':
46 self._later.shutdown_node()
48 self._timer.schedule(time.time() + 10,
49 self._later.await_slurm_drain)