6 %w{TERM INT}.each do |sig|
9 $stderr.puts "Received #{signame} signal"
14 ENV["RAILS_ENV"] = ARGV[0] || "development"
16 require File.dirname(__FILE__) + '/../config/boot'
17 require File.dirname(__FILE__) + '/../config/environment'
21 include ApplicationHelper
24 return act_as_system_user
32 if Server::Application.config.whjobmanager_wrapper.to_s.match /^slurm/
37 collect { |line| line.match /(\d+) +idle/ }.
39 @idle_slurm_nodes = re[1].to_i if re
49 if job.resource_limits['min_nodes']
50 min_nodes = begin job.resource_limits['min_nodes'].to_i rescue 1 end
53 next if @idle_slurm_nodes and @idle_slurm_nodes < min_nodes
55 next if @running[job.uuid]
59 case Server::Application.config.whjobmanager_wrapper
67 "--job-name=#{job.uuid}",
68 "--nodes=#{min_nodes}"]
70 raise "Unknown whjobmanager_wrapper: #{Server::Application.config.whjobmanager_wrapper}"
73 cmd_args << 'whjobmanager'
74 cmd_args << "id=#{job.uuid}"
75 cmd_args << "mrfunction=#{job.script}"
76 job.script_parameters.each do |k,v|
83 cmd_args << "#{k}=#{v}"
85 cmd_args << "revision=#{job.script_version}"
88 cmd_args << "stepspernode=#{job.resource_limits['max_tasks_per_node'].to_i}"
90 # OK if limit is not specified. OK to ignore if not integer.
93 $stderr.puts "dispatch: #{cmd_args.join ' '}"
96 i, o, e, t = Open3.popen3(*cmd_args)
98 $stderr.puts "dispatch: popen3: #{$!}"
103 $stderr.puts "dispatch: job #{job.uuid} start"
104 $stderr.puts "dispatch: child #{t.pid} start"
105 @running[job.uuid] = {
121 ActiveRecord::Base.transaction do
123 if job.is_locked_by.nil? and
124 job.update_attributes(is_locked_by: sysuser.uuid)
133 job.update_attributes is_locked_by: nil
137 @running.each do |job_uuid, j|
140 # Throw away child stdout
142 j[:stdout].read_nonblock(2**20)
143 rescue Errno::EAGAIN, EOFError
146 # Read whatever is available from child stderr
149 stderr_buf = j[:stderr].read_nonblock(2**20)
150 rescue Errno::EAGAIN, EOFError
154 j[:stderr_buf] << stderr_buf
155 if j[:stderr_buf].index "\n"
156 lines = j[:stderr_buf].lines "\n"
157 if j[:stderr_buf][-1] == "\n"
160 j[:stderr_buf] = lines.pop
163 $stderr.print "#{job_uuid} ! " unless line.index(job_uuid)
166 if (re = line.match(/#{job_uuid} (\d+) (\S*) (.*)/))
167 ignorethis, whjmpid, taskid, message = re.to_a
168 if taskid == '' and message == 'start'
169 $stderr.puts "dispatch: noticed #{job_uuid} started"
171 ActiveRecord::Base.transaction do
173 j[:job].update_attributes running: true, started_at: Time.now
175 elsif taskid == '' and (re = message.match /^revision (\S+)$/)
176 $stderr.puts "dispatch: noticed #{job_uuid} version #{re[1]}"
177 ActiveRecord::Base.transaction do
179 j[:job].script_version = re[1]
182 elsif taskid == '' and (re = message.match /^outputkey (\S+)$/)
183 $stderr.puts "dispatch: noticed #{job_uuid} output #{re[1]}"
185 elsif taskid == '' and (re = message.match /^meta key is (\S+)$/)
186 $stderr.puts "dispatch: noticed #{job_uuid} log #{re[1]}"
188 ActiveRecord::Base.transaction do
190 j[:job].update_attributes log: j[:log]
192 elsif taskid.match(/^\d+/) and (re = message.match /^failure /)
193 $stderr.puts "dispatch: noticed #{job_uuid} task fail"
194 ActiveRecord::Base.transaction do
196 j[:job].tasks_summary ||= {}
197 j[:job].tasks_summary[:failed] ||= 0
198 j[:job].tasks_summary[:failed] += 1
201 elsif (re = message.match(/^status: (\d+) done, (\d+) running, (\d+) todo/))
202 $stderr.puts "dispatch: noticed #{job_uuid} #{message}"
203 ActiveRecord::Base.transaction do
205 j[:job].tasks_summary ||= {}
206 j[:job].tasks_summary[:done] = re[1].to_i
207 j[:job].tasks_summary[:running] = re[2].to_i
208 j[:job].tasks_summary[:todo] = re[3].to_i
211 if re[2].to_i == 0 and re[3].to_i == 0
212 $stderr.puts "dispatch: noticed #{job_uuid} succeeded"
224 return if 0 == @running.size
230 pid_done = waitpid(-1, Process::WNOHANG | Process::WUNTRACED)
232 j_done = @running.values.
233 select { |j| j[:wait_thr].pid == pid_done }.
236 rescue SystemCallError
237 # I have @running processes but system reports I have no
238 # children. This is likely to happen repeatedly if it happens at
239 # all; I will log this no more than once per child process I
241 if 0 < @running.select { |uuid,j| j[:warned_waitpid_error].nil? }.size
242 children = @running.values.collect { |j| j[:wait_thr].pid }.join ' '
243 $stderr.puts "dispatch: IPC bug: waitpid() error (#{$!}), but I have children #{children}"
245 @running.each do |uuid,j| j[:warned_waitpid_error] = true end
248 @running.each do |uuid, j|
249 if j[:wait_thr].status == false
250 pid_done = j[:wait_thr].pid
258 job_done = j_done[:job]
259 $stderr.puts "dispatch: child #{pid_done} exit"
260 $stderr.puts "dispatch: job #{job_done.uuid} end"
262 # Ensure every last drop of stdout and stderr is consumed
264 if j_done[:stderr_buf] and j_done[:stderr_buf] != ''
265 $stderr.puts j_done[:stderr_buf] + "\n"
268 j_done[:wait_thr].value # wait the thread
271 # If the job never really started (due to a scheduling
272 # failure), just put it back in the queue
274 $stderr.puts "dispatch: job #{job_done.uuid} requeued"
276 # Otherwise, mark the job as finished
277 ActiveRecord::Base.transaction do
279 job_done.log = j_done[:log]
280 job_done.output = j_done[:output]
281 job_done.success = j_done[:success]
282 job_done.assert_finished
285 @running.delete job_done.uuid
291 $stderr.puts "dispatch: ready"
292 while !$signal[:term] or @running.size > 0
295 @running.each do |uuid, j|
296 if !j[:started] and j[:sent_int] < 2
298 Process.kill 'INT', j[:wait_thr].pid
300 # No such pid = race condition + desired result is
307 refresh_todo unless did_recently(:refresh_todo, 1.0)
308 start_jobs unless @todo.empty? or did_recently(:start_jobs, 1.0)
311 select(@running.values.collect { |j| [j[:stdout], j[:stderr]] }.flatten,
318 def did_recently(thing, min_interval)
320 if !@did_recently[thing] or @did_recently[thing] < Time.now - min_interval
321 @did_recently[thing] = Time.now