include Process
+$warned = {}
$signal = {}
%w{TERM INT}.each do |sig|
signame = sig
$signal[:term] = true
end
end
+Signal.trap('HUP') do
+ $stderr.puts "Received HUP signal"
+ $signal[:hup] = true
+end
+
+if ENV["CRUNCH_DISPATCH_LOCKFILE"]
+ lockfilename = ENV.delete "CRUNCH_DISPATCH_LOCKFILE"
+ lockfile = File.open(lockfilename, File::RDWR|File::CREAT, 0644)
+ unless lockfile.flock File::LOCK_EX|File::LOCK_NB
+ abort "Lock unavailable on #{lockfilename} - exit"
+ end
+end
ENV["RAILS_ENV"] = ARGV[0] || ENV["RAILS_ENV"] || "development"
require 'open3'
$redis ||= Redis.new
+LOG_BUFFER_SIZE = 2**20
class Dispatcher
include ApplicationHelper
@todo = Job.queue
end
- def start_jobs
+ def update_node_status
if Server::Application.config.crunch_job_wrapper.to_s.match /^slurm/
- @idle_slurm_nodes = 0
+ @nodes_in_state = {idle: 0, alloc: 0, down: 0}
+ @node_state ||= {}
+ node_seen = {}
begin
- `sinfo`.
+ `sinfo --noheader -o '%n:%t'`.
split("\n").
- collect { |line| line.match /(\d+) +idle/ }.
- each do |re|
- @idle_slurm_nodes = re[1].to_i if re
+ each do |line|
+ re = line.match /(\S+?):+(idle|alloc|down)/
+ next if !re
+
+ # sinfo tells us about a node N times if it is shared by N partitions
+ next if node_seen[re[1]]
+ node_seen[re[1]] = true
+
+ # count nodes in each state
+ @nodes_in_state[re[2].to_sym] += 1
+
+ # update our database (and cache) when a node's state changes
+ if @node_state[re[1]] != re[2]
+ @node_state[re[1]] = re[2]
+ node = Node.where('hostname=?', re[1]).first
+ if node
+ $stderr.puts "dispatch: update #{re[1]} state to #{re[2]}"
+ node.info[:slurm_state] = re[2]
+ node.save
+ elsif re[2] != 'down'
+ $stderr.puts "dispatch: sinfo reports '#{re[1]}' is not down, but no node has that name"
+ end
+ end
end
rescue
end
end
+ end
+ def start_jobs
@todo.each do |job|
min_nodes = 1
begin
- if job.resource_limits['min_nodes']
- min_nodes = begin job.resource_limits['min_nodes'].to_i rescue 1 end
+ if job.runtime_constraints['min_nodes']
+ min_nodes = begin job.runtime_constraints['min_nodes'].to_i rescue 1 end
end
end
- next if @idle_slurm_nodes and @idle_slurm_nodes < min_nodes
+
+ begin
+ next if @nodes_in_state[:idle] < min_nodes
+ rescue
+ end
next if @running[job.uuid]
next if !take(job)
cmd_args << '--job'
cmd_args << job.uuid
+ if cmd_args[0] == ''
+ raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
+ end
+
commit = Commit.where(sha1: job.script_version).first
if commit
cmd_args << '--git-dir'
untake(job)
next
end
- $stderr.puts "dispatch: job #{job.uuid} start"
- $stderr.puts "dispatch: child #{t.pid} start"
+
+ $stderr.puts "dispatch: job #{job.uuid}"
+ start_banner = "dispatch: child #{t.pid} start #{Time.now.ctime.to_s}"
+ $stderr.puts start_banner
+ $redis.set job.uuid, start_banner + "\n"
+ $redis.publish job.uuid, start_banner
+ $redis.publish job.owner_uuid, start_banner
+
@running[job.uuid] = {
stdin: i,
stdout: o,
lines.each do |line|
$stderr.print "#{job_uuid} ! " unless line.index(job_uuid)
$stderr.puts line
- $redis.publish job_uuid, "#{Time.now.ctime.to_s} #{line}"
+ pub_msg = "#{Time.now.ctime.to_s} #{line.strip}"
+ $redis.publish job.owner_uuid, pub_msg
+ $redis.publish job_uuid, pub_msg
+ $redis.append job_uuid, pub_msg + "\n"
+ if LOG_BUFFER_SIZE < $redis.strlen(job_uuid)
+ $redis.set(job_uuid,
+ $redis
+ .getrange(job_uuid, (LOG_BUFFER_SIZE >> 1), -1)
+ .sub(/^.*?\n/, ''))
+ end
end
end
end
end
end
else
+ if File.exists?(Rails.configuration.crunch_dispatch_hup_trigger)
+ begin
+ File.unlink(Rails.configuration.crunch_dispatch_hup_trigger)
+ $signal[:hup] = true
+ rescue Errno::ENOENT
+ $stderr.puts "Weird, hup_trigger file was deleted by someone else."
+ rescue Errno::EPERM
+ if not $warned[:hup_trigger_perm]
+ $warned[:hup_trigger_perm] = true
+ $stderr.puts "Install problem: I see the hup_trigger file but cannot delete it."
+ end
+ end
+ end
+ if $signal[:hup]
+ # Pass HUP through to all crunch-job processes.
+ @running.each do |uuid, j|
+ begin
+ Process.kill 'HUP', j[:wait_thr].pid
+ rescue Errno::ESRCH
+ # Process ended but hasn't been reaped. Nothing to do.
+ end
+ end
+ $signal.delete :hup
+ end
refresh_todo unless did_recently(:refresh_todo, 1.0)
+ update_node_status
start_jobs unless @todo.empty? or did_recently(:start_jobs, 1.0)
end
reap_children
projects / arvados.git / blobdiff