include Process
+$warned = {}
$signal = {}
%w{TERM INT}.each do |sig|
signame = sig
require 'open3'
$redis ||= Redis.new
+LOG_BUFFER_SIZE = 2**20
class Dispatcher
include ApplicationHelper
cmd_args << '--job'
cmd_args << job.uuid
+ if cmd_args[0] == ''
+ raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
+ end
+
commit = Commit.where(sha1: job.script_version).first
if commit
cmd_args << '--git-dir'
untake(job)
next
end
- $stderr.puts "dispatch: job #{job.uuid} start"
- $stderr.puts "dispatch: child #{t.pid} start"
+
+ $stderr.puts "dispatch: job #{job.uuid}"
+ start_banner = "dispatch: child #{t.pid} start #{Time.now.ctime.to_s}"
+ $stderr.puts start_banner
+ $redis.set job.uuid, start_banner + "\n"
+ $redis.publish job.uuid, start_banner
+ $redis.publish job.owner_uuid, start_banner
+
@running[job.uuid] = {
stdin: i,
stdout: o,
lines.each do |line|
$stderr.print "#{job_uuid} ! " unless line.index(job_uuid)
$stderr.puts line
- $redis.publish job_uuid, "#{Time.now.ctime.to_s} #{line.strip}"
+ pub_msg = "#{Time.now.ctime.to_s} #{line.strip}"
+ $redis.publish job.owner_uuid, pub_msg
+ $redis.publish job_uuid, pub_msg
+ $redis.append job_uuid, pub_msg + "\n"
+ if LOG_BUFFER_SIZE < $redis.strlen(job_uuid)
+ $redis.set(job_uuid,
+ $redis
+ .getrange(job_uuid, (LOG_BUFFER_SIZE >> 1), -1)
+ .sub(/^.*?\n/, ''))
+ end
end
end
end
else
refresh_todo unless did_recently(:refresh_todo, 1.0)
update_node_status
- start_jobs unless @todo.empty? or did_recently(:start_jobs, 1.0)
+ unless @todo.empty? or did_recently(:start_jobs, 1.0) or $signal[:term]
+ start_jobs
+ end
end
reap_children
select(@running.values.collect { |j| [j[:stdout], j[:stderr]] }.flatten,
end
end
+# This is how crunch-job child procs know where the "refresh" trigger file is
+ENV["CRUNCH_REFRESH_TRIGGER"] = Rails.configuration.crunch_refresh_trigger
+
Dispatcher.new.run