services/api/script/crunch-dispatch.rb

   1 #!/usr/bin/env ruby
   2
   3 include Process
   4
   5 $warned = {}
   6 $signal = {}
   7 %w{TERM INT}.each do |sig|
   8   signame = sig
   9   Signal.trap(sig) do
  10     $stderr.puts "Received #{signame} signal"
  11     $signal[:term] = true
  12   end
  13 end
  14 Signal.trap('HUP') do
  15   $stderr.puts "Received HUP signal"
  16   $signal[:hup] = true
  17 end
  18
  19 if ENV["CRUNCH_DISPATCH_LOCKFILE"]
  20   lockfilename = ENV.delete "CRUNCH_DISPATCH_LOCKFILE"
  21   lockfile = File.open(lockfilename, File::RDWR|File::CREAT, 0644)
  22   unless lockfile.flock File::LOCK_EX|File::LOCK_NB
  23     abort "Lock unavailable on #{lockfilename} - exit"
  24   end
  25 end
  26
  27 ENV["RAILS_ENV"] = ARGV[0] || ENV["RAILS_ENV"] || "development"
  28
  29 require File.dirname(__FILE__) + '/../config/boot'
  30 require File.dirname(__FILE__) + '/../config/environment'
  31 require 'open3'
  32
  33 $redis ||= Redis.new
  34 LOG_BUFFER_SIZE = 2**20
  35
  36 class Dispatcher
  37   include ApplicationHelper
  38
  39   def sysuser
  40     return act_as_system_user
  41   end
  42
  43   def refresh_todo
  44     @todo = Job.queue
  45   end
  46
  47   def update_node_status
  48     if Server::Application.config.crunch_job_wrapper.to_s.match /^slurm/
  49       @nodes_in_state = {idle: 0, alloc: 0, down: 0}
  50       @node_state ||= {}
  51       node_seen = {}
  52       begin
  53         `sinfo --noheader -o '%n:%t'`.
  54           split("\n").
  55           each do |line|
  56           re = line.match /(\S+?):+(idle|alloc|down)/
  57           next if !re
  58
  59           # sinfo tells us about a node N times if it is shared by N partitions
  60           next if node_seen[re[1]]
  61           node_seen[re[1]] = true
  62
  63           # count nodes in each state
  64           @nodes_in_state[re[2].to_sym] += 1
  65
  66           # update our database (and cache) when a node's state changes
  67           if @node_state[re[1]] != re[2]
  68             @node_state[re[1]] = re[2]
  69             node = Node.where('hostname=?', re[1]).first
  70             if node
  71               $stderr.puts "dispatch: update #{re[1]} state to #{re[2]}"
  72               node.info[:slurm_state] = re[2]
  73               node.save
  74             elsif re[2] != 'down'
  75               $stderr.puts "dispatch: sinfo reports '#{re[1]}' is not down, but no node has that name"
  76             end
  77           end
  78         end
  79       rescue
  80       end
  81     end
  82   end
  83
  84   def start_jobs
  85     @todo.each do |job|
  86
  87       min_nodes = 1
  88       begin
  89         if job.runtime_constraints['min_nodes']
  90           min_nodes = begin job.runtime_constraints['min_nodes'].to_i rescue 1 end
  91         end
  92       end
  93
  94       begin
  95         next if @nodes_in_state[:idle] < min_nodes
  96       rescue
  97       end
  98
  99       next if @running[job.uuid]
 100       next if !take(job)
 101
 102       cmd_args = nil
 103       case Server::Application.config.crunch_job_wrapper
 104       when :none
 105         cmd_args = []
 106       when :slurm_immediate
 107         cmd_args = ["salloc",
 108                     "--chdir=/",
 109                     "--immediate",
 110                     "--exclusive",
 111                     "--no-kill",
 112                     "--job-name=#{job.uuid}",
 113                     "--nodes=#{min_nodes}"]
 114       else
 115         raise "Unknown crunch_job_wrapper: #{Server::Application.config.crunch_job_wrapper}"
 116       end
 117
 118       if Server::Application.config.crunch_job_user
 119         cmd_args.unshift("sudo", "-E", "-u",
 120                          Server::Application.config.crunch_job_user,
 121                          "PERLLIB=#{ENV['PERLLIB']}")
 122       end
 123
 124       job_auth = ApiClientAuthorization.
 125         new(user: User.where('uuid=?', job.modified_by_user_uuid).first,
 126             api_client_id: 0)
 127       job_auth.save
 128
 129       cmd_args << (ENV['CRUNCH_JOB_BIN'] || `which crunch-job`.strip)
 130       cmd_args << '--job-api-token'
 131       cmd_args << job_auth.api_token
 132       cmd_args << '--job'
 133       cmd_args << job.uuid
 134
 135       if cmd_args[0] == ''
 136         raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
 137       end
 138
 139       commit = Commit.where(sha1: job.script_version).first
 140       if commit
 141         cmd_args << '--git-dir'
 142         if File.exists?(File.
 143                         join(Rails.configuration.git_repositories_dir,
 144                              commit.repository_name + '.git'))
 145           cmd_args << File.
 146             join(Rails.configuration.git_repositories_dir,
 147                  commit.repository_name + '.git')
 148         else
 149           cmd_args << File.
 150             join(Rails.configuration.git_repositories_dir,
 151                  commit.repository_name, '.git')
 152         end
 153       end
 154
 155       $stderr.puts "dispatch: #{cmd_args.join ' '}"
 156
 157       begin
 158         i, o, e, t = Open3.popen3(*cmd_args)
 159       rescue
 160         $stderr.puts "dispatch: popen3: #{$!}"
 161         sleep 1
 162         untake(job)
 163         next
 164       end
 165
 166       $stderr.puts "dispatch: job #{job.uuid}"
 167       start_banner = "dispatch: child #{t.pid} start #{Time.now.ctime.to_s}"
 168       $stderr.puts start_banner
 169       $redis.set job.uuid, start_banner + "\n"
 170       $redis.publish job.uuid, start_banner
 171       $redis.publish job.owner_uuid, start_banner
 172
 173       @running[job.uuid] = {
 174         stdin: i,
 175         stdout: o,
 176         stderr: e,
 177         wait_thr: t,
 178         job: job,
 179         stderr_buf: '',
 180         started: false,
 181         sent_int: 0,
 182         job_auth: job_auth
 183       }
 184       i.close
 185     end
 186   end
 187
 188   def take(job)
 189     # no-op -- let crunch-job take care of locking.
 190     true
 191   end
 192
 193   def untake(job)
 194     # no-op -- let crunch-job take care of locking.
 195     true
 196   end
 197
 198   def read_pipes
 199     @running.each do |job_uuid, j|
 200       job = j[:job]
 201
 202       # Throw away child stdout
 203       begin
 204         j[:stdout].read_nonblock(2**20)
 205       rescue Errno::EAGAIN, EOFError
 206       end
 207
 208       # Read whatever is available from child stderr
 209       stderr_buf = false
 210       begin
 211         stderr_buf = j[:stderr].read_nonblock(2**20)
 212       rescue Errno::EAGAIN, EOFError
 213       end
 214
 215       if stderr_buf
 216         j[:stderr_buf] << stderr_buf
 217         if j[:stderr_buf].index "\n"
 218           lines = j[:stderr_buf].lines("\n").to_a
 219           if j[:stderr_buf][-1] == "\n"
 220             j[:stderr_buf] = ''
 221           else
 222             j[:stderr_buf] = lines.pop
 223           end
 224           lines.each do |line|
 225             $stderr.print "#{job_uuid} ! " unless line.index(job_uuid)
 226             $stderr.puts line
 227             pub_msg = "#{Time.now.ctime.to_s} #{line.strip}"
 228             $redis.publish job.owner_uuid, pub_msg
 229             $redis.publish job_uuid, pub_msg
 230             $redis.append job_uuid, pub_msg + "\n"
 231             if LOG_BUFFER_SIZE < $redis.strlen(job_uuid)
 232               $redis.set(job_uuid,
 233                          $redis
 234                            .getrange(job_uuid, (LOG_BUFFER_SIZE >> 1), -1)
 235                            .sub(/^.*?\n/, ''))
 236             end
 237           end
 238         end
 239       end
 240     end
 241   end
 242
 243   def reap_children
 244     return if 0 == @running.size
 245     pid_done = nil
 246     j_done = nil
 247
 248     if false
 249       begin
 250         pid_done = waitpid(-1, Process::WNOHANG | Process::WUNTRACED)
 251         if pid_done
 252           j_done = @running.values.
 253             select { |j| j[:wait_thr].pid == pid_done }.
 254             first
 255         end
 256       rescue SystemCallError
 257         # I have @running processes but system reports I have no
 258         # children. This is likely to happen repeatedly if it happens at
 259         # all; I will log this no more than once per child process I
 260         # start.
 261         if 0 < @running.select { |uuid,j| j[:warned_waitpid_error].nil? }.size
 262           children = @running.values.collect { |j| j[:wait_thr].pid }.join ' '
 263           $stderr.puts "dispatch: IPC bug: waitpid() error (#{$!}), but I have children #{children}"
 264         end
 265         @running.each do |uuid,j| j[:warned_waitpid_error] = true end
 266       end
 267     else
 268       @running.each do |uuid, j|
 269         if j[:wait_thr].status == false
 270           pid_done = j[:wait_thr].pid
 271           j_done = j
 272         end
 273       end
 274     end
 275
 276     return if !pid_done
 277
 278     job_done = j_done[:job]
 279     $stderr.puts "dispatch: child #{pid_done} exit"
 280     $stderr.puts "dispatch: job #{job_done.uuid} end"
 281     $redis.publish job_done.uuid, "end"
 282
 283     # Ensure every last drop of stdout and stderr is consumed
 284     read_pipes
 285     if j_done[:stderr_buf] and j_done[:stderr_buf] != ''
 286       $stderr.puts j_done[:stderr_buf] + "\n"
 287     end
 288
 289     # Wait the thread
 290     j_done[:wait_thr].value
 291
 292     # Invalidate the per-job auth token
 293     j_done[:job_auth].update_attributes expires_at: Time.now
 294
 295     @running.delete job_done.uuid
 296   end
 297
 298   def run
 299     act_as_system_user
 300     @running ||= {}
 301     $stderr.puts "dispatch: ready"
 302     while !$signal[:term] or @running.size > 0
 303       read_pipes
 304       if $signal[:term]
 305         @running.each do |uuid, j|
 306           if !j[:started] and j[:sent_int] < 2
 307             begin
 308               Process.kill 'INT', j[:wait_thr].pid
 309             rescue Errno::ESRCH
 310               # No such pid = race condition + desired result is
 311               # already achieved
 312             end
 313             j[:sent_int] += 1
 314           end
 315         end
 316       else
 317         if File.exists?(Rails.configuration.crunch_dispatch_hup_trigger)
 318           begin
 319             File.unlink(Rails.configuration.crunch_dispatch_hup_trigger)
 320             $signal[:hup] = true
 321           rescue Errno::ENOENT
 322             $stderr.puts "Weird, hup_trigger file was deleted by someone else."
 323           rescue Errno::EPERM
 324             if not $warned[:hup_trigger_perm]
 325               $warned[:hup_trigger_perm] = true
 326               $stderr.puts "Install problem: I see the hup_trigger file but cannot delete it."
 327             end
 328           end
 329         end
 330         if $signal[:hup]
 331           # Pass HUP through to all crunch-job processes.
 332           @running.each do |uuid, j|
 333             begin
 334               Process.kill 'HUP', j[:wait_thr].pid
 335             rescue Errno::ESRCH
 336               # Process ended but hasn't been reaped. Nothing to do.
 337             end
 338           end
 339           $signal.delete :hup
 340         end
 341         refresh_todo unless did_recently(:refresh_todo, 1.0)
 342         update_node_status
 343         start_jobs unless @todo.empty? or did_recently(:start_jobs, 1.0)
 344       end
 345       reap_children
 346       select(@running.values.collect { |j| [j[:stdout], j[:stderr]] }.flatten,
 347              [], [], 1)
 348     end
 349   end
 350
 351   protected
 352
 353   def did_recently(thing, min_interval)
 354     @did_recently ||= {}
 355     if !@did_recently[thing] or @did_recently[thing] < Time.now - min_interval
 356       @did_recently[thing] = Time.now
 357       false
 358     else
 359       true
 360     end
 361   end
 362 end
 363
 364 Dispatcher.new.run