services/api/script/crunch-dispatch.rb

   1 #!/usr/bin/env ruby
   2
   3 include Process
   4
   5 $warned = {}
   6 $signal = {}
   7 %w{TERM INT}.each do |sig|
   8   signame = sig
   9   Signal.trap(sig) do
  10     $stderr.puts "Received #{signame} signal"
  11     $signal[:term] = true
  12   end
  13 end
  14
  15 if ENV["CRUNCH_DISPATCH_LOCKFILE"]
  16   lockfilename = ENV.delete "CRUNCH_DISPATCH_LOCKFILE"
  17   lockfile = File.open(lockfilename, File::RDWR|File::CREAT, 0644)
  18   unless lockfile.flock File::LOCK_EX|File::LOCK_NB
  19     abort "Lock unavailable on #{lockfilename} - exit"
  20   end
  21 end
  22
  23 ENV["RAILS_ENV"] = ARGV[0] || ENV["RAILS_ENV"] || "development"
  24
  25 require File.dirname(__FILE__) + '/../config/boot'
  26 require File.dirname(__FILE__) + '/../config/environment'
  27 require 'open3'
  28
  29 $redis ||= Redis.new
  30 LOG_BUFFER_SIZE = 2**20
  31
  32 class Dispatcher
  33   include ApplicationHelper
  34
  35   def sysuser
  36     return act_as_system_user
  37   end
  38
  39   def refresh_todo
  40     @todo = Job.queue
  41   end
  42
  43   def update_node_status
  44     if Server::Application.config.crunch_job_wrapper.to_s.match /^slurm/
  45       @nodes_in_state = {idle: 0, alloc: 0, down: 0}
  46       @node_state ||= {}
  47       node_seen = {}
  48       begin
  49         `sinfo --noheader -o '%n:%t'`.
  50           split("\n").
  51           each do |line|
  52           re = line.match /(\S+?):+(idle|alloc|down)/
  53           next if !re
  54
  55           # sinfo tells us about a node N times if it is shared by N partitions
  56           next if node_seen[re[1]]
  57           node_seen[re[1]] = true
  58
  59           # count nodes in each state
  60           @nodes_in_state[re[2].to_sym] += 1
  61
  62           # update our database (and cache) when a node's state changes
  63           if @node_state[re[1]] != re[2]
  64             @node_state[re[1]] = re[2]
  65             node = Node.where('hostname=?', re[1]).first
  66             if node
  67               $stderr.puts "dispatch: update #{re[1]} state to #{re[2]}"
  68               node.info[:slurm_state] = re[2]
  69               node.save
  70             elsif re[2] != 'down'
  71               $stderr.puts "dispatch: sinfo reports '#{re[1]}' is not down, but no node has that name"
  72             end
  73           end
  74         end
  75       rescue
  76       end
  77     end
  78   end
  79
  80   def start_jobs
  81     @todo.each do |job|
  82
  83       min_nodes = 1
  84       begin
  85         if job.runtime_constraints['min_nodes']
  86           min_nodes = begin job.runtime_constraints['min_nodes'].to_i rescue 1 end
  87         end
  88       end
  89
  90       begin
  91         next if @nodes_in_state[:idle] < min_nodes
  92       rescue
  93       end
  94
  95       next if @running[job.uuid]
  96       next if !take(job)
  97
  98       cmd_args = nil
  99       case Server::Application.config.crunch_job_wrapper
 100       when :none
 101         cmd_args = []
 102       when :slurm_immediate
 103         cmd_args = ["salloc",
 104                     "--chdir=/",
 105                     "--immediate",
 106                     "--exclusive",
 107                     "--no-kill",
 108                     "--job-name=#{job.uuid}",
 109                     "--nodes=#{min_nodes}"]
 110       else
 111         raise "Unknown crunch_job_wrapper: #{Server::Application.config.crunch_job_wrapper}"
 112       end
 113
 114       if Server::Application.config.crunch_job_user
 115         cmd_args.unshift("sudo", "-E", "-u",
 116                          Server::Application.config.crunch_job_user,
 117                          "PERLLIB=#{ENV['PERLLIB']}")
 118       end
 119
 120       job_auth = ApiClientAuthorization.
 121         new(user: User.where('uuid=?', job.modified_by_user_uuid).first,
 122             api_client_id: 0)
 123       job_auth.save
 124
 125       cmd_args << (ENV['CRUNCH_JOB_BIN'] || `which crunch-job`.strip)
 126       cmd_args << '--job-api-token'
 127       cmd_args << job_auth.api_token
 128       cmd_args << '--job'
 129       cmd_args << job.uuid
 130
 131       if cmd_args[0] == ''
 132         raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
 133       end
 134
 135       commit = Commit.where(sha1: job.script_version).first
 136       if commit
 137         cmd_args << '--git-dir'
 138         if File.exists?(File.
 139                         join(Rails.configuration.git_repositories_dir,
 140                              commit.repository_name + '.git'))
 141           cmd_args << File.
 142             join(Rails.configuration.git_repositories_dir,
 143                  commit.repository_name + '.git')
 144         else
 145           cmd_args << File.
 146             join(Rails.configuration.git_repositories_dir,
 147                  commit.repository_name, '.git')
 148         end
 149       end
 150
 151       $stderr.puts "dispatch: #{cmd_args.join ' '}"
 152
 153       begin
 154         i, o, e, t = Open3.popen3(*cmd_args)
 155       rescue
 156         $stderr.puts "dispatch: popen3: #{$!}"
 157         sleep 1
 158         untake(job)
 159         next
 160       end
 161
 162       $stderr.puts "dispatch: job #{job.uuid}"
 163       start_banner = "dispatch: child #{t.pid} start #{Time.now.ctime.to_s}"
 164       $stderr.puts start_banner
 165       $redis.set job.uuid, start_banner + "\n"
 166       $redis.publish job.uuid, start_banner
 167       $redis.publish job.owner_uuid, start_banner
 168
 169       @running[job.uuid] = {
 170         stdin: i,
 171         stdout: o,
 172         stderr: e,
 173         wait_thr: t,
 174         job: job,
 175         stderr_buf: '',
 176         started: false,
 177         sent_int: 0,
 178         job_auth: job_auth
 179       }
 180       i.close
 181     end
 182   end
 183
 184   def take(job)
 185     # no-op -- let crunch-job take care of locking.
 186     true
 187   end
 188
 189   def untake(job)
 190     # no-op -- let crunch-job take care of locking.
 191     true
 192   end
 193
 194   def read_pipes
 195     @running.each do |job_uuid, j|
 196       job = j[:job]
 197
 198       # Throw away child stdout
 199       begin
 200         j[:stdout].read_nonblock(2**20)
 201       rescue Errno::EAGAIN, EOFError
 202       end
 203
 204       # Read whatever is available from child stderr
 205       stderr_buf = false
 206       begin
 207         stderr_buf = j[:stderr].read_nonblock(2**20)
 208       rescue Errno::EAGAIN, EOFError
 209       end
 210
 211       if stderr_buf
 212         j[:stderr_buf] << stderr_buf
 213         if j[:stderr_buf].index "\n"
 214           lines = j[:stderr_buf].lines("\n").to_a
 215           if j[:stderr_buf][-1] == "\n"
 216             j[:stderr_buf] = ''
 217           else
 218             j[:stderr_buf] = lines.pop
 219           end
 220           lines.each do |line|
 221             $stderr.print "#{job_uuid} ! " unless line.index(job_uuid)
 222             $stderr.puts line
 223             pub_msg = "#{Time.now.ctime.to_s} #{line.strip}"
 224             $redis.publish job.owner_uuid, pub_msg
 225             $redis.publish job_uuid, pub_msg
 226             $redis.append job_uuid, pub_msg + "\n"
 227             if LOG_BUFFER_SIZE < $redis.strlen(job_uuid)
 228               $redis.set(job_uuid,
 229                          $redis
 230                            .getrange(job_uuid, (LOG_BUFFER_SIZE >> 1), -1)
 231                            .sub(/^.*?\n/, ''))
 232             end
 233           end
 234         end
 235       end
 236     end
 237   end
 238
 239   def reap_children
 240     return if 0 == @running.size
 241     pid_done = nil
 242     j_done = nil
 243
 244     if false
 245       begin
 246         pid_done = waitpid(-1, Process::WNOHANG | Process::WUNTRACED)
 247         if pid_done
 248           j_done = @running.values.
 249             select { |j| j[:wait_thr].pid == pid_done }.
 250             first
 251         end
 252       rescue SystemCallError
 253         # I have @running processes but system reports I have no
 254         # children. This is likely to happen repeatedly if it happens at
 255         # all; I will log this no more than once per child process I
 256         # start.
 257         if 0 < @running.select { |uuid,j| j[:warned_waitpid_error].nil? }.size
 258           children = @running.values.collect { |j| j[:wait_thr].pid }.join ' '
 259           $stderr.puts "dispatch: IPC bug: waitpid() error (#{$!}), but I have children #{children}"
 260         end
 261         @running.each do |uuid,j| j[:warned_waitpid_error] = true end
 262       end
 263     else
 264       @running.each do |uuid, j|
 265         if j[:wait_thr].status == false
 266           pid_done = j[:wait_thr].pid
 267           j_done = j
 268         end
 269       end
 270     end
 271
 272     return if !pid_done
 273
 274     job_done = j_done[:job]
 275     $stderr.puts "dispatch: child #{pid_done} exit"
 276     $stderr.puts "dispatch: job #{job_done.uuid} end"
 277     $redis.publish job_done.uuid, "end"
 278
 279     # Ensure every last drop of stdout and stderr is consumed
 280     read_pipes
 281     if j_done[:stderr_buf] and j_done[:stderr_buf] != ''
 282       $stderr.puts j_done[:stderr_buf] + "\n"
 283     end
 284
 285     # Wait the thread
 286     j_done[:wait_thr].value
 287
 288     # Invalidate the per-job auth token
 289     j_done[:job_auth].update_attributes expires_at: Time.now
 290
 291     @running.delete job_done.uuid
 292   end
 293
 294   def run
 295     act_as_system_user
 296     @running ||= {}
 297     $stderr.puts "dispatch: ready"
 298     while !$signal[:term] or @running.size > 0
 299       read_pipes
 300       if $signal[:term]
 301         @running.each do |uuid, j|
 302           if !j[:started] and j[:sent_int] < 2
 303             begin
 304               Process.kill 'INT', j[:wait_thr].pid
 305             rescue Errno::ESRCH
 306               # No such pid = race condition + desired result is
 307               # already achieved
 308             end
 309             j[:sent_int] += 1
 310           end
 311         end
 312       else
 313         refresh_todo unless did_recently(:refresh_todo, 1.0)
 314         update_node_status
 315         unless @todo.empty? or did_recently(:start_jobs, 1.0) or $signal[:term]
 316           start_jobs
 317         end
 318       end
 319       reap_children
 320       select(@running.values.collect { |j| [j[:stdout], j[:stderr]] }.flatten,
 321              [], [], 1)
 322     end
 323   end
 324
 325   protected
 326
 327   def did_recently(thing, min_interval)
 328     @did_recently ||= {}
 329     if !@did_recently[thing] or @did_recently[thing] < Time.now - min_interval
 330       @did_recently[thing] = Time.now
 331       false
 332     else
 333       true
 334     end
 335   end
 336 end
 337
 338 # This is how crunch-job child procs know where the "refresh" trigger file is
 339 ENV["CRUNCH_REFRESH_TRIGGER"] = Rails.configuration.crunch_refresh_trigger
 340
 341 Dispatcher.new.run