services/api/script/crunch-dispatch.rb

   1 #!/usr/bin/env ruby
   2
   3 include Process
   4
   5 $warned = {}
   6 $signal = {}
   7 %w{TERM INT}.each do |sig|
   8   signame = sig
   9   Signal.trap(sig) do
  10     $stderr.puts "Received #{signame} signal"
  11     $signal[:term] = true
  12   end
  13 end
  14
  15 if ENV["CRUNCH_DISPATCH_LOCKFILE"]
  16   lockfilename = ENV.delete "CRUNCH_DISPATCH_LOCKFILE"
  17   lockfile = File.open(lockfilename, File::RDWR|File::CREAT, 0644)
  18   unless lockfile.flock File::LOCK_EX|File::LOCK_NB
  19     abort "Lock unavailable on #{lockfilename} - exit"
  20   end
  21 end
  22
  23 ENV["RAILS_ENV"] = ARGV[0] || ENV["RAILS_ENV"] || "development"
  24
  25 require File.dirname(__FILE__) + '/../config/boot'
  26 require File.dirname(__FILE__) + '/../config/environment'
  27 require 'open3'
  28
  29 $redis ||= Redis.new
  30 LOG_BUFFER_SIZE = 2**20
  31
  32 class Dispatcher
  33   include ApplicationHelper
  34
  35   def sysuser
  36     return act_as_system_user
  37   end
  38
  39   def refresh_todo
  40     @todo = Job.queue
  41     @todo_pipelines = PipelineInstance.queue
  42   end
  43
  44   def sinfo
  45     @@slurm_version ||= Gem::Version.new(`sinfo --version`.match(/\b[\d\.]+\b/)[0])
  46     if Gem::Version.new('2.3') <= @@slurm_version
  47       `sinfo --noheader -o '%n:%t'`.strip
  48     else
  49       # Expand rows with hostname ranges (like "foo[1-3,5,9-12]:idle")
  50       # into multiple rows with one hostname each.
  51       `sinfo --noheader -o '%N:%t'`.split("\n").collect do |line|
  52         tokens = line.split ":"
  53         if (re = tokens[0].match /^(.*?)\[([-,\d]+)\]$/)
  54           re[2].split(",").collect do |range|
  55             range = range.split("-").collect(&:to_i)
  56             (range[0]..range[-1]).collect do |n|
  57               [re[1] + n.to_s, tokens[1..-1]].join ":"
  58             end
  59           end
  60         else
  61           tokens.join ":"
  62         end
  63       end.flatten.join "\n"
  64     end
  65   end
  66
  67   def update_node_status
  68     if Server::Application.config.crunch_job_wrapper.to_s.match /^slurm/
  69       @nodes_in_state = {idle: 0, alloc: 0, down: 0}
  70       @node_state ||= {}
  71       node_seen = {}
  72       begin
  73         sinfo.split("\n").
  74           each do |line|
  75           re = line.match /(\S+?):+(idle|alloc|down)/
  76           next if !re
  77
  78           # sinfo tells us about a node N times if it is shared by N partitions
  79           next if node_seen[re[1]]
  80           node_seen[re[1]] = true
  81
  82           # count nodes in each state
  83           @nodes_in_state[re[2].to_sym] += 1
  84
  85           # update our database (and cache) when a node's state changes
  86           if @node_state[re[1]] != re[2]
  87             @node_state[re[1]] = re[2]
  88             node = Node.where('hostname=?', re[1]).first
  89             if node
  90               $stderr.puts "dispatch: update #{re[1]} state to #{re[2]}"
  91               node.info[:slurm_state] = re[2]
  92               node.save
  93             elsif re[2] != 'down'
  94               $stderr.puts "dispatch: sinfo reports '#{re[1]}' is not down, but no node has that name"
  95             end
  96           end
  97         end
  98       rescue
  99       end
 100     end
 101   end
 102
 103   def start_jobs
 104     @todo.each do |job|
 105
 106       min_nodes = 1
 107       begin
 108         if job.runtime_constraints['min_nodes']
 109           min_nodes = begin job.runtime_constraints['min_nodes'].to_i rescue 1 end
 110         end
 111       end
 112
 113       begin
 114         next if @nodes_in_state[:idle] < min_nodes
 115       rescue
 116       end
 117
 118       next if @running[job.uuid]
 119       next if !take(job)
 120
 121       cmd_args = nil
 122       case Server::Application.config.crunch_job_wrapper
 123       when :none
 124         cmd_args = []
 125       when :slurm_immediate
 126         cmd_args = ["salloc",
 127                     "--chdir=/",
 128                     "--immediate",
 129                     "--exclusive",
 130                     "--no-kill",
 131                     "--job-name=#{job.uuid}",
 132                     "--nodes=#{min_nodes}"]
 133       else
 134         raise "Unknown crunch_job_wrapper: #{Server::Application.config.crunch_job_wrapper}"
 135       end
 136
 137       if Server::Application.config.crunch_job_user
 138         cmd_args.unshift("sudo", "-E", "-u",
 139                          Server::Application.config.crunch_job_user,
 140                          "PERLLIB=#{ENV['PERLLIB']}")
 141       end
 142
 143       job_auth = ApiClientAuthorization.
 144         new(user: User.where('uuid=?', job.modified_by_user_uuid).first,
 145             api_client_id: 0)
 146       job_auth.save
 147
 148       cmd_args << (ENV['CRUNCH_JOB_BIN'] || `which crunch-job`.strip)
 149       cmd_args << '--job-api-token'
 150       cmd_args << job_auth.api_token
 151       cmd_args << '--job'
 152       cmd_args << job.uuid
 153
 154       if cmd_args[0] == ''
 155         raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
 156       end
 157
 158       commit = Commit.where(sha1: job.script_version).first
 159       if commit
 160         cmd_args << '--git-dir'
 161         if File.exists?(File.
 162                         join(Rails.configuration.git_repositories_dir,
 163                              commit.repository_name + '.git'))
 164           cmd_args << File.
 165             join(Rails.configuration.git_repositories_dir,
 166                  commit.repository_name + '.git')
 167         else
 168           cmd_args << File.
 169             join(Rails.configuration.git_repositories_dir,
 170                  commit.repository_name, '.git')
 171         end
 172       end
 173
 174       $stderr.puts "dispatch: #{cmd_args.join ' '}"
 175
 176       begin
 177         i, o, e, t = Open3.popen3(*cmd_args)
 178       rescue
 179         $stderr.puts "dispatch: popen3: #{$!}"
 180         sleep 1
 181         untake(job)
 182         next
 183       end
 184
 185       $stderr.puts "dispatch: job #{job.uuid}"
 186       start_banner = "dispatch: child #{t.pid} start #{Time.now.ctime.to_s}"
 187       $stderr.puts start_banner
 188       $redis.set job.uuid, start_banner + "\n"
 189       $redis.publish job.uuid, start_banner
 190       $redis.publish job.owner_uuid, start_banner
 191
 192       @running[job.uuid] = {
 193         stdin: i,
 194         stdout: o,
 195         stderr: e,
 196         wait_thr: t,
 197         job: job,
 198         stderr_buf: '',
 199         started: false,
 200         sent_int: 0,
 201         job_auth: job_auth
 202       }
 203       i.close
 204     end
 205   end
 206
 207   def take(job)
 208     # no-op -- let crunch-job take care of locking.
 209     true
 210   end
 211
 212   def untake(job)
 213     # no-op -- let crunch-job take care of locking.
 214     true
 215   end
 216
 217   def read_pipes
 218     @running.each do |job_uuid, j|
 219       job = j[:job]
 220
 221       # Throw away child stdout
 222       begin
 223         j[:stdout].read_nonblock(2**20)
 224       rescue Errno::EAGAIN, EOFError
 225       end
 226
 227       # Read whatever is available from child stderr
 228       stderr_buf = false
 229       begin
 230         stderr_buf = j[:stderr].read_nonblock(2**20)
 231       rescue Errno::EAGAIN, EOFError
 232       end
 233
 234       if stderr_buf
 235         j[:stderr_buf] << stderr_buf
 236         if j[:stderr_buf].index "\n"
 237           lines = j[:stderr_buf].lines("\n").to_a
 238           if j[:stderr_buf][-1] == "\n"
 239             j[:stderr_buf] = ''
 240           else
 241             j[:stderr_buf] = lines.pop
 242           end
 243           lines.each do |line|
 244             $stderr.print "#{job_uuid} ! " unless line.index(job_uuid)
 245             $stderr.puts line
 246             pub_msg = "#{Time.now.ctime.to_s} #{line.strip}"
 247             $redis.publish job.owner_uuid, pub_msg
 248             $redis.publish job_uuid, pub_msg
 249             $redis.append job_uuid, pub_msg + "\n"
 250             if LOG_BUFFER_SIZE < $redis.strlen(job_uuid)
 251               $redis.set(job_uuid,
 252                          $redis
 253                            .getrange(job_uuid, (LOG_BUFFER_SIZE >> 1), -1)
 254                            .sub(/^.*?\n/, ''))
 255             end
 256           end
 257         end
 258       end
 259     end
 260   end
 261
 262   def reap_children
 263     return if 0 == @running.size
 264     pid_done = nil
 265     j_done = nil
 266
 267     if false
 268       begin
 269         pid_done = waitpid(-1, Process::WNOHANG | Process::WUNTRACED)
 270         if pid_done
 271           j_done = @running.values.
 272             select { |j| j[:wait_thr].pid == pid_done }.
 273             first
 274         end
 275       rescue SystemCallError
 276         # I have @running processes but system reports I have no
 277         # children. This is likely to happen repeatedly if it happens at
 278         # all; I will log this no more than once per child process I
 279         # start.
 280         if 0 < @running.select { |uuid,j| j[:warned_waitpid_error].nil? }.size
 281           children = @running.values.collect { |j| j[:wait_thr].pid }.join ' '
 282           $stderr.puts "dispatch: IPC bug: waitpid() error (#{$!}), but I have children #{children}"
 283         end
 284         @running.each do |uuid,j| j[:warned_waitpid_error] = true end
 285       end
 286     else
 287       @running.each do |uuid, j|
 288         if j[:wait_thr].status == false
 289           pid_done = j[:wait_thr].pid
 290           j_done = j
 291         end
 292       end
 293     end
 294
 295     return if !pid_done
 296
 297     job_done = j_done[:job]
 298     $stderr.puts "dispatch: child #{pid_done} exit"
 299     $stderr.puts "dispatch: job #{job_done.uuid} end"
 300     $redis.publish job_done.uuid, "end"
 301
 302     # Ensure every last drop of stdout and stderr is consumed
 303     read_pipes
 304     if j_done[:stderr_buf] and j_done[:stderr_buf] != ''
 305       $stderr.puts j_done[:stderr_buf] + "\n"
 306     end
 307
 308     # Wait the thread
 309     j_done[:wait_thr].value
 310
 311     # Invalidate the per-job auth token
 312     j_done[:job_auth].update_attributes expires_at: Time.now
 313
 314     @running.delete job_done.uuid
 315   end
 316
 317   def update_pipelines
 318     @todo_pipelines.each do |p|
 319       pipe_auth = ApiClientAuthorization.
 320         new(user: User.where('uuid=?', p.modified_by_user_uuid).first,
 321             api_client_id: 0)
 322       pipe_auth.save
 323
 324       puts `export ARVADOS_API_TOKEN=#{pipe_auth.api_token} && arv-run-pipeline-instance --run-here --no-wait --instance #{p.uuid}`
 325     end
 326   end
 327
 328   def run
 329     act_as_system_user
 330     @running ||= {}
 331     $stderr.puts "dispatch: ready"
 332     while !$signal[:term] or @running.size > 0
 333       read_pipes
 334       if $signal[:term]
 335         @running.each do |uuid, j|
 336           if !j[:started] and j[:sent_int] < 2
 337             begin
 338               Process.kill 'INT', j[:wait_thr].pid
 339             rescue Errno::ESRCH
 340               # No such pid = race condition + desired result is
 341               # already achieved
 342             end
 343             j[:sent_int] += 1
 344           end
 345         end
 346       else
 347         refresh_todo unless did_recently(:refresh_todo, 1.0)
 348         update_node_status
 349         unless @todo.empty? or did_recently(:start_jobs, 1.0) or $signal[:term]
 350           start_jobs
 351         end
 352         unless @todo_pipelines.empty? or did_recently(:update_pipelines, 5.0)
 353           update_pipelines
 354         end
 355       end
 356       reap_children
 357       select(@running.values.collect { |j| [j[:stdout], j[:stderr]] }.flatten,
 358              [], [], 1)
 359     end
 360   end
 361
 362
 363
 364   protected
 365
 366   def did_recently(thing, min_interval)
 367     @did_recently ||= {}
 368     if !@did_recently[thing] or @did_recently[thing] < Time.now - min_interval
 369       @did_recently[thing] = Time.now
 370       false
 371     else
 372       true
 373     end
 374   end
 375 end
 376
 377 # This is how crunch-job child procs know where the "refresh" trigger file is
 378 ENV["CRUNCH_REFRESH_TRIGGER"] = Rails.configuration.crunch_refresh_trigger
 379
 380 Dispatcher.new.run