X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/9816d2cf5f88d19e4e492c1e965874e5a5b3055c..263cd68f5ae4b114d3c1c89f84be46b0f64f9c9e:/services/api/script/crunch-dispatch.rb diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb index 1002f91347..4a1fdbce75 100755 --- a/services/api/script/crunch-dispatch.rb +++ b/services/api/script/crunch-dispatch.rb @@ -1,5 +1,9 @@ #!/usr/bin/env ruby +# We want files written by crunch-dispatch to be writable by other processes +# with the same GID, see bug #7228 +File.umask(0002) + require 'shellwords' include Process @@ -54,6 +58,8 @@ class Dispatcher include ApplicationHelper EXIT_TEMPFAIL = 75 + EXIT_RETRY_UNLOCKED = 93 + RETRY_UNLOCKED_LIMIT = 3 def initialize @crunch_job_bin = (ENV['CRUNCH_JOB_BIN'] || `which arv-crunch-job`.strip) @@ -61,6 +67,8 @@ class Dispatcher raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path." end + @docker_bin = ENV['CRUNCH_JOB_DOCKER_BIN'] + @arvados_internal = Rails.configuration.git_internal_dir if not File.exists? @arvados_internal $stderr.puts `mkdir -p #{@arvados_internal.shellescape} && git init --bare #{@arvados_internal.shellescape}` @@ -77,6 +85,8 @@ class Dispatcher @pipe_auth_tokens = {} @running = {} @todo = [] + @todo_job_retries = {} + @job_retry_counts = Hash.new(0) @todo_pipelines = [] end @@ -86,7 +96,7 @@ class Dispatcher def refresh_todo if $options[:jobs] - @todo = Job.queue.select(&:repository) + @todo = @todo_job_retries.values + Job.queue.select(&:repository) end if $options[:pipelines] @todo_pipelines = PipelineInstance.queue @@ -368,6 +378,7 @@ class Dispatcher if Server::Application.config.crunch_job_user cmd_args.unshift("sudo", "-E", "-u", Server::Application.config.crunch_job_user, + "LD_LIBRARY_PATH=#{ENV['LD_LIBRARY_PATH']}", "PATH=#{ENV['PATH']}", "PERLLIB=#{ENV['PERLLIB']}", "PYTHONPATH=#{ENV['PYTHONPATH']}", @@ -417,6 +428,14 @@ class Dispatcher '--job', job.uuid, '--git-dir', @arvados_internal] + if @docker_bin + cmd_args += ['--docker-bin', @docker_bin] + end + + if @todo_job_retries.include?(job.uuid) + cmd_args << "--force-unlock" + end + $stderr.puts "dispatch: #{cmd_args.join ' '}" begin @@ -452,6 +471,7 @@ class Dispatcher log_throttle_bytes_skipped: 0, } i.close + @todo_job_retries.delete(job.uuid) update_node_status end end @@ -634,8 +654,6 @@ class Dispatcher return if !pid_done job_done = j_done[:job] - $stderr.puts "dispatch: child #{pid_done} exit" - $stderr.puts "dispatch: job #{job_done.uuid} end" # Ensure every last drop of stdout and stderr is consumed. read_pipes @@ -652,23 +670,49 @@ class Dispatcher # Wait the thread (returns a Process::Status) exit_status = j_done[:wait_thr].value.exitstatus + exit_tempfail = exit_status == EXIT_TEMPFAIL + + $stderr.puts "dispatch: child #{pid_done} exit #{exit_status}" + $stderr.puts "dispatch: job #{job_done.uuid} end" jobrecord = Job.find_by_uuid(job_done.uuid) - if exit_status != EXIT_TEMPFAIL and jobrecord.state == "Running" - # crunch-job did not return exit code 75 (see below) and left the job in - # the "Running" state, which means there was an unhandled error. Fail - # the job. - jobrecord.state = "Failed" - if not jobrecord.save - $stderr.puts "dispatch: jobrecord.save failed" + + if exit_status == EXIT_RETRY_UNLOCKED + # The job failed because all of the nodes allocated to it + # failed. Only this crunch-dispatch process can retry the job: + # it's already locked, and there's no way to put it back in the + # Queued state. Put it in our internal todo list unless the job + # has failed this way excessively. + @job_retry_counts[jobrecord.uuid] += 1 + exit_tempfail = @job_retry_counts[jobrecord.uuid] <= RETRY_UNLOCKED_LIMIT + if exit_tempfail + @todo_job_retries[jobrecord.uuid] = jobrecord + else + $stderr.puts("dispatch: job #{jobrecord.uuid} exceeded node failure retry limit -- giving up") + end + end + + if !exit_tempfail + @job_retry_counts.delete(jobrecord.uuid) + if jobrecord.state == "Running" + # Apparently there was an unhandled error. That could potentially + # include "all allocated nodes failed" when we don't to retry + # because the job has already been retried RETRY_UNLOCKED_LIMIT + # times. Fail the job. + jobrecord.state = "Failed" + if not jobrecord.save + $stderr.puts "dispatch: jobrecord.save failed" + end end else - # Don't fail the job if crunch-job didn't even get as far as - # starting it. If the job failed to run due to an infrastructure + # If the job failed to run due to an infrastructure # issue with crunch-job or slurm, we want the job to stay in the # queue. If crunch-job exited after losing a race to another # crunch-job process, it exits 75 and we should leave the job - # record alone so the winner of the race do its thing. + # record alone so the winner of the race can do its thing. + # If crunch-job exited after all of its allocated nodes failed, + # it exits 93, and we want to retry it later (see the + # EXIT_RETRY_UNLOCKED `if` block). # # There is still an unhandled race condition: If our crunch-job # process is about to lose a race with another crunch-job @@ -682,7 +726,7 @@ class Dispatcher # Invalidate the per-job auth token, unless the job is still queued and we # might want to try it again. - if jobrecord.state != "Queued" + if jobrecord.state != "Queued" and !@todo_job_retries.include?(jobrecord.uuid) j_done[:job_auth].update_attributes expires_at: Time.now end @@ -707,6 +751,7 @@ class Dispatcher def run act_as_system_user + User.first.group_permissions $stderr.puts "dispatch: ready" while !$signal[:term] or @running.size > 0 read_pipes @@ -736,6 +781,14 @@ class Dispatcher select(@running.values.collect { |j| [j[:stdout], j[:stderr]] }.flatten, [], [], 1) end + # If there are jobs we wanted to retry, we have to mark them as failed now. + # Other dispatchers can't pick them up because we hold their lock. + @todo_job_retries.each_key do |job_uuid| + job = Job.find_by_uuid(job_uuid) + if job.state == "Running" + fail_job(job, "crunch-dispatch was stopped during job's tempfail retry loop") + end + end end protected