X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/4a72800f68ab43d2ed50895fa4ce83e512b4719b..4d351f0d0c67093440663cee3b52421e3eba7d77:/services/api/script/crunch-dispatch.rb diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb index 6f3aeda413..b1c0e7d316 100755 --- a/services/api/script/crunch-dispatch.rb +++ b/services/api/script/crunch-dispatch.rb @@ -166,6 +166,10 @@ class Dispatcher cmd_args = nil case Server::Application.config.crunch_job_wrapper when :none + if @running.size > 0 + # Don't run more than one at a time. + return + end cmd_args = [] when :slurm_immediate nodelist = nodes_available_for_job(job) @@ -263,6 +267,8 @@ class Dispatcher job_auth: job_auth, stderr_buf_to_flush: '', stderr_flushed_at: 0, + bytes_logged: 0, + events_logged: 0, log_truncated: false } i.close @@ -448,17 +454,13 @@ class Dispatcher protected - def too_many_bytes_logged_for_job(running_job) - bytes_logged = Log.where(uuid: running_job[:uuid]).map { |event| - (event.properties["text"] || "").length - }.sum - return (bytes_logged + j[:stderr_buf_to_flush].size > + def too_many_bytes_logged_for_job(j) + return (j[:bytes_logged] + j[:stderr_buf_to_flush].size > Rails.configuration.crunch_limit_log_event_bytes_per_job) end - def too_many_events_logged_for_job(running_job) - log_count = Log.where(uuid: running_job[:uuid]).count - return (log_count >= Rails.configuration.crunch_limit_log_events_per_job) + def too_many_events_logged_for_job(j) + return (j[:events_logged] >= Rails.configuration.crunch_limit_log_events_per_job) end def did_recently(thing, min_interval) @@ -493,6 +495,8 @@ class Dispatcher owner_uuid: running_job[:job].owner_uuid, properties: {"text" => running_job[:stderr_buf_to_flush]}) log.save! + running_job[:bytes_logged] += running_job[:stderr_buf_to_flush].size + running_job[:events_logged] += 1 running_job[:stderr_buf_to_flush] = '' running_job[:stderr_flushed_at] = Time.now.to_i end