# Maximum number of log events that may be generated by a single job.
crunch_limit_log_events_per_job: 65536
- # These two settings control how frequently log events are flushed
- # to the database. If a job generates two or more events within
- # crunch_log_seconds_between_events, the log data is not flushed
- # until crunch_log_bytes_per_event has been reached.
+ # These two settings control how frequently log events are flushed to the
+ # database. Logs lines are buffered until either crunch_log_bytes_per_event
+ # has been reached or crunch_log_seconds_between_events has ellapsed since
+ # the last flush.
crunch_log_bytes_per_event: 4096
crunch_log_seconds_between_events: 1
- # Maximum number of total bytes that may be logged by a single job. Logs
- # that are throttled (see below) are not counted against this total.
- crunch_limit_log_bytes_per_job: 67108864
+ # The sample period for throttling logs, in seconds.
+ crunch_log_throttle_period: 60
- # The sample period for throttling logs, in seconds (see below)
- crunch_log_throttle_period: 30
+ # Maximum number of bytes that job can log over crunch_log_throttle_period
+ # before being silenced until the end of the period.
+ crunch_log_throttle_bytes: 65536
- # Maximum number of bytes that job can log over
- # crunch_limit_log_event_throttle_period before being silenced
- crunch_log_throttle_bytes: 60000
+ # Maximum number of lines that job can log over crunch_log_throttle_period
+ # before being silenced until the end of the period.
+ crunch_log_throttle_lines: 1024
- # Maximum number of lines that job can log over
- # crunch_limit_log_event_throttle_period before being silenced
- crunch_log_throttle_lines: 1000
+ # Maximum bytes that may be logged by a single job. Log bytes that are
+ # silenced by throttling are not counted against this total.
+ crunch_limit_log_bytes_per_job: 67108864
# Path to /etc/dnsmasq.d, or false = do not update dnsmasq data.
dnsmasq_conf_dir: false
# Replace log line with a message about skipping the log
remaining_time = throttle_period - (now - running_job[:log_throttle_timestamp])
if running_job[:log_throttle_bytes_so_far] > Rails.configuration.crunch_log_throttle_bytes
- line.replace "Exceeded crunch_log_throttle_bytes rate of #{Rails.configuration.crunch_log_throttle_bytes} bytes per #{throttle_period} seconds, logging will be silenced for the next #{remaining_time.round} seconds\n"
+ line.replace "Exceeded rate #{Rails.configuration.crunch_log_throttle_bytes} bytes per #{throttle_period} seconds (crunch_log_throttle_bytes), logging will be silenced for the next #{remaining_time.round} seconds\n"
else
- line.replace "Exceeded crunch_log_throttle_lines rate of #{Rails.configuration.crunch_log_throttle_lines} lines per #{throttle_period} seconds, logging will be silenced for the next #{remaining_time.round} seconds\n"
+ line.replace "Exceeded rate #{Rails.configuration.crunch_log_throttle_lines} lines per #{throttle_period} seconds (crunch_log_throttle_lines), logging will be silenced for the next #{remaining_time.round} seconds\n"
end
end
if running_job[:bytes_logged] > Rails.configuration.crunch_limit_log_bytes_per_job
# Replace log line with a message about truncating the log
- line.replace "Exceed hard job log cap crunch_limit_log_bytes_per_job of #{Rails.configuration.crunch_limit_log_bytes_per_job}. Subsequent logs will be truncated."
+ line.replace "Exceeded log limit #{Rails.configuration.crunch_limit_log_bytes_per_job} bytes (crunch_limit_log_bytes_per_job). Log will be truncated."
end
true
# Just reached crunch_limit_log_events_per_job so replace log with notification.
if running_job[:events_logged] == Rails.configuration.crunch_limit_log_events_per_job
running_job[:stderr_buf_to_flush] =
- "Server configured limit reached (crunch_limit_log_events_per_job: #{Rails.configuration.crunch_limit_log_events_per_job}). Subsequent logs truncated"
+ "Exceeded live log limit #{Rails.configuration.crunch_limit_log_events_per_job} events (crunch_limit_log_events_per_job). Live log will be truncated."
end
log = Log.new(object_uuid: running_job[:job].uuid,
event_type: 'stderr',