crunchLogThrottleLines: Rails.application.config.crunch_log_throttle_lines,
crunchLimitLogBytesPerJob: Rails.application.config.crunch_limit_log_bytes_per_job,
crunchLogPartialLineThrottlePeriod: Rails.application.config.crunch_log_partial_line_throttle_period,
+ crunchLogUpdatePeriod: Rails.application.config.crunch_log_update_period,
+ crunchLogUpdateSize: Rails.application.config.crunch_log_update_size,
remoteHosts: Rails.configuration.remote_hosts,
remoteHostsViaDNS: Rails.configuration.remote_hosts_via_dns,
websocketUrl: Rails.application.config.websocket_address,
crunch_log_partial_line_throttle_period: 5
+ # Container logs are written to Keep and saved in a collection,
+ # which is updated periodically while the container runs. This
+ # value sets the interval (given in seconds) between collection
+ # updates.
+ crunch_log_update_period: 1800
+
+ # The log collection is also updated when the specified amount of
+ # log data (given in bytes) is produced in less than one update
+ # period.
+ crunch_log_update_size: 33554432
+
# Attributes to suppress in events and audit logs. Notably,
# specifying ["manifest_text"] here typically makes the database
# smaller and faster.
}
}
-func (runner *ContainerRunner) checkpointLogs() {
- ticker := time.NewTicker(crunchLogCheckpointMaxDuration / 360)
+func (runner *ContainerRunner) updateLogs() {
+ ticker := time.NewTicker(crunchLogUpdatePeriod / 360)
defer ticker.Stop()
- saveAtTime := time.Now().Add(crunchLogCheckpointMaxDuration)
- saveAtSize := crunchLogCheckpointMaxBytes
+ sigusr1 := make(chan os.Signal, 1)
+ signal.Notify(sigusr1, syscall.SIGUSR1)
+ defer signal.Stop(sigusr1)
+
+ saveAtTime := time.Now().Add(crunchLogUpdatePeriod)
+ saveAtSize := crunchLogUpdateSize
var savedSize int64
- for range ticker.C {
+ for {
+ select {
+ case <-ticker.C:
+ case <-sigusr1:
+ saveAtTime = time.Now()
+ }
runner.logMtx.Lock()
done := runner.LogsPDH != nil
runner.logMtx.Unlock()
if size == savedSize || (time.Now().Before(saveAtTime) && size < saveAtSize) {
continue
}
- saveAtTime = time.Now().Add(crunchLogCheckpointMaxDuration)
- saveAtSize = runner.LogCollection.Size() + crunchLogCheckpointMaxBytes
+ saveAtTime = time.Now().Add(crunchLogUpdatePeriod)
+ saveAtSize = runner.LogCollection.Size() + crunchLogUpdateSize
saved, err := runner.saveLogCollection()
if err != nil {
runner.CrunchLog.Printf("error updating log collection: %s", err)
cr.CrunchLog.Immediate = log.New(os.Stderr, containerUUID+" ", 0)
loadLogThrottleParams(api)
- go cr.checkpointLogs()
+ go cr.updateLogs()
return cr, nil
}
var crunchLogPartialLineThrottlePeriod time.Duration = time.Second * 5
var crunchLogBytesPerEvent int64 = 4096
var crunchLogSecondsBetweenEvents = time.Second
-var crunchLogCheckpointMaxDuration = time.Hour / 2
-var crunchLogCheckpointMaxBytes = int64(1 << 25)
+var crunchLogUpdatePeriod = time.Hour / 2
+var crunchLogUpdateSize = int64(1 << 25)
// ArvLogWriter is an io.WriteCloser that processes each write by
// writing it through to another io.WriteCloser (typically a
loadDuration(&crunchLogPartialLineThrottlePeriod, "crunchLogPartialLineThrottlePeriod")
loadInt64(&crunchLogBytesPerEvent, "crunchLogBytesPerEvent")
loadDuration(&crunchLogSecondsBetweenEvents, "crunchLogSecondsBetweenEvents")
- loadInt64(&crunchLogCheckpointMaxBytes, "crunchLogCheckpointMaxBytes")
- loadDuration(&crunchLogCheckpointMaxDuration, "crunchLogCheckpointMaxDuration")
+ loadInt64(&crunchLogUpdateSize, "crunchLogUpdateSize")
+ loadDuration(&crunchLogUpdatePeriod, "crunchLogUpdatePeriod")
}
func (s *LoggingTestSuite) SetUpTest(c *C) {
s.client = arvados.NewClientFromEnv()
- crunchLogCheckpointMaxDuration = time.Hour * 24 * 365
- crunchLogCheckpointMaxBytes = 1 << 50
+ crunchLogUpdatePeriod = time.Hour * 24 * 365
+ crunchLogUpdateSize = 1 << 50
}
func (s *LoggingTestSuite) TestWriteLogs(c *C) {
c.Check(mt, Equals, ". 48f9023dc683a850b1c9b482b14c4b97+163 0:83:crunch-run.txt 83:80:stdout.txt\n")
}
-func (s *LoggingTestSuite) TestLogCheckpoint(c *C) {
+func (s *LoggingTestSuite) TestLogUpdate(c *C) {
for _, trial := range []struct {
maxBytes int64
maxDuration time.Duration
{1000000, time.Millisecond},
} {
c.Logf("max %d bytes, %s", trial.maxBytes, trial.maxDuration)
- crunchLogCheckpointMaxBytes = trial.maxBytes
- crunchLogCheckpointMaxDuration = trial.maxDuration
+ crunchLogUpdateSize = trial.maxBytes
+ crunchLogUpdatePeriod = trial.maxDuration
api := &ArvTestClient{}
kc := &KeepTestClient{}
mt, err := cr.LogCollection.MarshalManifest(".")
c.Check(err, IsNil)
- // Block packing depends on whether there's a
- // checkpoint between the two Goodbyes -- either way
- // the first block will be 4dc76.
+ // Block packing depends on whether there's an update
+ // between the two Goodbyes -- either way the first
+ // block will be 4dc76.
c.Check(mt, Matches, `. 4dc76e0a212bfa30c39d76d8c16da0c0\+1038 (afc503bc1b9a828b4bb543cb629e936c\+78|90699dc22545cd74a0664303f70bc05a\+39 276b49339fd5203d15a93ff3de11bfb9\+39) 0:1077:crunch-run.txt 1077:39:stdout.txt\n`)
}
}
projects / arvados.git / commitdiff