Added different event types for started, partially complete and final log entries.

diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index e6e1ed64df3a17988c236231147c961c9282c261..e7bc8d14c331481b52efab5b04f7bc940d4c8010 100644 (file)
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -1,9 +1,8 @@
 // Logger periodically writes a log to the Arvados SDK.
 //
 // This package is useful for maintaining a log object that is updated
-// over time. Every time the object is updated, it will be written to
-// the log. Writes will be throttled to no more frequent than
-// WriteInterval.
+// over time. This log object will be periodically written to the log,
+// as specified by WriteInterval in the Params.
 //
 // This package is safe for concurrent use as long as:
 // The maps passed to a LogMutator are not accessed outside of the
@@ -26,10 +25,16 @@ import (
        "time"
 )
 
+const (
+       startSuffix   = "-start"
+       partialSuffix = "-partial"
+       finalSuffix   = "-final"
+)
+
 type LoggerParams struct {
-       Client        arvadosclient.ArvadosClient // The client we use to write log entries
-       EventType     string                      // The event type to assign to the log entry.
-       WriteInterval time.Duration               // Wait at least this long between log writes
+       Client          arvadosclient.ArvadosClient // The client we use to write log entries
+       EventTypePrefix string                      // The prefix we use for the event type in the log entry
+       WriteInterval   time.Duration               // Wait at least this long between log writes
 }
 
 // A LogMutator is a function which modifies the log entry.
@@ -57,6 +62,7 @@ type Logger struct {
        modified    bool            // Has this data been modified since the last write?
        workToDo    chan LogMutator // Work to do in the worker thread.
        writeTicker *time.Ticker    // On each tick we write the log data to arvados, if it has been modified.
+       hasWritten  bool            // Whether we've written at all yet.
 
        writeHooks []LogMutator // Mutators we call before each write.
 }
@@ -67,8 +73,8 @@ func NewLogger(params LoggerParams) *Logger {
        if &params.Client == nil {
                log.Fatal("Nil arvados client in LoggerParams passed in to NewLogger()")
        }
-       if params.EventType == "" {
-               log.Fatal("Empty event type in LoggerParams passed in to NewLogger()")
+       if params.EventTypePrefix == "" {
+               log.Fatal("Empty event type prefix in LoggerParams passed in to NewLogger()")
        }
 
        l := &Logger{data: make(map[string]interface{}),
@@ -119,11 +125,9 @@ func (l *Logger) FinalUpdate(mutator LogMutator) {
        // Apply the final update
        l.workToDo <- mutator
 
-       // Perform the write and signal that we can return.
+       // Perform the final write and signal that we can return.
        l.workToDo <- func(p map[string]interface{}, e map[string]interface{}) {
-               // TODO(misha): Add a boolean arg to write() to indicate that it's
-               // final so that we can set the appropriate event type.
-               l.write()
+               l.write(true)
                done <- true
        }
 
@@ -146,7 +150,7 @@ func (l *Logger) work() {
                select {
                case <-l.writeTicker.C:
                        if l.modified {
-                               l.write()
+                               l.write(false)
                                l.modified = false
                        }
                case mutator := <-l.workToDo:
@@ -157,16 +161,22 @@ func (l *Logger) work() {
 }
 
 // Actually writes the log entry.
-func (l *Logger) write() {
+func (l *Logger) write(isFinal bool) {
 
        // Run all our hooks
        for _, hook := range l.writeHooks {
                hook(l.properties, l.entry)
        }
 
-       // Update the event type in case it was modified or is missing.
-       // TODO(misha): Fix this to write different event types.
-       l.entry["event_type"] = l.params.EventType
+       // Update the event type.
+       if isFinal {
+               l.entry["event_type"] = l.params.EventTypePrefix + finalSuffix
+       } else if l.hasWritten {
+               l.entry["event_type"] = l.params.EventTypePrefix + partialSuffix
+       } else {
+               l.entry["event_type"] = l.params.EventTypePrefix + startSuffix
+       }
+       l.hasWritten = true
 
        // Write the log entry.
        // This is a network write and will take a while, which is bad

diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index bd68db112ad26f4eedfe61c6cf5657d48b33cb24..2c5c36d44e275ec5d324dbd8808a61698e7de297 100644 (file)
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -15,15 +15,15 @@ import (
 )
 
 var (
-       logEventType        string
+       logEventTypePrefix        string
        logFrequencySeconds int
 )
 
 func init() {
-       flag.StringVar(&logEventType,
-               "log-event-type",
-               "experimental-data-manager-report",
-               "event_type to use in our arvados log entries. Set to empty to turn off logging")
+       flag.StringVar(&logEventTypePrefix,
+               "log-event-type-prefix",
+               "experimental-data-manager",
+               "Prefix to use in the event_type of our arvados log entries. Set to empty to turn off logging")
        flag.IntVar(&logFrequencySeconds,
                "log-frequency-seconds",
                20,
@@ -45,9 +45,9 @@ func main() {
        }
 
        var arvLogger *logger.Logger
-       if logEventType != "" {
+       if logEventTypePrefix != "" {
                arvLogger = logger.NewLogger(logger.LoggerParams{Client: arv,
-                       EventType:     logEventType,
+                       EventTypePrefix:     logEventTypePrefix,
                        WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
        }