Added memory alloc in use to stats exported to log. Also added EditHooks to Logger...

diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index 80cb627a09e6ebfe0ecd68284268558262e94492..fd40cce29db31923745f1be0c53d33bddc44be26 100644 (file)
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -50,6 +50,8 @@ type Logger struct {
 
        lastWrite   time.Time  // The last time we wrote a log entry
        modified    bool       // Has this data been modified since the last write
+
+       editHooks   []func(map[string]interface{},map[string]interface{})
 }
 
 // Create a new logger based on the specified parameters.
@@ -75,9 +77,26 @@ func NewLogger(params LoggerParams) *Logger {
 func (l *Logger) Edit() (properties map[string]interface{}, entry map[string]interface{}) {
        l.lock.Lock()
        l.modified = true  // We don't actually know the caller will modifiy the data, but we assume they will.
+
+       // Run all our hooks
+       for _, hook := range l.editHooks {
+               hook(l.properties, l.entry)
+       }
+
        return l.properties, l.entry
 }
 
+// Adds a hook which will be called everytime Edit() is called.
+// The hook takes properties and entry as arguments, in that order.
+// This is useful for stuff like memory profiling.
+// This must be called between Edit() and Record().
+// For convenience AddEditHook will call hook when it is added as well.
+func (l *Logger) AddEditHook(hook func(map[string]interface{},
+       map[string]interface{})) {
+       l.editHooks = append(l.editHooks, hook)
+       hook(l.properties, l.entry)
+}
+
 // Write the log entry you've built up so far. Do not edit the maps
 // returned by Edit() after calling this method.
 // If you have already written within MinimumWriteInterval, then this

diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index c64160cac0c9fe96dca349c5997c156210cabf4d..212e33f7b370c6bd12593f445b74c0232c0ec506 100644 (file)
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -168,7 +168,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
                        properties,_ := params.Logger.Edit()
                        collectionInfo := properties["collection_info"].(map[string]interface{})
                        collectionInfo["collections_read"] = totalCollections
-                       collectionInfo["latest_modified_date"] = sdkParams["filters"].([][]string)[0][2]
+                       collectionInfo["latest_modified_date_seen"] = sdkParams["filters"].([][]string)[0][2]
                        collectionInfo["total_manifest_size"] = totalManifestSize
                        collectionInfo["max_manifest_size"] = maxManifestSize
                        params.Logger.Record()

diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index dc6a431a87175008b8ba41b62480005912c59e5c..bf989026c45189db29a7a1ca10c8c785c7a383ac 100644 (file)
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -11,6 +11,7 @@ import (
        "git.curoverse.com/arvados.git/services/datamanager/keep"
        "log"
        "os"
+       "runtime"
        "time"
 )
 
@@ -64,6 +65,9 @@ func main() {
                }
                runInfo["pid"] = os.Getpid()
                properties["run_info"] = runInfo
+
+               arvLogger.AddEditHook(LogMemoryAlloc)
+
                arvLogger.Record()
        }
 
@@ -71,7 +75,7 @@ func main() {
        // This requires waiting on them to finish before you let main() exit.
 
        RunCollections(collection.GetCollectionsParams{
-               Client: arv, Logger: arvLogger, BatchSize: 500})
+               Client: arv, Logger: arvLogger, BatchSize: 100})
 
        RunKeep(keep.GetKeepServersParams{Client: arv, Limit: 1000})
 }
@@ -115,3 +119,11 @@ func ComputeSizeOfOwnedCollections(readCollections collection.ReadCollections) (
        }
        return
 }
+
+func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
+       _ = entry  // keep the compiler from complaining
+       runInfo := properties["run_info"].(map[string]interface{})
+       var memStats runtime.MemStats
+       runtime.ReadMemStats(&memStats)
+       runInfo["alloc_bytes_in_use"] = memStats.Alloc
+}