From 970095751e2e836ed296152ae3e9ccb6caa62f62 Mon Sep 17 00:00:00 2001
From: mishaz <misha@curoverse.com>
Date: Thu, 8 Jan 2015 23:17:52 +0000
Subject: [PATCH] Added memory alloc in use to stats exported to log. Also
 added EditHooks to Logger, enabling users to add functions to get called on
 each Edit() call.

---
 sdk/go/logger/logger.go                       | 19 +++++++++++++++++++
 services/datamanager/collection/collection.go |  2 +-
 services/datamanager/datamanager.go           | 14 +++++++++++++-
 3 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index 80cb627a09..fd40cce29d 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -50,6 +50,8 @@ type Logger struct {
 
 	lastWrite   time.Time  // The last time we wrote a log entry
 	modified    bool       // Has this data been modified since the last write
+
+	editHooks   []func(map[string]interface{},map[string]interface{})
 }
 
 // Create a new logger based on the specified parameters.
@@ -75,9 +77,26 @@ func NewLogger(params LoggerParams) *Logger {
 func (l *Logger) Edit() (properties map[string]interface{}, entry map[string]interface{}) {
 	l.lock.Lock()
 	l.modified = true  // We don't actually know the caller will modifiy the data, but we assume they will.
+
+	// Run all our hooks
+	for _, hook := range l.editHooks {
+		hook(l.properties, l.entry)
+	}
+
 	return l.properties, l.entry
 }
 
+// Adds a hook which will be called everytime Edit() is called.
+// The hook takes properties and entry as arguments, in that order.
+// This is useful for stuff like memory profiling.
+// This must be called between Edit() and Record().
+// For convenience AddEditHook will call hook when it is added as well.
+func (l *Logger) AddEditHook(hook func(map[string]interface{},
+	map[string]interface{})) {
+	l.editHooks = append(l.editHooks, hook)
+	hook(l.properties, l.entry)
+}
+
 // Write the log entry you've built up so far. Do not edit the maps
 // returned by Edit() after calling this method.
 // If you have already written within MinimumWriteInterval, then this
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index c64160cac0..212e33f7b3 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -168,7 +168,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
 			properties,_ := params.Logger.Edit()
 			collectionInfo := properties["collection_info"].(map[string]interface{})
 			collectionInfo["collections_read"] = totalCollections
-			collectionInfo["latest_modified_date"] = sdkParams["filters"].([][]string)[0][2]
+			collectionInfo["latest_modified_date_seen"] = sdkParams["filters"].([][]string)[0][2]
 			collectionInfo["total_manifest_size"] = totalManifestSize
 			collectionInfo["max_manifest_size"] = maxManifestSize
 			params.Logger.Record()
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index dc6a431a87..bf989026c4 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -11,6 +11,7 @@ import (
 	"git.curoverse.com/arvados.git/services/datamanager/keep"
 	"log"
 	"os"
+	"runtime"
 	"time"
 )
 
@@ -64,6 +65,9 @@ func main() {
 		}
 		runInfo["pid"] = os.Getpid()
 		properties["run_info"] = runInfo
+
+		arvLogger.AddEditHook(LogMemoryAlloc)
+
 		arvLogger.Record()
 	}
 
@@ -71,7 +75,7 @@ func main() {
 	// This requires waiting on them to finish before you let main() exit.
 
 	RunCollections(collection.GetCollectionsParams{
-		Client: arv, Logger: arvLogger, BatchSize: 500})
+		Client: arv, Logger: arvLogger, BatchSize: 100})
 
 	RunKeep(keep.GetKeepServersParams{Client: arv, Limit: 1000})
 }
@@ -115,3 +119,11 @@ func ComputeSizeOfOwnedCollections(readCollections collection.ReadCollections) (
 	}
 	return
 }
+
+func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
+	_ = entry  // keep the compiler from complaining
+	runInfo := properties["run_info"].(map[string]interface{})
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+	runInfo["alloc_bytes_in_use"] = memStats.Alloc
+}
-- 
2.30.2