import (
"flag"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/logger"
"git.curoverse.com/arvados.git/sdk/go/util"
"git.curoverse.com/arvados.git/services/datamanager/collection"
-// "git.curoverse.com/arvados.git/services/datamanager/keep"
+ "git.curoverse.com/arvados.git/services/datamanager/keep"
+ "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
"log"
+ "time"
)
+var (
+ logEventType string
+ logFrequencySeconds int
+)
+
+func init() {
+ flag.StringVar(&logEventType,
+ "log-event-type",
+ "experimental-data-manager-report",
+ "event_type to use in our arvados log entries. Set to empty to turn off logging")
+ flag.IntVar(&logFrequencySeconds,
+ "log-frequency-seconds",
+ 20,
+ "How frequently we'll write log entries in seconds.")
+}
+
func main() {
flag.Parse()
log.Fatalf("Current user is not an admin. Datamanager can only be run by admins.")
}
- // TODO(misha): Read Collections and Keep Contents concurrently as goroutines.
-
- readCollections := collection.GetCollections(
- collection.GetCollectionsParams{
- Client: arv, BatchSize: 500})
+ var arvLogger *logger.Logger
+ if logEventType != "" {
+ arvLogger = logger.NewLogger(logger.LoggerParams{Client: arv,
+ EventType: logEventType,
+ WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
+ }
- //log.Printf("Read Collections: %v", readCollections)
+ loggerutil.LogRunInfo(arvLogger)
+ if arvLogger != nil {
+ arvLogger.AddWriteHook(loggerutil.LogMemoryAlloc)
+ }
- UserUsage := ComputeSizeOfOwnedCollections(readCollections)
- log.Printf("Uuid to Size used: %v", UserUsage)
+ collectionChannel := make(chan collection.ReadCollections)
- // TODO(misha): Add a "readonly" flag. If we're in readonly mode,
- // lots of behaviors can become warnings (and obviously we can't
- // write anything).
- // if !readCollections.ReadAllCollections {
- // log.Fatalf("Did not read all collections")
- // }
+ go func() {
+ collectionChannel <- collection.GetCollectionsAndSummarize(
+ collection.GetCollectionsParams{
+ Client: arv, Logger: arvLogger, BatchSize: 50})
+ }()
- log.Printf("Read and processed %d collections",
- len(readCollections.UuidToCollection))
+ keepServerInfo := keep.GetKeepServersAndSummarize(
+ keep.GetKeepServersParams{Client: arv, Logger: arvLogger, Limit: 1000})
- // readServers := keep.GetKeepServers(
- // keep.GetKeepServersParams{Client: arv, Limit: 1000})
+ readCollections := <-collectionChannel
- // log.Printf("Returned %d keep disks", len(readServers.AddressToContents))
-}
+ // TODO(misha): Use these together to verify replication.
+ _ = readCollections
+ _ = keepServerInfo
-func ComputeSizeOfOwnedCollections(readCollections collection.ReadCollections) (
- results map[string]int) {
- results = make(map[string]int)
- for _, coll := range readCollections.UuidToCollection {
- results[coll.OwnerUuid] = results[coll.OwnerUuid] + coll.TotalSize
+ // Log that we're finished. We force the recording, since go will
+ // not wait for the timer before exiting.
+ if arvLogger != nil {
+ arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {
+ p["run_info"].(map[string]interface{})["time_finished"] = time.Now()
+ })
}
- return
}