services/datamanager/datamanager.go

   1 /* Keep Datamanager. Responsible for checking on and reporting on Keep Storage */
   2
   3 package main
   4
   5 import (
   6         "flag"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   8         "git.curoverse.com/arvados.git/sdk/go/logger"
   9         "git.curoverse.com/arvados.git/sdk/go/util"
  10         "git.curoverse.com/arvados.git/services/datamanager/collection"
  11         "git.curoverse.com/arvados.git/services/datamanager/keep"
  12         "log"
  13         "os"
  14         "runtime"
  15         "time"
  16 )
  17
  18 var (
  19         logEventType        string
  20         logFrequencySeconds int
  21 )
  22
  23 func init() {
  24         flag.StringVar(&logEventType,
  25                 "log-event-type",
  26                 "experimental-data-manager-report",
  27                 "event_type to use in our arvados log entries. Set to empty to turn off logging")
  28         flag.IntVar(&logFrequencySeconds,
  29                 "log-frequency-seconds",
  30                 20,
  31                 "How frequently we'll write log entries in seconds.")
  32 }
  33
  34 func main() {
  35         flag.Parse()
  36
  37         arv, err := arvadosclient.MakeArvadosClient()
  38         if err != nil {
  39                 log.Fatalf("Error setting up arvados client %s", err.Error())
  40         }
  41
  42         if is_admin, err := util.UserIsAdmin(arv); err != nil {
  43                 log.Fatalf("Error querying current arvados user %s", err.Error())
  44         } else if !is_admin {
  45                 log.Fatalf("Current user is not an admin. Datamanager can only be run by admins.")
  46         }
  47
  48         var arvLogger *logger.Logger
  49         if logEventType != "" {
  50                 arvLogger = logger.NewLogger(logger.LoggerParams{Client: arv,
  51                         EventType:            logEventType,
  52                         MinimumWriteInterval: time.Second * time.Duration(logFrequencySeconds)})
  53         }
  54
  55         if arvLogger != nil {
  56                 arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
  57                         runInfo := make(map[string]interface{})
  58                         runInfo["start_time"] = time.Now()
  59                         runInfo["args"] = os.Args
  60                         hostname, err := os.Hostname()
  61                         if err != nil {
  62                                 runInfo["hostname_error"] = err.Error()
  63                         } else {
  64                                 runInfo["hostname"] = hostname
  65                         }
  66                         runInfo["pid"] = os.Getpid()
  67                         p["run_info"] = runInfo
  68                 })
  69
  70                 arvLogger.AddWriteHook(LogMemoryAlloc)
  71         }
  72
  73         collectionChannel := make(chan collection.ReadCollections)
  74
  75         go func() {
  76                 collectionChannel <- collection.GetCollectionsAndSummarize(
  77                         collection.GetCollectionsParams{
  78                                 Client: arv, Logger: arvLogger, BatchSize: 50})
  79         }()
  80
  81         keepServerInfo := keep.GetKeepServersAndSummarize(
  82                 keep.GetKeepServersParams{Client: arv, Logger: arvLogger, Limit: 1000})
  83
  84         readCollections := <-collectionChannel
  85
  86         // TODO(misha): Use these together to verify replication.
  87         _ = readCollections
  88         _ = keepServerInfo
  89
  90         // Log that we're finished. We force the recording, since go will
  91         // not wait for the timer before exiting.
  92         if arvLogger != nil {
  93                 arvLogger.ForceUpdate(func(p map[string]interface{}, e map[string]interface{}) {
  94                         p["run_info"].(map[string]interface{})["end_time"] = time.Now()
  95                 })
  96         }
  97 }
  98
  99 // TODO(misha): Consider moving this to loggerutil
 100 func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
 101         runInfo := properties["run_info"].(map[string]interface{})
 102         var memStats runtime.MemStats
 103         runtime.ReadMemStats(&memStats)
 104         runInfo["alloc_bytes_in_use"] = memStats.Alloc
 105 }