services/datamanager/datamanager.go

   1 /* Keep Datamanager. Responsible for checking on and reporting on Keep Storage */
   2
   3 package main
   4
   5 import (
   6         "flag"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   8         "git.curoverse.com/arvados.git/sdk/go/logger"
   9         "git.curoverse.com/arvados.git/sdk/go/util"
  10         "git.curoverse.com/arvados.git/services/datamanager/collection"
  11         "git.curoverse.com/arvados.git/services/datamanager/keep"
  12         "log"
  13         "os"
  14         "runtime"
  15         "time"
  16 )
  17
  18 var (
  19         logEventType        string
  20         logFrequencySeconds int
  21 )
  22
  23 func init() {
  24         flag.StringVar(&logEventType,
  25                 "log-event-type",
  26                 "experimental-data-manager-report",
  27                 "event_type to use in our arvados log entries. Set to empty to turn off logging")
  28         flag.IntVar(&logFrequencySeconds,
  29                 "log-frequency-seconds",
  30                 20,
  31                 "How frequently we'll write log entries in seconds.")
  32 }
  33
  34 func main() {
  35         flag.Parse()
  36
  37         arv, err := arvadosclient.MakeArvadosClient()
  38         if err != nil {
  39                 log.Fatalf("Error setting up arvados client %s", err.Error())
  40         }
  41
  42         if is_admin, err := util.UserIsAdmin(arv); err != nil {
  43                 log.Fatalf("Error querying current arvados user %s", err.Error())
  44         } else if !is_admin {
  45                 log.Fatalf("Current user is not an admin. Datamanager can only be run by admins.")
  46         }
  47
  48         var arvLogger *logger.Logger
  49         if logEventType != "" {
  50                 arvLogger = logger.NewLogger(logger.LoggerParams{Client: arv,
  51                         EventType:            logEventType,
  52                         MinimumWriteInterval: time.Second * time.Duration(logFrequencySeconds)})
  53         }
  54
  55         if arvLogger != nil {
  56                 properties, _ := arvLogger.Edit()
  57                 runInfo := make(map[string]interface{})
  58                 runInfo["start_time"] = time.Now()
  59                 runInfo["args"] = os.Args
  60                 hostname, err := os.Hostname()
  61                 if err != nil {
  62                         runInfo["hostname_error"] = err.Error()
  63                 } else {
  64                         runInfo["hostname"] = hostname
  65                 }
  66                 runInfo["pid"] = os.Getpid()
  67                 properties["run_info"] = runInfo
  68
  69                 arvLogger.AddWriteHook(LogMemoryAlloc)
  70
  71                 arvLogger.Record()
  72         }
  73
  74         collectionChannel := make(chan collection.ReadCollections)
  75
  76         go func() {
  77                 collectionChannel <- collection.GetCollectionsAndSummarize(
  78                         collection.GetCollectionsParams{
  79                                 Client: arv, Logger: arvLogger, BatchSize: 50})
  80         }()
  81
  82         keepServerInfo := keep.GetKeepServersAndSummarize(
  83                 keep.GetKeepServersParams{Client: arv, Logger: arvLogger, Limit: 1000})
  84
  85         readCollections := <-collectionChannel
  86
  87         // Make compiler happy.
  88         _ = readCollections
  89         _ = keepServerInfo
  90
  91         // Log that we're finished
  92         if arvLogger != nil {
  93                 properties, _ := arvLogger.Edit()
  94                 properties["run_info"].(map[string]interface{})["end_time"] = time.Now()
  95                 // Force the recording, since go will not wait for the timer before exiting.
  96                 arvLogger.ForceRecord()
  97         }
  98 }
  99
 100 func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
 101         _ = entry // keep the compiler from complaining
 102         runInfo := properties["run_info"].(map[string]interface{})
 103         var memStats runtime.MemStats
 104         runtime.ReadMemStats(&memStats)
 105         runInfo["alloc_bytes_in_use"] = memStats.Alloc
 106 }