package main
import (
+ "errors"
"flag"
+ "fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/logger"
"git.curoverse.com/arvados.git/sdk/go/util"
"git.curoverse.com/arvados.git/services/datamanager/collection"
"git.curoverse.com/arvados.git/services/datamanager/keep"
+ "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
+ "git.curoverse.com/arvados.git/services/datamanager/summary"
"log"
- "os"
"time"
)
var (
- logEventType string
+ logEventTypePrefix string
logFrequencySeconds int
+ minutesBetweenRuns int
)
func init() {
- flag.StringVar(&logEventType,
- "log-event-type",
- "experimental-data-manager-report",
- "event_type to use in our arvados log entries. Set to empty to turn off logging")
- flag.IntVar(&logFrequencySeconds,
+ flag.StringVar(&logEventTypePrefix,
+ "log-event-type-prefix",
+ "experimental-data-manager",
+ "Prefix to use in the event_type of our arvados log entries. Set to empty to turn off logging")
+ flag.IntVar(&logFrequencySeconds,
"log-frequency-seconds",
20,
"How frequently we'll write log entries in seconds.")
+ flag.IntVar(&minutesBetweenRuns,
+ "minutes-between-runs",
+ 0,
+ "How many minutes we wait betwen data manager runs. 0 means run once and exit.")
}
func main() {
flag.Parse()
-
- arv, err := arvadosclient.MakeArvadosClient()
- if err != nil {
- log.Fatalf("Error setting up arvados client %s", err.Error())
+ if minutesBetweenRuns == 0 {
+ arv, err := makeArvadosClient()
+ if err != nil {
+ log.Fatalf("makeArvadosClient: %v", err)
+ }
+ err = singlerun(arv)
+ if err != nil {
+ log.Fatalf("singlerun: %v", err)
+ }
+ } else {
+ waitTime := time.Minute * time.Duration(minutesBetweenRuns)
+ for {
+ log.Println("Beginning Run")
+ arv, err := makeArvadosClient()
+ if err != nil {
+ log.Fatalf("makeArvadosClient: %v", err)
+ }
+ err = singlerun(arv)
+ if err != nil {
+ log.Printf("singlerun: %v", err)
+ }
+ log.Printf("Sleeping for %d minutes", minutesBetweenRuns)
+ time.Sleep(waitTime)
+ }
}
+}
+
+func makeArvadosClient() (arvadosclient.ArvadosClient, error) {
+ return arvadosclient.MakeArvadosClient()
+}
- if is_admin, err := util.UserIsAdmin(arv); err != nil {
- log.Fatalf("Error querying current arvados user %s", err.Error())
- } else if !is_admin {
- log.Fatalf("Current user is not an admin. Datamanager can only be run by admins.")
+func singlerun(arv arvadosclient.ArvadosClient) error {
+ var err error
+ if isAdmin, err := util.UserIsAdmin(arv); err != nil {
+ return errors.New("Error verifying admin token: " + err.Error())
+ } else if !isAdmin {
+ return errors.New("Current user is not an admin. Datamanager requires a privileged token.")
}
var arvLogger *logger.Logger
- if logEventType != "" {
- arvLogger = logger.NewLogger(logger.LoggerParams{Client: arv,
- EventType: logEventType,
- MinimumWriteInterval: time.Second * time.Duration(logFrequencySeconds)})
+ if logEventTypePrefix != "" {
+ arvLogger = logger.NewLogger(logger.LoggerParams{
+ Client: arv,
+ EventTypePrefix: logEventTypePrefix,
+ WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
}
+ loggerutil.LogRunInfo(arvLogger)
if arvLogger != nil {
- properties, _ := arvLogger.Edit()
- properties["start_time"] = time.Now()
- properties["args"] = os.Args
- hostname, err := os.Hostname()
- if err != nil {
- properties["hostname_error"] = err.Error()
- } else {
- properties["hostname"] = hostname
- }
- arvLogger.Record()
+ arvLogger.AddWriteHook(loggerutil.LogMemoryAlloc)
}
- // TODO(misha): Read Collections and Keep Contents concurrently as goroutines.
- // This requires waiting on them to finish before you let main() exit.
+ var (
+ dataFetcher summary.DataFetcher
+ readCollections collection.ReadCollections
+ keepServerInfo keep.ReadServers
+ )
- RunCollections(collection.GetCollectionsParams{
- Client: arv, Logger: arvLogger, BatchSize: 500})
+ if summary.ShouldReadData() {
+ dataFetcher = summary.ReadData
+ } else {
+ dataFetcher = BuildDataFetcher(arv)
+ }
- RunKeep(keep.GetKeepServersParams{Client: arv, Limit: 1000})
-}
+ dataFetcher(arvLogger, &readCollections, &keepServerInfo)
-func RunCollections(params collection.GetCollectionsParams) {
- readCollections := collection.GetCollections(params)
+ summary.MaybeWriteData(arvLogger, readCollections, keepServerInfo)
- UserUsage := ComputeSizeOfOwnedCollections(readCollections)
- log.Printf("Uuid to Size used: %v", UserUsage)
+ buckets := summary.BucketReplication(readCollections, keepServerInfo)
+ bucketCounts := buckets.Counts()
- // TODO(misha): Add a "readonly" flag. If we're in readonly mode,
- // lots of behaviors can become warnings (and obviously we can't
- // write anything).
- // if !readCollections.ReadAllCollections {
- // log.Fatalf("Did not read all collections")
- // }
+ replicationSummary := buckets.SummarizeBuckets(readCollections)
+ replicationCounts := replicationSummary.ComputeCounts()
- log.Printf("Read and processed %d collections",
- len(readCollections.UuidToCollection))
-}
+ log.Printf("Blocks In Collections: %d, "+
+ "\nBlocks In Keep: %d.",
+ len(readCollections.BlockToDesiredReplication),
+ len(keepServerInfo.BlockToServers))
+ log.Println(replicationCounts.PrettyPrint())
+
+ log.Printf("Blocks Histogram:")
+ for _, rlbss := range bucketCounts {
+ log.Printf("%+v: %10d",
+ rlbss.Levels,
+ rlbss.Count)
+ }
+
+ kc, err := keepclient.MakeKeepClient(&arv)
+ if err != nil {
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error setting up keep client %s", err.Error()))
+ }
+
+ // Log that we're finished. We force the recording, since go will
+ // not wait for the write timer before exiting.
+ if arvLogger != nil {
+ defer arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {
+ summaryInfo := logger.GetOrCreateMap(p, "summary_info")
+ summaryInfo["block_replication_counts"] = bucketCounts
+ summaryInfo["replication_summary"] = replicationCounts
+ p["summary_info"] = summaryInfo
+
+ p["run_info"].(map[string]interface{})["finished_at"] = time.Now()
+ })
+ }
+
+ pullServers := summary.ComputePullServers(kc,
+ &keepServerInfo,
+ readCollections.BlockToDesiredReplication,
+ replicationSummary.UnderReplicatedBlocks)
+
+ pullLists := summary.BuildPullLists(pullServers)
-func RunKeep(params keep.GetKeepServersParams) {
- readServers := keep.GetKeepServers(params)
+ trashLists, trashErr := summary.BuildTrashLists(kc,
+ &keepServerInfo,
+ replicationSummary.KeepBlocksNotInCollections)
- log.Printf("Returned %d keep disks", len(readServers.ServerToContents))
+ summary.WritePullLists(arvLogger, pullLists)
- blockReplicationCounts := make(map[int]int)
- for _, infos := range readServers.BlockToServers {
- replication := len(infos)
- blockReplicationCounts[replication] += 1
+ if trashErr != nil {
+ return err
}
+ keep.SendTrashLists(kc, trashLists)
- log.Printf("Replication level distribution: %v", blockReplicationCounts)
+ return nil
}
-func ComputeSizeOfOwnedCollections(readCollections collection.ReadCollections) (
- results map[string]int) {
- results = make(map[string]int)
- for _, coll := range readCollections.UuidToCollection {
- results[coll.OwnerUuid] = results[coll.OwnerUuid] + coll.TotalSize
+// BuildDataFetcher returns a data fetcher that fetches data from remote servers.
+func BuildDataFetcher(arv arvadosclient.ArvadosClient) summary.DataFetcher {
+ return func(arvLogger *logger.Logger,
+ readCollections *collection.ReadCollections,
+ keepServerInfo *keep.ReadServers) {
+ collectionChannel := make(chan collection.ReadCollections)
+
+ go func() {
+ collectionChannel <- collection.GetCollectionsAndSummarize(
+ collection.GetCollectionsParams{
+ Client: arv,
+ Logger: arvLogger,
+ BatchSize: 50})
+ }()
+
+ *keepServerInfo = keep.GetKeepServersAndSummarize(
+ keep.GetKeepServersParams{
+ Client: arv,
+ Logger: arvLogger,
+ Limit: 1000})
+
+ *readCollections = <-collectionChannel
}
- return
}
projects / arvados.git / blobdiff