import (
"encoding/json"
"flag"
- "io/ioutil"
"log"
"os"
"os/signal"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/config"
)
// Config specifies site configuration, like API credentials and the
// How often to check
RunPeriod arvados.Duration
+
+ // Number of collections to request in each API call
+ CollectionBatchSize int
+
+ // Max collections to buffer in memory (bigger values consume
+ // more memory, but can reduce store-and-forward latency when
+ // fetching pages)
+ CollectionBuffers int
}
// RunOptions controls runtime behavior. The flags/options that belong
CommitTrash bool
Logger *log.Logger
Dumper *log.Logger
+
+ // SafeRendezvousState from the most recent balance operation,
+ // or "" if unknown. If this changes from one run to the next,
+ // we need to watch out for races. See
+ // (*Balancer)ClearTrashLists.
+ SafeRendezvousState string
}
var debugf = func(string, ...interface{}) {}
var runOptions RunOptions
configPath := flag.String("config", "",
- "`path` of json configuration file")
+ "`path` of JSON or YAML configuration file")
serviceListPath := flag.String("config.KeepServiceList", "",
- "`path` of json file with list of keep services to balance, as given by \"arv keep_service list\" "+
+ "`path` of JSON or YAML file with list of keep services to balance, as given by \"arv keep_service list\" "+
"(default: config[\"KeepServiceList\"], or if none given, get all available services and filter by config[\"KeepServiceTypes\"])")
flag.BoolVar(&runOptions.Once, "once", false,
"balance once and then exit")
if *configPath == "" {
log.Fatal("You must specify a config file (see `keep-balance -help`)")
}
- mustReadJSON(&config, *configPath)
+ mustReadConfig(&config, *configPath)
if *serviceListPath != "" {
- mustReadJSON(&config.KeepServiceList, *serviceListPath)
+ mustReadConfig(&config.KeepServiceList, *serviceListPath)
}
if *debugFlag {
if err != nil {
// (don't run)
} else if runOptions.Once {
- err = (&Balancer{}).Run(config, runOptions)
+ _, err = (&Balancer{}).Run(config, runOptions)
} else {
err = RunForever(config, runOptions, nil)
}
}
}
-func mustReadJSON(dst interface{}, path string) {
- if buf, err := ioutil.ReadFile(path); err != nil {
- log.Fatalf("Reading %q: %v", path, err)
- } else if err = json.Unmarshal(buf, dst); err != nil {
- log.Fatalf("Decoding %q: %v", path, err)
+func mustReadConfig(dst interface{}, path string) {
+ if err := config.LoadFile(dst, path); err != nil {
+ log.Fatal(err)
}
}
logger.Print("======= Consider using -commit-pulls and -commit-trash flags.")
}
- err := (&Balancer{}).Run(config, runOptions)
+ bal := &Balancer{}
+ var err error
+ runOptions, err = bal.Run(config, runOptions)
if err != nil {
logger.Print("run failed: ", err)
} else {