X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/689901263ebfdd996da3711236615038e6245db3..a5fef23f2863cd0183ff596f4579110e2ddb3b3d:/lib/config/load.go diff --git a/lib/config/load.go b/lib/config/load.go index 526a050fbb..be6181bbe9 100644 --- a/lib/config/load.go +++ b/lib/config/load.go @@ -8,39 +8,178 @@ import ( "bytes" "encoding/json" "errors" + "flag" "fmt" "io" "io/ioutil" "os" "strings" - "git.curoverse.com/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvados" "github.com/ghodss/yaml" "github.com/imdario/mergo" + "github.com/sirupsen/logrus" ) -type logger interface { - Warnf(string, ...interface{}) +var ErrNoClustersDefined = errors.New("config does not define any clusters") + +type Loader struct { + Stdin io.Reader + Logger logrus.FieldLogger + SkipDeprecated bool // Don't load deprecated config keys + SkipLegacy bool // Don't load legacy config files + SkipAPICalls bool // Don't do checks that call RailsAPI/controller + + Path string + KeepstorePath string + KeepWebPath string + CrunchDispatchSlurmPath string + WebsocketPath string + KeepproxyPath string + GitHttpdPath string + KeepBalancePath string + + configdata []byte } -func LoadFile(path string, log logger) (*arvados.Config, error) { - f, err := os.Open(path) - if err != nil { - return nil, err +// NewLoader returns a new Loader with Stdin and Logger set to the +// given values, and all config paths set to their default values. +func NewLoader(stdin io.Reader, logger logrus.FieldLogger) *Loader { + ldr := &Loader{Stdin: stdin, Logger: logger} + // Calling SetupFlags on a throwaway FlagSet has the side + // effect of assigning default values to the configurable + // fields. + ldr.SetupFlags(flag.NewFlagSet("", flag.ContinueOnError)) + return ldr +} + +// SetupFlags configures a flagset so arguments like -config X can be +// used to change the loader's Path fields. +// +// ldr := NewLoader(os.Stdin, logrus.New()) +// flagset := flag.NewFlagSet("", flag.ContinueOnError) +// ldr.SetupFlags(flagset) +// // ldr.Path == "/etc/arvados/config.yml" +// flagset.Parse([]string{"-config", "/tmp/c.yaml"}) +// // ldr.Path == "/tmp/c.yaml" +func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) { + flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)") + if !ldr.SkipLegacy { + flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`") + flagset.StringVar(&ldr.KeepWebPath, "legacy-keepweb-config", defaultKeepWebConfigPath, "Legacy keep-web configuration `file`") + flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`") + flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`") + flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`") + flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`") + flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`") + flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files") } - defer f.Close() - return Load(f, log) } -func Load(rdr io.Reader, log logger) (*arvados.Config, error) { - return load(rdr, log, true) +// MungeLegacyConfigArgs checks args for a -config flag whose argument +// is a regular file (or a symlink to one), but doesn't have a +// top-level "Clusters" key and therefore isn't a valid cluster +// configuration file. If it finds such a flag, it replaces -config +// with legacyConfigArg (e.g., "-legacy-keepstore-config"). +// +// This is used by programs that still need to accept "-config" as a +// way to specify a per-component config file until their config has +// been migrated. +// +// If any errors are encountered while reading or parsing a config +// file, the given args are not munged. We presume the same errors +// will be encountered again and reported later on when trying to load +// cluster configuration from the same file, regardless of which +// struct we end up using. +func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string, legacyConfigArg string) []string { + munged := append([]string(nil), args...) + for i := 0; i < len(args); i++ { + if !strings.HasPrefix(args[i], "-") || strings.SplitN(strings.TrimPrefix(args[i], "-"), "=", 2)[0] != "config" { + continue + } + var operand string + if strings.Contains(args[i], "=") { + operand = strings.SplitN(args[i], "=", 2)[1] + } else if i+1 < len(args) && !strings.HasPrefix(args[i+1], "-") { + i++ + operand = args[i] + } else { + continue + } + if fi, err := os.Stat(operand); err != nil || !fi.Mode().IsRegular() { + continue + } + f, err := os.Open(operand) + if err != nil { + continue + } + defer f.Close() + buf, err := ioutil.ReadAll(f) + if err != nil { + continue + } + var cfg arvados.Config + err = yaml.Unmarshal(buf, &cfg) + if err != nil { + continue + } + if len(cfg.Clusters) == 0 { + lgr.Warnf("%s is not a cluster config file -- interpreting %s as %s (please migrate your config!)", operand, "-config", legacyConfigArg) + if operand == args[i] { + munged[i-1] = legacyConfigArg + } else { + munged[i] = legacyConfigArg + "=" + operand + } + } + } + + // Disable legacy config loading for components other than the + // one that was specified + if legacyConfigArg != "-legacy-keepstore-config" { + ldr.KeepstorePath = "" + } + if legacyConfigArg != "-legacy-crunch-dispatch-slurm-config" { + ldr.CrunchDispatchSlurmPath = "" + } + if legacyConfigArg != "-legacy-ws-config" { + ldr.WebsocketPath = "" + } + if legacyConfigArg != "-legacy-keepweb-config" { + ldr.KeepWebPath = "" + } + if legacyConfigArg != "-legacy-keepproxy-config" { + ldr.KeepproxyPath = "" + } + if legacyConfigArg != "-legacy-git-httpd-config" { + ldr.GitHttpdPath = "" + } + if legacyConfigArg != "-legacy-keepbalance-config" { + ldr.KeepBalancePath = "" + } + + return munged } -func load(rdr io.Reader, log logger, useDeprecated bool) (*arvados.Config, error) { - buf, err := ioutil.ReadAll(rdr) +func (ldr *Loader) loadBytes(path string) ([]byte, error) { + if path == "-" { + return ioutil.ReadAll(ldr.Stdin) + } + f, err := os.Open(path) if err != nil { return nil, err } + defer f.Close() + return ioutil.ReadAll(f) +} + +func (ldr *Loader) Load() (*arvados.Config, error) { + if ldr.configdata == nil { + buf, err := ldr.loadBytes(ldr.Path) + if err != nil { + return nil, err + } + ldr.configdata = buf + } // Load the config into a dummy map to get the cluster ID // keys, discarding the values; then set up defaults for each @@ -49,12 +188,12 @@ func load(rdr io.Reader, log logger, useDeprecated bool) (*arvados.Config, error var dummy struct { Clusters map[string]struct{} } - err = yaml.Unmarshal(buf, &dummy) + err := yaml.Unmarshal(ldr.configdata, &dummy) if err != nil { return nil, err } if len(dummy.Clusters) == 0 { - return nil, errors.New("config does not define any clusters") + return nil, ErrNoClustersDefined } // We can't merge deep structs here; instead, we unmarshal the @@ -74,11 +213,12 @@ func load(rdr io.Reader, log logger, useDeprecated bool) (*arvados.Config, error } } var src map[string]interface{} - err = yaml.Unmarshal(buf, &src) + err = yaml.Unmarshal(ldr.configdata, &src) if err != nil { return nil, fmt.Errorf("loading config data: %s", err) } - logExtraKeys(log, merged, src, "") + ldr.logExtraKeys(merged, src, "") + removeSampleKeys(merged) err = mergo.Merge(&merged, src, mergo.WithOverride) if err != nil { return nil, fmt.Errorf("merging config data: %s", err) @@ -100,18 +240,43 @@ func load(rdr io.Reader, log logger, useDeprecated bool) (*arvados.Config, error return nil, fmt.Errorf("transcoding config data: %s", err) } - if useDeprecated { - err = applyDeprecatedConfig(&cfg, buf, log) + if !ldr.SkipDeprecated { + err = ldr.applyDeprecatedConfig(&cfg) if err != nil { return nil, err } } + if !ldr.SkipLegacy { + // legacy file is required when either: + // * a non-default location was specified + // * no primary config was loaded, and this is the + // legacy config file for the current component + for _, err := range []error{ + ldr.loadOldEnvironmentVariables(&cfg), + ldr.loadOldKeepstoreConfig(&cfg), + ldr.loadOldKeepWebConfig(&cfg), + ldr.loadOldCrunchDispatchSlurmConfig(&cfg), + ldr.loadOldWebsocketConfig(&cfg), + ldr.loadOldKeepproxyConfig(&cfg), + ldr.loadOldGitHttpdConfig(&cfg), + ldr.loadOldKeepBalanceConfig(&cfg), + } { + if err != nil { + return nil, err + } + } + } // Check for known mistakes for id, cc := range cfg.Clusters { - err = checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection) - if err != nil { - return nil, err + for _, err = range []error{ + checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection), + ldr.checkEmptyKeepstores(cc), + ldr.checkUnlistedKeepstores(cc), + } { + if err != nil { + return nil, err + } } } return &cfg, nil @@ -129,21 +294,43 @@ func checkKeyConflict(label string, m map[string]string) error { return nil } -func logExtraKeys(log logger, expected, supplied map[string]interface{}, prefix string) { - if log == nil { +func removeSampleKeys(m map[string]interface{}) { + delete(m, "SAMPLE") + for _, v := range m { + if v, _ := v.(map[string]interface{}); v != nil { + removeSampleKeys(v) + } + } +} + +func (ldr *Loader) logExtraKeys(expected, supplied map[string]interface{}, prefix string) { + if ldr.Logger == nil { return } + allowed := map[string]interface{}{} + for k, v := range expected { + allowed[strings.ToLower(k)] = v + } for k, vsupp := range supplied { - if vexp, ok := expected[k]; !ok { - log.Warnf("deprecated or unknown config entry: %s%s", prefix, k) - } else if vsupp, ok := vsupp.(map[string]interface{}); !ok { + if k == "SAMPLE" { + // entry will be dropped in removeSampleKeys anyway + continue + } + vexp, ok := allowed[strings.ToLower(k)] + if expected["SAMPLE"] != nil { + vexp = expected["SAMPLE"] + } else if !ok { + ldr.Logger.Warnf("deprecated or unknown config entry: %s%s", prefix, k) + continue + } + if vsupp, ok := vsupp.(map[string]interface{}); !ok { // if vsupp is a map but vexp isn't map, this // will be caught elsewhere; see TestBadType. continue } else if vexp, ok := vexp.(map[string]interface{}); !ok { - log.Warnf("unexpected object in config entry: %s%s", prefix, k) + ldr.Logger.Warnf("unexpected object in config entry: %s%s", prefix, k) } else { - logExtraKeys(log, vexp, vsupp, prefix+k+".") + ldr.logExtraKeys(vexp, vsupp, prefix+k+".") } } }