X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b49229f98012d7c08ce02b8d28dbcc165c8a6c53..c1fa7d1c6840ce03e763191c92d3548da8494388:/lib/config/load.go diff --git a/lib/config/load.go b/lib/config/load.go index 86a8f7df6d..169b252a0e 100644 --- a/lib/config/load.go +++ b/lib/config/load.go @@ -13,6 +13,7 @@ import ( "io" "io/ioutil" "os" + "regexp" "strings" "git.arvados.org/arvados.git/sdk/go/arvados" @@ -64,14 +65,16 @@ func NewLoader(stdin io.Reader, logger logrus.FieldLogger) *Loader { // // ldr.Path == "/tmp/c.yaml" func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) { flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)") - flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`") - flagset.StringVar(&ldr.KeepWebPath, "legacy-keepweb-config", defaultKeepWebConfigPath, "Legacy keep-web configuration `file`") - flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`") - flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`") - flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`") - flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`") - flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`") - flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files") + if !ldr.SkipLegacy { + flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`") + flagset.StringVar(&ldr.KeepWebPath, "legacy-keepweb-config", defaultKeepWebConfigPath, "Legacy keep-web configuration `file`") + flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`") + flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`") + flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`") + flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`") + flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`") + flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files") + } } // MungeLegacyConfigArgs checks args for a -config flag whose argument @@ -179,6 +182,11 @@ func (ldr *Loader) Load() (*arvados.Config, error) { ldr.configdata = buf } + // FIXME: We should reject YAML if the same key is used twice + // in a map/object, like {foo: bar, foo: baz}. Maybe we'll get + // this fixed free when we upgrade ghodss/yaml to a version + // that uses go-yaml v3. + // Load the config into a dummy map to get the cluster ID // keys, discarding the values; then set up defaults for each // cluster ID; then load the real config on top of the @@ -238,39 +246,58 @@ func (ldr *Loader) Load() (*arvados.Config, error) { return nil, fmt.Errorf("transcoding config data: %s", err) } + var loadFuncs []func(*arvados.Config) error if !ldr.SkipDeprecated { - err = ldr.applyDeprecatedConfig(&cfg) - if err != nil { - return nil, err - } + loadFuncs = append(loadFuncs, + ldr.applyDeprecatedConfig, + ldr.applyDeprecatedVolumeDriverParameters, + ) } if !ldr.SkipLegacy { // legacy file is required when either: // * a non-default location was specified // * no primary config was loaded, and this is the // legacy config file for the current component - for _, err := range []error{ - ldr.loadOldEnvironmentVariables(&cfg), - ldr.loadOldKeepstoreConfig(&cfg), - ldr.loadOldKeepWebConfig(&cfg), - ldr.loadOldCrunchDispatchSlurmConfig(&cfg), - ldr.loadOldWebsocketConfig(&cfg), - ldr.loadOldKeepproxyConfig(&cfg), - ldr.loadOldGitHttpdConfig(&cfg), - ldr.loadOldKeepBalanceConfig(&cfg), - } { - if err != nil { - return nil, err - } + loadFuncs = append(loadFuncs, + ldr.loadOldEnvironmentVariables, + ldr.loadOldKeepstoreConfig, + ldr.loadOldKeepWebConfig, + ldr.loadOldCrunchDispatchSlurmConfig, + ldr.loadOldWebsocketConfig, + ldr.loadOldKeepproxyConfig, + ldr.loadOldGitHttpdConfig, + ldr.loadOldKeepBalanceConfig, + ) + } + for _, f := range loadFuncs { + err = f(&cfg) + if err != nil { + return nil, err } } // Check for known mistakes for id, cc := range cfg.Clusters { + for remote := range cc.RemoteClusters { + if remote == "*" || remote == "SAMPLE" { + continue + } + err = ldr.checkClusterID(fmt.Sprintf("Clusters.%s.RemoteClusters.%s", id, remote), remote, true) + if err != nil { + return nil, err + } + } for _, err = range []error{ + ldr.checkClusterID(fmt.Sprintf("Clusters.%s", id), id, false), + ldr.checkClusterID(fmt.Sprintf("Clusters.%s.Login.LoginCluster", id), cc.Login.LoginCluster, true), + ldr.checkToken(fmt.Sprintf("Clusters.%s.ManagementToken", id), cc.ManagementToken), + ldr.checkToken(fmt.Sprintf("Clusters.%s.SystemRootToken", id), cc.SystemRootToken), + ldr.checkToken(fmt.Sprintf("Clusters.%s.Collections.BlobSigningKey", id), cc.Collections.BlobSigningKey), checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection), ldr.checkEmptyKeepstores(cc), ldr.checkUnlistedKeepstores(cc), + // TODO: check non-empty Rendezvous on + // services other than Keepstore } { if err != nil { return nil, err @@ -280,6 +307,35 @@ func (ldr *Loader) Load() (*arvados.Config, error) { return &cfg, nil } +var acceptableClusterIDRe = regexp.MustCompile(`^[a-z0-9]{5}$`) + +func (ldr *Loader) checkClusterID(label, clusterID string, emptyStringOk bool) error { + if emptyStringOk && clusterID == "" { + return nil + } else if !acceptableClusterIDRe.MatchString(clusterID) { + return fmt.Errorf("%s: cluster ID should be 5 alphanumeric characters", label) + } + return nil +} + +var acceptableTokenRe = regexp.MustCompile(`^[a-zA-Z0-9]+$`) +var acceptableTokenLength = 32 + +func (ldr *Loader) checkToken(label, token string) error { + if token == "" { + if ldr.Logger != nil { + ldr.Logger.Warnf("%s: secret token is not set (use %d+ random characters from a-z, A-Z, 0-9)", label, acceptableTokenLength) + } + } else if !acceptableTokenRe.MatchString(token) { + return fmt.Errorf("%s: unacceptable characters in token (only a-z, A-Z, 0-9 are acceptable)", label) + } else if len(token) < acceptableTokenLength { + if ldr.Logger != nil { + ldr.Logger.Warnf("%s: token is too short (should be at least %d characters)", label, acceptableTokenLength) + } + } + return nil +} + func checkKeyConflict(label string, m map[string]string) error { saw := map[string]bool{} for k := range m { @@ -305,20 +361,33 @@ func (ldr *Loader) logExtraKeys(expected, supplied map[string]interface{}, prefi if ldr.Logger == nil { return } - allowed := map[string]interface{}{} - for k, v := range expected { - allowed[strings.ToLower(k)] = v - } for k, vsupp := range supplied { if k == "SAMPLE" { // entry will be dropped in removeSampleKeys anyway continue } - vexp, ok := allowed[strings.ToLower(k)] + vexp, ok := expected[k] if expected["SAMPLE"] != nil { + // use the SAMPLE entry's keys as the + // "expected" map when checking vsupp + // recursively. vexp = expected["SAMPLE"] } else if !ok { - ldr.Logger.Warnf("deprecated or unknown config entry: %s%s", prefix, k) + // check for a case-insensitive match + hint := "" + for ek := range expected { + if strings.EqualFold(k, ek) { + hint = " (perhaps you meant " + ek + "?)" + // If we don't delete this, it + // will end up getting merged, + // unpredictably + // merging/overriding the + // default. + delete(supplied, k) + break + } + } + ldr.Logger.Warnf("deprecated or unknown config entry: %s%s%s", prefix, k, hint) continue } if vsupp, ok := vsupp.(map[string]interface{}); !ok {