X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/4b5ec8651ba83f2a79fe40708021e03c86275093..ae92d144610446849eb568247a44f02ae985c281:/lib/config/load.go diff --git a/lib/config/load.go b/lib/config/load.go index 15174ae9cf..6099215edc 100644 --- a/lib/config/load.go +++ b/lib/config/load.go @@ -6,6 +6,7 @@ package config import ( "bytes" + _ "embed" "encoding/json" "errors" "flag" @@ -14,6 +15,7 @@ import ( "io/ioutil" "os" "regexp" + "strconv" "strings" "git.arvados.org/arvados.git/sdk/go/arvados" @@ -22,6 +24,9 @@ import ( "github.com/sirupsen/logrus" ) +//go:embed config.default.yml +var DefaultYAML []byte + var ErrNoClustersDefined = errors.New("config does not define any clusters") type Loader struct { @@ -71,7 +76,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) { flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`") flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`") flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`") - flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`") + flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arvados-git-httpd configuration `file`") flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`") flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files") } @@ -182,6 +187,11 @@ func (ldr *Loader) Load() (*arvados.Config, error) { ldr.configdata = buf } + // FIXME: We should reject YAML if the same key is used twice + // in a map/object, like {foo: bar, foo: baz}. Maybe we'll get + // this fixed free when we upgrade ghodss/yaml to a version + // that uses go-yaml v3. + // Load the config into a dummy map to get the cluster ID // keys, discarding the values; then set up defaults for each // cluster ID; then load the real config on top of the @@ -220,6 +230,9 @@ func (ldr *Loader) Load() (*arvados.Config, error) { } ldr.logExtraKeys(merged, src, "") removeSampleKeys(merged) + // We merge the loaded config into the default, overriding any existing keys. + // Make sure we do not override a default with a key that has a 'null' value. + removeNullKeys(src) err = mergo.Merge(&merged, src, mergo.WithOverride) if err != nil { return nil, fmt.Errorf("merging config data: %s", err) @@ -264,6 +277,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) { ldr.loadOldKeepBalanceConfig, ) } + loadFuncs = append(loadFuncs, ldr.setImplicitStorageClasses) for _, f := range loadFuncs { err = f(&cfg) if err != nil { @@ -271,6 +285,19 @@ func (ldr *Loader) Load() (*arvados.Config, error) { } } + // Preprocess/automate some configs + for id, cc := range cfg.Clusters { + ldr.autofillPreemptible("Clusters."+id, &cc) + + if strings.Count(cc.Users.AnonymousUserToken, "/") == 3 { + // V2 token, strip it to just a secret + tmp := strings.Split(cc.Users.AnonymousUserToken, "/") + cc.Users.AnonymousUserToken = tmp[2] + } + + cfg.Clusters[id] = cc + } + // Check for known mistakes for id, cc := range cfg.Clusters { for remote := range cc.RemoteClusters { @@ -285,12 +312,18 @@ func (ldr *Loader) Load() (*arvados.Config, error) { for _, err = range []error{ ldr.checkClusterID(fmt.Sprintf("Clusters.%s", id), id, false), ldr.checkClusterID(fmt.Sprintf("Clusters.%s.Login.LoginCluster", id), cc.Login.LoginCluster, true), - ldr.checkToken(fmt.Sprintf("Clusters.%s.ManagementToken", id), cc.ManagementToken), - ldr.checkToken(fmt.Sprintf("Clusters.%s.SystemRootToken", id), cc.SystemRootToken), - ldr.checkToken(fmt.Sprintf("Clusters.%s.Collections.BlobSigningKey", id), cc.Collections.BlobSigningKey), + ldr.checkToken(fmt.Sprintf("Clusters.%s.ManagementToken", id), cc.ManagementToken, true, false), + ldr.checkToken(fmt.Sprintf("Clusters.%s.SystemRootToken", id), cc.SystemRootToken, true, false), + ldr.checkToken(fmt.Sprintf("Clusters.%s.Users.AnonymousUserToken", id), cc.Users.AnonymousUserToken, false, true), + ldr.checkToken(fmt.Sprintf("Clusters.%s.Collections.BlobSigningKey", id), cc.Collections.BlobSigningKey, true, false), checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection), + ldr.checkEnum("Containers.LocalKeepLogsToContainerLog", cc.Containers.LocalKeepLogsToContainerLog, "none", "all", "errors"), ldr.checkEmptyKeepstores(cc), ldr.checkUnlistedKeepstores(cc), + ldr.checkStorageClasses(cc), + ldr.checkCUDAVersions(cc), + // TODO: check non-empty Rendezvous on + // services other than Keepstore } { if err != nil { return nil, err @@ -314,13 +347,34 @@ func (ldr *Loader) checkClusterID(label, clusterID string, emptyStringOk bool) e var acceptableTokenRe = regexp.MustCompile(`^[a-zA-Z0-9]+$`) var acceptableTokenLength = 32 -func (ldr *Loader) checkToken(label, token string) error { - if token == "" { - if ldr.Logger != nil { - ldr.Logger.Warnf("%s: secret token is not set (use %d+ random characters from a-z, A-Z, 0-9)", label, acceptableTokenLength) +func (ldr *Loader) checkToken(label, token string, mandatory bool, acceptV2 bool) error { + if len(token) == 0 { + if !mandatory { + // when a token is not mandatory, the acceptable length and content is only checked if its length is non-zero + return nil + } else { + if ldr.Logger != nil { + ldr.Logger.Warnf("%s: secret token is not set (use %d+ random characters from a-z, A-Z, 0-9)", label, acceptableTokenLength) + } } } else if !acceptableTokenRe.MatchString(token) { - return fmt.Errorf("%s: unacceptable characters in token (only a-z, A-Z, 0-9 are acceptable)", label) + if !acceptV2 { + return fmt.Errorf("%s: unacceptable characters in token (only a-z, A-Z, 0-9 are acceptable)", label) + } + // Test for a proper V2 token + tmp := strings.SplitN(token, "/", 3) + if len(tmp) != 3 { + return fmt.Errorf("%s: unacceptable characters in token (only a-z, A-Z, 0-9 are acceptable)", label) + } + if !strings.HasPrefix(token, "v2/") { + return fmt.Errorf("%s: unacceptable characters in token (only a-z, A-Z, 0-9 are acceptable)", label) + } + if !acceptableTokenRe.MatchString(tmp[2]) { + return fmt.Errorf("%s: unacceptable characters in V2 token secret (only a-z, A-Z, 0-9 are acceptable)", label) + } + if len(tmp[2]) < acceptableTokenLength { + ldr.Logger.Warnf("%s: secret is too short (should be at least %d characters)", label, acceptableTokenLength) + } } else if len(token) < acceptableTokenLength { if ldr.Logger != nil { ldr.Logger.Warnf("%s: token is too short (should be at least %d characters)", label, acceptableTokenLength) @@ -329,6 +383,84 @@ func (ldr *Loader) checkToken(label, token string) error { return nil } +func (ldr *Loader) checkEnum(label, value string, accepted ...string) error { + for _, s := range accepted { + if s == value { + return nil + } + } + return fmt.Errorf("%s: unacceptable value %q: must be one of %q", label, value, accepted) +} + +func (ldr *Loader) setImplicitStorageClasses(cfg *arvados.Config) error { +cluster: + for id, cc := range cfg.Clusters { + if len(cc.StorageClasses) > 0 { + continue cluster + } + for _, vol := range cc.Volumes { + if len(vol.StorageClasses) > 0 { + continue cluster + } + } + // No explicit StorageClasses config info at all; fill + // in implicit defaults. + for id, vol := range cc.Volumes { + vol.StorageClasses = map[string]bool{"default": true} + cc.Volumes[id] = vol + } + cc.StorageClasses = map[string]arvados.StorageClassConfig{"default": {Default: true}} + cfg.Clusters[id] = cc + } + return nil +} + +func (ldr *Loader) checkStorageClasses(cc arvados.Cluster) error { + classOnVolume := map[string]bool{} + for volid, vol := range cc.Volumes { + if len(vol.StorageClasses) == 0 { + return fmt.Errorf("%s: volume has no StorageClasses listed", volid) + } + for classid := range vol.StorageClasses { + if _, ok := cc.StorageClasses[classid]; !ok { + return fmt.Errorf("%s: volume refers to storage class %q that is not defined in StorageClasses", volid, classid) + } + classOnVolume[classid] = true + } + } + haveDefault := false + for classid, sc := range cc.StorageClasses { + if !classOnVolume[classid] && len(cc.Volumes) > 0 { + ldr.Logger.Warnf("there are no volumes providing storage class %q", classid) + } + if sc.Default { + haveDefault = true + } + } + if !haveDefault { + return fmt.Errorf("there is no default storage class (at least one entry in StorageClasses must have Default: true)") + } + return nil +} + +func (ldr *Loader) checkCUDAVersions(cc arvados.Cluster) error { + for _, it := range cc.InstanceTypes { + if it.CUDA.DeviceCount == 0 { + continue + } + + _, err := strconv.ParseFloat(it.CUDA.DriverVersion, 64) + if err != nil { + return fmt.Errorf("InstanceType %q has invalid CUDA.DriverVersion %q, expected format X.Y (%v)", it.Name, it.CUDA.DriverVersion, err) + } + _, err = strconv.ParseFloat(it.CUDA.HardwareCapability, 64) + if err != nil { + return fmt.Errorf("InstanceType %q has invalid CUDA.HardwareCapability %q, expected format X.Y (%v)", it.Name, it.CUDA.HardwareCapability, err) + } + } + return nil +} + func checkKeyConflict(label string, m map[string]string) error { saw := map[string]bool{} for k := range m { @@ -341,6 +473,17 @@ func checkKeyConflict(label string, m map[string]string) error { return nil } +func removeNullKeys(m map[string]interface{}) { + for k, v := range m { + if v == nil { + delete(m, k) + } + if v, _ := v.(map[string]interface{}); v != nil { + removeNullKeys(v) + } + } +} + func removeSampleKeys(m map[string]interface{}) { delete(m, "SAMPLE") for _, v := range m { @@ -394,3 +537,21 @@ func (ldr *Loader) logExtraKeys(expected, supplied map[string]interface{}, prefi } } } + +func (ldr *Loader) autofillPreemptible(label string, cc *arvados.Cluster) { + if factor := cc.Containers.PreemptiblePriceFactor; factor > 0 { + for name, it := range cc.InstanceTypes { + if !it.Preemptible { + it.Preemptible = true + it.Price = it.Price * factor + it.Name = name + ".preemptible" + if it2, exists := cc.InstanceTypes[it.Name]; exists && it2 != it { + ldr.Logger.Warnf("%s.InstanceTypes[%s]: already exists, so not automatically adding a preemptible variant of %s", label, it.Name, name) + continue + } + cc.InstanceTypes[it.Name] = it + } + } + } + +}