17417: Merge branch 'main' into 17417-add-arm64

[arvados.git] / lib / config / load.go

diff --git a/lib/config/load.go b/lib/config/load.go
index 73f0a244572279a75eabd76f1b151d954b05bfe8..956a47b1a4ac2ef992958739d5189eaf5e519ed5 100644 (file)
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -182,6 +182,11 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                ldr.configdata = buf
        }
 
+       // FIXME: We should reject YAML if the same key is used twice
+       // in a map/object, like {foo: bar, foo: baz}. Maybe we'll get
+       // this fixed free when we upgrade ghodss/yaml to a version
+       // that uses go-yaml v3.
+
        // Load the config into a dummy map to get the cluster ID
        // keys, discarding the values; then set up defaults for each
        // cluster ID; then load the real config on top of the
@@ -264,6 +269,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                        ldr.loadOldKeepBalanceConfig,
                )
        }
+       loadFuncs = append(loadFuncs, ldr.setImplicitStorageClasses)
        for _, f := range loadFuncs {
                err = f(&cfg)
                if err != nil {
@@ -289,8 +295,12 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                        ldr.checkToken(fmt.Sprintf("Clusters.%s.SystemRootToken", id), cc.SystemRootToken),
                        ldr.checkToken(fmt.Sprintf("Clusters.%s.Collections.BlobSigningKey", id), cc.Collections.BlobSigningKey),
                        checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection),
+                       ldr.checkEnum("Containers.LocalKeepLogsToContainerLog", cc.Containers.LocalKeepLogsToContainerLog, "none", "all", "errors"),
                        ldr.checkEmptyKeepstores(cc),
                        ldr.checkUnlistedKeepstores(cc),
+                       ldr.checkStorageClasses(cc),
+                       // TODO: check non-empty Rendezvous on
+                       // services other than Keepstore
                } {
                        if err != nil {
                                return nil, err
@@ -329,6 +339,66 @@ func (ldr *Loader) checkToken(label, token string) error {
        return nil
 }
 
+func (ldr *Loader) checkEnum(label, value string, accepted ...string) error {
+       for _, s := range accepted {
+               if s == value {
+                       return nil
+               }
+       }
+       return fmt.Errorf("%s: unacceptable value %q: must be one of %q", label, value, accepted)
+}
+
+func (ldr *Loader) setImplicitStorageClasses(cfg *arvados.Config) error {
+cluster:
+       for id, cc := range cfg.Clusters {
+               if len(cc.StorageClasses) > 0 {
+                       continue cluster
+               }
+               for _, vol := range cc.Volumes {
+                       if len(vol.StorageClasses) > 0 {
+                               continue cluster
+                       }
+               }
+               // No explicit StorageClasses config info at all; fill
+               // in implicit defaults.
+               for id, vol := range cc.Volumes {
+                       vol.StorageClasses = map[string]bool{"default": true}
+                       cc.Volumes[id] = vol
+               }
+               cc.StorageClasses = map[string]arvados.StorageClassConfig{"default": {Default: true}}
+               cfg.Clusters[id] = cc
+       }
+       return nil
+}
+
+func (ldr *Loader) checkStorageClasses(cc arvados.Cluster) error {
+       classOnVolume := map[string]bool{}
+       for volid, vol := range cc.Volumes {
+               if len(vol.StorageClasses) == 0 {
+                       return fmt.Errorf("%s: volume has no StorageClasses listed", volid)
+               }
+               for classid := range vol.StorageClasses {
+                       if _, ok := cc.StorageClasses[classid]; !ok {
+                               return fmt.Errorf("%s: volume refers to storage class %q that is not defined in StorageClasses", volid, classid)
+                       }
+                       classOnVolume[classid] = true
+               }
+       }
+       haveDefault := false
+       for classid, sc := range cc.StorageClasses {
+               if !classOnVolume[classid] && len(cc.Volumes) > 0 {
+                       ldr.Logger.Warnf("there are no volumes providing storage class %q", classid)
+               }
+               if sc.Default {
+                       haveDefault = true
+               }
+       }
+       if !haveDefault {
+               return fmt.Errorf("there is no default storage class (at least one entry in StorageClasses must have Default: true)")
+       }
+       return nil
+}
+
 func checkKeyConflict(label string, m map[string]string) error {
        saw := map[string]bool{}
        for k := range m {
@@ -361,6 +431,9 @@ func (ldr *Loader) logExtraKeys(expected, supplied map[string]interface{}, prefi
                }
                vexp, ok := expected[k]
                if expected["SAMPLE"] != nil {
+                       // use the SAMPLE entry's keys as the
+                       // "expected" map when checking vsupp
+                       // recursively.
                        vexp = expected["SAMPLE"]
                } else if !ok {
                        // check for a case-insensitive match
@@ -368,6 +441,12 @@ func (ldr *Loader) logExtraKeys(expected, supplied map[string]interface{}, prefi
                        for ek := range expected {
                                if strings.EqualFold(k, ek) {
                                        hint = " (perhaps you meant " + ek + "?)"
+                                       // If we don't delete this, it
+                                       // will end up getting merged,
+                                       // unpredictably
+                                       // merging/overriding the
+                                       // default.
+                                       delete(supplied, k)
                                        break
                                }
                        }