import (
"bytes"
+ "encoding/json"
"errors"
"fmt"
"io"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"github.com/ghodss/yaml"
+ "github.com/imdario/mergo"
)
type logger interface {
Warnf(string, ...interface{})
}
-type deprecatedConfig struct {
- Clusters map[string]struct {
- NodeProfiles map[string]arvados.NodeProfile
+func loadFileOrStdin(path string, stdin io.Reader, log logger) (*arvados.Config, error) {
+ if path == "-" {
+ return load(stdin, log, true)
+ } else {
+ return LoadFile(path, log)
}
}
+func LoadFile(path string, log logger) (*arvados.Config, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ return Load(f, log)
+}
+
func Load(rdr io.Reader, log logger) (*arvados.Config, error) {
- var cfg arvados.Config
+ return load(rdr, log, true)
+}
+
+func load(rdr io.Reader, log logger, useDeprecated bool) (*arvados.Config, error) {
buf, err := ioutil.ReadAll(rdr)
if err != nil {
return nil, err
if len(dummy.Clusters) == 0 {
return nil, errors.New("config does not define any clusters")
}
+
+ // We can't merge deep structs here; instead, we unmarshal the
+ // default & loaded config files into generic maps, merge
+ // those, and then json-encode+decode the result into the
+ // config struct type.
+ var merged map[string]interface{}
for id := range dummy.Clusters {
- err = yaml.Unmarshal(bytes.Replace(DefaultYAML, []byte("xxxxx"), []byte(id), -1), &cfg)
+ var src map[string]interface{}
+ err = yaml.Unmarshal(bytes.Replace(DefaultYAML, []byte(" xxxxx:"), []byte(" "+id+":"), -1), &src)
if err != nil {
return nil, fmt.Errorf("loading defaults for %s: %s", id, err)
}
+ err = mergo.Merge(&merged, src, mergo.WithOverride)
+ if err != nil {
+ return nil, fmt.Errorf("merging defaults for %s: %s", id, err)
+ }
}
- err = yaml.Unmarshal(buf, &cfg)
+ var src map[string]interface{}
+ err = yaml.Unmarshal(buf, &src)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("loading config data: %s", err)
}
-
- // Check for deprecated config values, and apply them to cfg.
- var dc deprecatedConfig
- err = yaml.Unmarshal(buf, &dc)
+ logExtraKeys(log, merged, src, "")
+ removeSampleKeys(merged)
+ err = mergo.Merge(&merged, src, mergo.WithOverride)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("merging config data: %s", err)
+ }
+
+ // map[string]interface{} => json => arvados.Config
+ var cfg arvados.Config
+ var errEnc error
+ pr, pw := io.Pipe()
+ go func() {
+ errEnc = json.NewEncoder(pw).Encode(merged)
+ pw.Close()
+ }()
+ err = json.NewDecoder(pr).Decode(&cfg)
+ if errEnc != nil {
+ err = errEnc
}
- err = applyDeprecatedConfig(&cfg, &dc, log)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("transcoding config data: %s", err)
}
- return &cfg, nil
-}
-func applyDeprecatedConfig(cfg *arvados.Config, dc *deprecatedConfig, log logger) error {
- hostname, err := os.Hostname()
- if err != nil {
- return err
+ if useDeprecated {
+ err = applyDeprecatedConfig(&cfg, buf, log)
+ if err != nil {
+ return nil, err
+ }
}
- for id, dcluster := range dc.Clusters {
- cluster, ok := cfg.Clusters[id]
- if !ok {
- return fmt.Errorf("can't load legacy config %q that is not present in current config", id)
+
+ // Check for known mistakes
+ for id, cc := range cfg.Clusters {
+ err = checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection)
+ if err != nil {
+ return nil, err
}
- for name, np := range dcluster.NodeProfiles {
- if name == "*" || name == os.Getenv("ARVADOS_NODE_PROFILE") || name == hostname {
- applyDeprecatedNodeProfile(hostname, np.RailsAPI, &cluster.Services.RailsAPI)
- applyDeprecatedNodeProfile(hostname, np.Controller, &cluster.Services.Controller)
- applyDeprecatedNodeProfile(hostname, np.DispatchCloud, &cluster.Services.DispatchCloud)
- }
+ }
+ return &cfg, nil
+}
+
+func checkKeyConflict(label string, m map[string]string) error {
+ saw := map[string]bool{}
+ for k := range m {
+ k = strings.ToLower(k)
+ if saw[k] {
+ return fmt.Errorf("%s: multiple entries for %q (fix by using same capitalization as default/example file)", label, k)
}
- cfg.Clusters[id] = cluster
+ saw[k] = true
}
return nil
}
-func applyDeprecatedNodeProfile(hostname string, ssi arvados.SystemServiceInstance, svc *arvados.Service) {
- scheme := "https"
- if !ssi.TLS {
- scheme = "http"
- }
- if svc.InternalURLs == nil {
- svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{}
+func removeSampleKeys(m map[string]interface{}) {
+ delete(m, "SAMPLE")
+ for _, v := range m {
+ if v, _ := v.(map[string]interface{}); v != nil {
+ removeSampleKeys(v)
+ }
}
- host := ssi.Listen
- if host == "" {
+}
+
+func logExtraKeys(log logger, expected, supplied map[string]interface{}, prefix string) {
+ if log == nil {
return
}
- if strings.HasPrefix(host, ":") {
- host = hostname + host
+ allowed := map[string]interface{}{}
+ for k, v := range expected {
+ allowed[strings.ToLower(k)] = v
+ }
+ for k, vsupp := range supplied {
+ if k == "SAMPLE" {
+ // entry will be dropped in removeSampleKeys anyway
+ continue
+ }
+ vexp, ok := allowed[strings.ToLower(k)]
+ if expected["SAMPLE"] != nil {
+ vexp = expected["SAMPLE"]
+ } else if !ok {
+ log.Warnf("deprecated or unknown config entry: %s%s", prefix, k)
+ continue
+ }
+ if vsupp, ok := vsupp.(map[string]interface{}); !ok {
+ // if vsupp is a map but vexp isn't map, this
+ // will be caught elsewhere; see TestBadType.
+ continue
+ } else if vexp, ok := vexp.(map[string]interface{}); !ok {
+ log.Warnf("unexpected object in config entry: %s%s", prefix, k)
+ } else {
+ logExtraKeys(log, vexp, vsupp, prefix+k+".")
+ }
}
- svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host}] = arvados.ServiceInstance{}
}
projects / arvados.git / blobdiff