lib/config/load_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "bytes"
   9         "io"
  10         "os"
  11         "os/exec"
  12         "reflect"
  13         "strings"
  14         "testing"
  15
  16         "git.curoverse.com/arvados.git/sdk/go/arvados"
  17         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  18         "github.com/ghodss/yaml"
  19         "github.com/sirupsen/logrus"
  20         check "gopkg.in/check.v1"
  21 )
  22
  23 // Gocheck boilerplate
  24 func Test(t *testing.T) {
  25         check.TestingT(t)
  26 }
  27
  28 var _ = check.Suite(&LoadSuite{})
  29
  30 type LoadSuite struct{}
  31
  32 func (s *LoadSuite) TestEmpty(c *check.C) {
  33         cfg, err := Load(&bytes.Buffer{}, ctxlog.TestLogger(c))
  34         c.Check(cfg, check.IsNil)
  35         c.Assert(err, check.ErrorMatches, `config does not define any clusters`)
  36 }
  37
  38 func (s *LoadSuite) TestNoConfigs(c *check.C) {
  39         cfg, err := Load(bytes.NewBufferString(`Clusters: {"z1111": {}}`), ctxlog.TestLogger(c))
  40         c.Assert(err, check.IsNil)
  41         c.Assert(cfg.Clusters, check.HasLen, 1)
  42         cc, err := cfg.GetCluster("z1111")
  43         c.Assert(err, check.IsNil)
  44         c.Check(cc.ClusterID, check.Equals, "z1111")
  45         c.Check(cc.API.MaxRequestAmplification, check.Equals, 4)
  46         c.Check(cc.API.MaxItemsPerResponse, check.Equals, 1000)
  47 }
  48
  49 func (s *LoadSuite) TestSampleKeys(c *check.C) {
  50         for _, yaml := range []string{
  51                 `{"Clusters":{"z1111":{}}}`,
  52                 `{"Clusters":{"z1111":{"InstanceTypes":{"Foo":{"RAM": "12345M"}}}}}`,
  53         } {
  54                 cfg, err := Load(bytes.NewBufferString(yaml), ctxlog.TestLogger(c))
  55                 c.Assert(err, check.IsNil)
  56                 cc, err := cfg.GetCluster("z1111")
  57                 _, hasSample := cc.InstanceTypes["SAMPLE"]
  58                 c.Check(hasSample, check.Equals, false)
  59                 if strings.Contains(yaml, "Foo") {
  60                         c.Check(cc.InstanceTypes["Foo"].RAM, check.Equals, arvados.ByteSize(12345000000))
  61                         c.Check(cc.InstanceTypes["Foo"].Price, check.Equals, 0.0)
  62                 }
  63         }
  64 }
  65
  66 func (s *LoadSuite) TestMultipleClusters(c *check.C) {
  67         cfg, err := Load(bytes.NewBufferString(`{"Clusters":{"z1111":{},"z2222":{}}}`), ctxlog.TestLogger(c))
  68         c.Assert(err, check.IsNil)
  69         c1, err := cfg.GetCluster("z1111")
  70         c.Assert(err, check.IsNil)
  71         c.Check(c1.ClusterID, check.Equals, "z1111")
  72         c2, err := cfg.GetCluster("z2222")
  73         c.Assert(err, check.IsNil)
  74         c.Check(c2.ClusterID, check.Equals, "z2222")
  75 }
  76
  77 func (s *LoadSuite) TestDeprecatedOrUnknownWarning(c *check.C) {
  78         var logbuf bytes.Buffer
  79         logger := logrus.New()
  80         logger.Out = &logbuf
  81         _, err := Load(bytes.NewBufferString(`
  82 Clusters:
  83   zzzzz:
  84     postgresql: {}
  85     BadKey: {}
  86     Containers: {}
  87     RemoteClusters:
  88       z2222:
  89         Host: z2222.arvadosapi.com
  90         Proxy: true
  91         BadKey: badValue
  92 `), logger)
  93         c.Assert(err, check.IsNil)
  94         logs := strings.Split(strings.TrimSuffix(logbuf.String(), "\n"), "\n")
  95         for _, log := range logs {
  96                 c.Check(log, check.Matches, `.*deprecated or unknown config entry:.*BadKey.*`)
  97         }
  98         c.Check(logs, check.HasLen, 2)
  99 }
 100
 101 func (s *LoadSuite) checkSAMPLEKeys(c *check.C, path string, x interface{}) {
 102         v := reflect.Indirect(reflect.ValueOf(x))
 103         switch v.Kind() {
 104         case reflect.Map:
 105                 var stringKeys, sampleKey bool
 106                 iter := v.MapRange()
 107                 for iter.Next() {
 108                         k := iter.Key()
 109                         if k.Kind() == reflect.String {
 110                                 stringKeys = true
 111                                 if k.String() == "SAMPLE" || k.String() == "xxxxx" {
 112                                         sampleKey = true
 113                                         s.checkSAMPLEKeys(c, path+"."+k.String(), iter.Value().Interface())
 114                                 }
 115                         }
 116                 }
 117                 if stringKeys && !sampleKey {
 118                         c.Errorf("%s is a map with string keys (type %T) but config.default.yml has no SAMPLE key", path, x)
 119                 }
 120                 return
 121         case reflect.Struct:
 122                 for i := 0; i < v.NumField(); i++ {
 123                         val := v.Field(i)
 124                         if val.CanInterface() {
 125                                 s.checkSAMPLEKeys(c, path+"."+v.Type().Field(i).Name, val.Interface())
 126                         }
 127                 }
 128         }
 129 }
 130
 131 func (s *LoadSuite) TestDefaultConfigHasAllSAMPLEKeys(c *check.C) {
 132         cfg, err := Load(bytes.NewBuffer(DefaultYAML), ctxlog.TestLogger(c))
 133         c.Assert(err, check.IsNil)
 134         s.checkSAMPLEKeys(c, "", cfg)
 135 }
 136
 137 func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
 138         var logbuf bytes.Buffer
 139         logger := logrus.New()
 140         logger.Out = &logbuf
 141         var supplied map[string]interface{}
 142         yaml.Unmarshal(DefaultYAML, &supplied)
 143         cfg, err := Load(bytes.NewBuffer(DefaultYAML), logger)
 144         c.Assert(err, check.IsNil)
 145         var loaded map[string]interface{}
 146         buf, err := yaml.Marshal(cfg)
 147         c.Assert(err, check.IsNil)
 148         err = yaml.Unmarshal(buf, &loaded)
 149         c.Assert(err, check.IsNil)
 150
 151         logExtraKeys(logger, loaded, supplied, "")
 152         c.Check(logbuf.String(), check.Equals, "")
 153 }
 154
 155 func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
 156         var logbuf bytes.Buffer
 157         logger := logrus.New()
 158         logger.Out = &logbuf
 159         cfg, err := Load(bytes.NewBufferString(`{"Clusters":{"zzzzz":{}}}`), logger)
 160         c.Assert(err, check.IsNil)
 161         yaml, err := yaml.Marshal(cfg)
 162         c.Assert(err, check.IsNil)
 163         cfgDumped, err := Load(bytes.NewBuffer(yaml), logger)
 164         c.Assert(err, check.IsNil)
 165         c.Check(cfg, check.DeepEquals, cfgDumped)
 166         c.Check(logbuf.String(), check.Equals, "")
 167 }
 168
 169 func (s *LoadSuite) TestPostgreSQLKeyConflict(c *check.C) {
 170         _, err := Load(bytes.NewBufferString(`
 171 Clusters:
 172  zzzzz:
 173   postgresql:
 174    connection:
 175      DBName: dbname
 176      Host: host
 177 `), ctxlog.TestLogger(c))
 178         c.Check(err, check.ErrorMatches, `Clusters.zzzzz.PostgreSQL.Connection: multiple entries for "(dbname|host)".*`)
 179 }
 180
 181 func (s *LoadSuite) TestBadType(c *check.C) {
 182         for _, data := range []string{`
 183 Clusters:
 184  zzzzz:
 185   PostgreSQL: true
 186 `, `
 187 Clusters:
 188  zzzzz:
 189   PostgreSQL:
 190    ConnectionPool: true
 191 `, `
 192 Clusters:
 193  zzzzz:
 194   PostgreSQL:
 195    ConnectionPool: "foo"
 196 `, `
 197 Clusters:
 198  zzzzz:
 199   PostgreSQL:
 200    ConnectionPool: []
 201 `, `
 202 Clusters:
 203  zzzzz:
 204   PostgreSQL:
 205    ConnectionPool: [] # {foo: bar} isn't caught here; we rely on config-check
 206 `,
 207         } {
 208                 c.Log(data)
 209                 v, err := Load(bytes.NewBufferString(data), ctxlog.TestLogger(c))
 210                 if v != nil {
 211                         c.Logf("%#v", v.Clusters["zzzzz"].PostgreSQL.ConnectionPool)
 212                 }
 213                 c.Check(err, check.ErrorMatches, `.*cannot unmarshal .*PostgreSQL.*`)
 214         }
 215 }
 216
 217 func (s *LoadSuite) TestMovedKeys(c *check.C) {
 218         s.checkEquivalent(c, `# config has old keys only
 219 Clusters:
 220  zzzzz:
 221   RequestLimits:
 222    MultiClusterRequestConcurrency: 3
 223    MaxItemsPerResponse: 999
 224 `, `
 225 Clusters:
 226  zzzzz:
 227   API:
 228    MaxRequestAmplification: 3
 229    MaxItemsPerResponse: 999
 230 `)
 231         s.checkEquivalent(c, `# config has both old and new keys; old values win
 232 Clusters:
 233  zzzzz:
 234   RequestLimits:
 235    MultiClusterRequestConcurrency: 0
 236    MaxItemsPerResponse: 555
 237   API:
 238    MaxRequestAmplification: 3
 239    MaxItemsPerResponse: 999
 240 `, `
 241 Clusters:
 242  zzzzz:
 243   API:
 244    MaxRequestAmplification: 0
 245    MaxItemsPerResponse: 555
 246 `)
 247 }
 248
 249 func (s *LoadSuite) checkEquivalent(c *check.C, goty, expectedy string) {
 250         got, err := Load(bytes.NewBufferString(goty), ctxlog.TestLogger(c))
 251         c.Assert(err, check.IsNil)
 252         expected, err := Load(bytes.NewBufferString(expectedy), ctxlog.TestLogger(c))
 253         c.Assert(err, check.IsNil)
 254         if !c.Check(got, check.DeepEquals, expected) {
 255                 cmd := exec.Command("diff", "-u", "--label", "expected", "--label", "got", "/dev/fd/3", "/dev/fd/4")
 256                 for _, obj := range []interface{}{expected, got} {
 257                         y, _ := yaml.Marshal(obj)
 258                         pr, pw, err := os.Pipe()
 259                         c.Assert(err, check.IsNil)
 260                         defer pr.Close()
 261                         go func() {
 262                                 io.Copy(pw, bytes.NewBuffer(y))
 263                                 pw.Close()
 264                         }()
 265                         cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
 266                 }
 267                 diff, err := cmd.CombinedOutput()
 268                 c.Log(string(diff))
 269                 c.Check(err, check.IsNil)
 270         }
 271 }