lib/config/load_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "io"
  11         "io/ioutil"
  12         "os"
  13         "os/exec"
  14         "reflect"
  15         "strings"
  16         "testing"
  17
  18         "git.curoverse.com/arvados.git/sdk/go/arvados"
  19         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  20         "github.com/ghodss/yaml"
  21         "github.com/sirupsen/logrus"
  22         check "gopkg.in/check.v1"
  23 )
  24
  25 // Gocheck boilerplate
  26 func Test(t *testing.T) {
  27         check.TestingT(t)
  28 }
  29
  30 var _ = check.Suite(&LoadSuite{})
  31
  32 // Return a new Loader that reads cluster config from configdata
  33 // (instead of the usual default /etc/arvados/config.yml), and logs to
  34 // logdst or (if that's nil) c.Log.
  35 func testLoader(c *check.C, configdata string, logdst io.Writer) *Loader {
  36         logger := ctxlog.TestLogger(c)
  37         if logdst != nil {
  38                 lgr := logrus.New()
  39                 lgr.Out = logdst
  40                 logger = lgr
  41         }
  42         ldr := NewLoader(bytes.NewBufferString(configdata), logger)
  43         ldr.Path = "-"
  44         return ldr
  45 }
  46
  47 type LoadSuite struct{}
  48
  49 func (s *LoadSuite) TestEmpty(c *check.C) {
  50         cfg, err := testLoader(c, "", nil).Load()
  51         c.Check(cfg, check.IsNil)
  52         c.Assert(err, check.ErrorMatches, `config does not define any clusters`)
  53 }
  54
  55 func (s *LoadSuite) TestNoConfigs(c *check.C) {
  56         cfg, err := testLoader(c, `Clusters: {"z1111": {}}`, nil).Load()
  57         c.Assert(err, check.IsNil)
  58         c.Assert(cfg.Clusters, check.HasLen, 1)
  59         cc, err := cfg.GetCluster("z1111")
  60         c.Assert(err, check.IsNil)
  61         c.Check(cc.ClusterID, check.Equals, "z1111")
  62         c.Check(cc.API.MaxRequestAmplification, check.Equals, 4)
  63         c.Check(cc.API.MaxItemsPerResponse, check.Equals, 1000)
  64 }
  65
  66 func (s *LoadSuite) TestMungeLegacyConfigArgs(c *check.C) {
  67         f, err := ioutil.TempFile("", "")
  68         c.Check(err, check.IsNil)
  69         defer os.Remove(f.Name())
  70         io.WriteString(f, "Debug: true\n")
  71         oldfile := f.Name()
  72
  73         f, err = ioutil.TempFile("", "")
  74         c.Check(err, check.IsNil)
  75         defer os.Remove(f.Name())
  76         io.WriteString(f, "Clusters: {aaaaa: {}}\n")
  77         newfile := f.Name()
  78
  79         for _, trial := range []struct {
  80                 argsIn  []string
  81                 argsOut []string
  82         }{
  83                 {
  84                         []string{"-config", oldfile},
  85                         []string{"-old-config", oldfile},
  86                 },
  87                 {
  88                         []string{"-config=" + oldfile},
  89                         []string{"-old-config=" + oldfile},
  90                 },
  91                 {
  92                         []string{"-config", newfile},
  93                         []string{"-config", newfile},
  94                 },
  95                 {
  96                         []string{"-config=" + newfile},
  97                         []string{"-config=" + newfile},
  98                 },
  99                 {
 100                         []string{"-foo", oldfile},
 101                         []string{"-foo", oldfile},
 102                 },
 103                 {
 104                         []string{"-foo=" + oldfile},
 105                         []string{"-foo=" + oldfile},
 106                 },
 107                 {
 108                         []string{"-foo", "-config=" + oldfile},
 109                         []string{"-foo", "-old-config=" + oldfile},
 110                 },
 111                 {
 112                         []string{"-foo", "bar", "-config=" + oldfile},
 113                         []string{"-foo", "bar", "-old-config=" + oldfile},
 114                 },
 115                 {
 116                         []string{"-foo=bar", "baz", "-config=" + oldfile},
 117                         []string{"-foo=bar", "baz", "-old-config=" + oldfile},
 118                 },
 119                 {
 120                         []string{"-config=/dev/null"},
 121                         []string{"-config=/dev/null"},
 122                 },
 123                 {
 124                         []string{"-config=-"},
 125                         []string{"-config=-"},
 126                 },
 127                 {
 128                         []string{"-config="},
 129                         []string{"-config="},
 130                 },
 131                 {
 132                         []string{"-foo=bar", "baz", "-config"},
 133                         []string{"-foo=bar", "baz", "-config"},
 134                 },
 135                 {
 136                         []string{},
 137                         nil,
 138                 },
 139         } {
 140                 var logbuf bytes.Buffer
 141                 logger := logrus.New()
 142                 logger.Out = &logbuf
 143
 144                 var ldr Loader
 145                 args := ldr.MungeLegacyConfigArgs(logger, trial.argsIn, "-old-config")
 146                 c.Check(args, check.DeepEquals, trial.argsOut)
 147                 if fmt.Sprintf("%v", trial.argsIn) != fmt.Sprintf("%v", trial.argsOut) {
 148                         c.Check(logbuf.String(), check.Matches, `.*`+oldfile+` is not a cluster config file -- interpreting -config as -old-config.*\n`)
 149                 }
 150         }
 151 }
 152
 153 func (s *LoadSuite) TestSampleKeys(c *check.C) {
 154         for _, yaml := range []string{
 155                 `{"Clusters":{"z1111":{}}}`,
 156                 `{"Clusters":{"z1111":{"InstanceTypes":{"Foo":{"RAM": "12345M"}}}}}`,
 157         } {
 158                 cfg, err := testLoader(c, yaml, nil).Load()
 159                 c.Assert(err, check.IsNil)
 160                 cc, err := cfg.GetCluster("z1111")
 161                 _, hasSample := cc.InstanceTypes["SAMPLE"]
 162                 c.Check(hasSample, check.Equals, false)
 163                 if strings.Contains(yaml, "Foo") {
 164                         c.Check(cc.InstanceTypes["Foo"].RAM, check.Equals, arvados.ByteSize(12345000000))
 165                         c.Check(cc.InstanceTypes["Foo"].Price, check.Equals, 0.0)
 166                 }
 167         }
 168 }
 169
 170 func (s *LoadSuite) TestMultipleClusters(c *check.C) {
 171         ldr := testLoader(c, `{"Clusters":{"z1111":{},"z2222":{}}}`, nil)
 172         ldr.SkipDeprecated = true
 173         cfg, err := ldr.Load()
 174         c.Assert(err, check.IsNil)
 175         c1, err := cfg.GetCluster("z1111")
 176         c.Assert(err, check.IsNil)
 177         c.Check(c1.ClusterID, check.Equals, "z1111")
 178         c2, err := cfg.GetCluster("z2222")
 179         c.Assert(err, check.IsNil)
 180         c.Check(c2.ClusterID, check.Equals, "z2222")
 181 }
 182
 183 func (s *LoadSuite) TestDeprecatedOrUnknownWarning(c *check.C) {
 184         var logbuf bytes.Buffer
 185         _, err := testLoader(c, `
 186 Clusters:
 187   zzzzz:
 188     postgresql: {}
 189     BadKey: {}
 190     Containers: {}
 191     RemoteClusters:
 192       z2222:
 193         Host: z2222.arvadosapi.com
 194         Proxy: true
 195         BadKey: badValue
 196 `, &logbuf).Load()
 197         c.Assert(err, check.IsNil)
 198         logs := strings.Split(strings.TrimSuffix(logbuf.String(), "\n"), "\n")
 199         for _, log := range logs {
 200                 c.Check(log, check.Matches, `.*deprecated or unknown config entry:.*BadKey.*`)
 201         }
 202         c.Check(logs, check.HasLen, 2)
 203 }
 204
 205 func (s *LoadSuite) checkSAMPLEKeys(c *check.C, path string, x interface{}) {
 206         v := reflect.Indirect(reflect.ValueOf(x))
 207         switch v.Kind() {
 208         case reflect.Map:
 209                 var stringKeys, sampleKey bool
 210                 iter := v.MapRange()
 211                 for iter.Next() {
 212                         k := iter.Key()
 213                         if k.Kind() == reflect.String {
 214                                 stringKeys = true
 215                                 if k.String() == "SAMPLE" || k.String() == "xxxxx" {
 216                                         sampleKey = true
 217                                         s.checkSAMPLEKeys(c, path+"."+k.String(), iter.Value().Interface())
 218                                 }
 219                         }
 220                 }
 221                 if stringKeys && !sampleKey {
 222                         c.Errorf("%s is a map with string keys (type %T) but config.default.yml has no SAMPLE key", path, x)
 223                 }
 224                 return
 225         case reflect.Struct:
 226                 for i := 0; i < v.NumField(); i++ {
 227                         val := v.Field(i)
 228                         if val.CanInterface() {
 229                                 s.checkSAMPLEKeys(c, path+"."+v.Type().Field(i).Name, val.Interface())
 230                         }
 231                 }
 232         }
 233 }
 234
 235 func (s *LoadSuite) TestDefaultConfigHasAllSAMPLEKeys(c *check.C) {
 236         var logbuf bytes.Buffer
 237         loader := testLoader(c, string(DefaultYAML), &logbuf)
 238         cfg, err := loader.Load()
 239         c.Assert(err, check.IsNil)
 240         s.checkSAMPLEKeys(c, "", cfg)
 241 }
 242
 243 func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
 244         var logbuf bytes.Buffer
 245         var supplied map[string]interface{}
 246         yaml.Unmarshal(DefaultYAML, &supplied)
 247
 248         loader := testLoader(c, string(DefaultYAML), &logbuf)
 249         cfg, err := loader.Load()
 250         c.Assert(err, check.IsNil)
 251         var loaded map[string]interface{}
 252         buf, err := yaml.Marshal(cfg)
 253         c.Assert(err, check.IsNil)
 254         err = yaml.Unmarshal(buf, &loaded)
 255         c.Assert(err, check.IsNil)
 256
 257         loader.logExtraKeys(loaded, supplied, "")
 258         c.Check(logbuf.String(), check.Equals, "")
 259 }
 260
 261 func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
 262         var logbuf bytes.Buffer
 263         logger := logrus.New()
 264         logger.Out = &logbuf
 265         cfg, err := testLoader(c, `{"Clusters":{"zzzzz":{}}}`, &logbuf).Load()
 266         c.Assert(err, check.IsNil)
 267         yaml, err := yaml.Marshal(cfg)
 268         c.Assert(err, check.IsNil)
 269         cfgDumped, err := testLoader(c, string(yaml), &logbuf).Load()
 270         c.Assert(err, check.IsNil)
 271         c.Check(cfg, check.DeepEquals, cfgDumped)
 272         c.Check(logbuf.String(), check.Equals, "")
 273 }
 274
 275 func (s *LoadSuite) TestPostgreSQLKeyConflict(c *check.C) {
 276         _, err := testLoader(c, `
 277 Clusters:
 278  zzzzz:
 279   postgresql:
 280    connection:
 281      DBName: dbname
 282      Host: host
 283 `, nil).Load()
 284         c.Check(err, check.ErrorMatches, `Clusters.zzzzz.PostgreSQL.Connection: multiple entries for "(dbname|host)".*`)
 285 }
 286
 287 func (s *LoadSuite) TestBadType(c *check.C) {
 288         for _, data := range []string{`
 289 Clusters:
 290  zzzzz:
 291   PostgreSQL: true
 292 `, `
 293 Clusters:
 294  zzzzz:
 295   PostgreSQL:
 296    ConnectionPool: true
 297 `, `
 298 Clusters:
 299  zzzzz:
 300   PostgreSQL:
 301    ConnectionPool: "foo"
 302 `, `
 303 Clusters:
 304  zzzzz:
 305   PostgreSQL:
 306    ConnectionPool: []
 307 `, `
 308 Clusters:
 309  zzzzz:
 310   PostgreSQL:
 311    ConnectionPool: [] # {foo: bar} isn't caught here; we rely on config-check
 312 `,
 313         } {
 314                 c.Log(data)
 315                 v, err := testLoader(c, data, nil).Load()
 316                 if v != nil {
 317                         c.Logf("%#v", v.Clusters["zzzzz"].PostgreSQL.ConnectionPool)
 318                 }
 319                 c.Check(err, check.ErrorMatches, `.*cannot unmarshal .*PostgreSQL.*`)
 320         }
 321 }
 322
 323 func (s *LoadSuite) TestMovedKeys(c *check.C) {
 324         checkEquivalent(c, `# config has old keys only
 325 Clusters:
 326  zzzzz:
 327   RequestLimits:
 328    MultiClusterRequestConcurrency: 3
 329    MaxItemsPerResponse: 999
 330 `, `
 331 Clusters:
 332  zzzzz:
 333   API:
 334    MaxRequestAmplification: 3
 335    MaxItemsPerResponse: 999
 336 `)
 337         checkEquivalent(c, `# config has both old and new keys; old values win
 338 Clusters:
 339  zzzzz:
 340   RequestLimits:
 341    MultiClusterRequestConcurrency: 0
 342    MaxItemsPerResponse: 555
 343   API:
 344    MaxRequestAmplification: 3
 345    MaxItemsPerResponse: 999
 346 `, `
 347 Clusters:
 348  zzzzz:
 349   API:
 350    MaxRequestAmplification: 0
 351    MaxItemsPerResponse: 555
 352 `)
 353 }
 354
 355 func checkEquivalent(c *check.C, goty, expectedy string) {
 356         gotldr := testLoader(c, goty, nil)
 357         expectedldr := testLoader(c, expectedy, nil)
 358         checkEquivalentLoaders(c, gotldr, expectedldr)
 359 }
 360
 361 func checkEqualYAML(c *check.C, got, expected interface{}) {
 362         expectedyaml, err := yaml.Marshal(expected)
 363         c.Assert(err, check.IsNil)
 364         gotyaml, err := yaml.Marshal(got)
 365         c.Assert(err, check.IsNil)
 366         if !bytes.Equal(gotyaml, expectedyaml) {
 367                 cmd := exec.Command("diff", "-u", "--label", "expected", "--label", "got", "/dev/fd/3", "/dev/fd/4")
 368                 for _, y := range [][]byte{expectedyaml, gotyaml} {
 369                         pr, pw, err := os.Pipe()
 370                         c.Assert(err, check.IsNil)
 371                         defer pr.Close()
 372                         go func(data []byte) {
 373                                 pw.Write(data)
 374                                 pw.Close()
 375                         }(y)
 376                         cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
 377                 }
 378                 diff, err := cmd.CombinedOutput()
 379                 // diff should report differences and exit non-zero.
 380                 c.Check(err, check.NotNil)
 381                 c.Log(string(diff))
 382                 c.Error("got != expected; see diff (-expected +got) above")
 383         }
 384 }
 385
 386 func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) {
 387         got, err := gotldr.Load()
 388         c.Assert(err, check.IsNil)
 389         expected, err := expectedldr.Load()
 390         c.Assert(err, check.IsNil)
 391         checkEqualYAML(c, got, expected)
 392 }
 393
 394 func checkListKeys(path string, x interface{}) (err error) {
 395         v := reflect.Indirect(reflect.ValueOf(x))
 396         switch v.Kind() {
 397         case reflect.Map:
 398                 iter := v.MapRange()
 399                 for iter.Next() {
 400                         k := iter.Key()
 401                         if k.Kind() == reflect.String {
 402                                 if err = checkListKeys(path+"."+k.String(), iter.Value().Interface()); err != nil {
 403                                         return
 404                                 }
 405                         }
 406                 }
 407                 return
 408
 409         case reflect.Struct:
 410                 for i := 0; i < v.NumField(); i++ {
 411                         val := v.Field(i)
 412                         structField := v.Type().Field(i)
 413                         fieldname := structField.Name
 414                         endsWithList := strings.HasSuffix(fieldname, "List")
 415                         isAnArray := structField.Type.Kind() == reflect.Slice
 416                         if endsWithList != isAnArray {
 417                                 if endsWithList {
 418                                         err = fmt.Errorf("%s.%s ends with 'List' but field is not an array (type %v)", path, fieldname, val.Kind())
 419                                         return
 420                                 }
 421                                 if isAnArray && structField.Type.Elem().Kind() != reflect.Uint8 {
 422                                         err = fmt.Errorf("%s.%s is an array but field name does not end in 'List' (slice of %v)", path, fieldname, structField.Type.Elem().Kind())
 423                                         return
 424                                 }
 425                         }
 426                         if val.CanInterface() {
 427                                 checkListKeys(path+"."+fieldname, val.Interface())
 428                         }
 429                 }
 430         }
 431         return
 432 }
 433
 434 func (s *LoadSuite) TestListKeys(c *check.C) {
 435         v1 := struct {
 436                 EndInList []string
 437         }{[]string{"a", "b"}}
 438         var m1 = make(map[string]interface{})
 439         m1["c"] = &v1
 440         if err := checkListKeys("", m1); err != nil {
 441                 c.Error(err)
 442         }
 443
 444         v2 := struct {
 445                 DoesNot []string
 446         }{[]string{"a", "b"}}
 447         var m2 = make(map[string]interface{})
 448         m2["c"] = &v2
 449         if err := checkListKeys("", m2); err == nil {
 450                 c.Errorf("Should have produced an error")
 451         }
 452
 453         v3 := struct {
 454                 EndInList string
 455         }{"a"}
 456         var m3 = make(map[string]interface{})
 457         m3["c"] = &v3
 458         if err := checkListKeys("", m3); err == nil {
 459                 c.Errorf("Should have produced an error")
 460         }
 461
 462         var logbuf bytes.Buffer
 463         loader := testLoader(c, string(DefaultYAML), &logbuf)
 464         cfg, err := loader.Load()
 465         c.Assert(err, check.IsNil)
 466         if err := checkListKeys("", cfg); err != nil {
 467                 c.Error(err)
 468         }
 469 }