lib/config/load_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "io"
  11         "io/ioutil"
  12         "os"
  13         "os/exec"
  14         "strings"
  15         "testing"
  16
  17         "git.curoverse.com/arvados.git/sdk/go/arvados"
  18         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  19         "github.com/ghodss/yaml"
  20         "github.com/sirupsen/logrus"
  21         check "gopkg.in/check.v1"
  22 )
  23
  24 // Gocheck boilerplate
  25 func Test(t *testing.T) {
  26         check.TestingT(t)
  27 }
  28
  29 var _ = check.Suite(&LoadSuite{})
  30
  31 // Return a new Loader that reads cluster config from configdata
  32 // (instead of the usual default /etc/arvados/config.yml), and logs to
  33 // logdst or (if that's nil) c.Log.
  34 func testLoader(c *check.C, configdata string, logdst io.Writer) *Loader {
  35         logger := ctxlog.TestLogger(c)
  36         if logdst != nil {
  37                 lgr := logrus.New()
  38                 lgr.Out = logdst
  39                 logger = lgr
  40         }
  41         ldr := NewLoader(bytes.NewBufferString(configdata), logger)
  42         ldr.Path = "-"
  43         return ldr
  44 }
  45
  46 type LoadSuite struct{}
  47
  48 func (s *LoadSuite) TestEmpty(c *check.C) {
  49         cfg, err := testLoader(c, "", nil).Load()
  50         c.Check(cfg, check.IsNil)
  51         c.Assert(err, check.ErrorMatches, `config does not define any clusters`)
  52 }
  53
  54 func (s *LoadSuite) TestNoConfigs(c *check.C) {
  55         cfg, err := testLoader(c, `Clusters: {"z1111": {}}`, nil).Load()
  56         c.Assert(err, check.IsNil)
  57         c.Assert(cfg.Clusters, check.HasLen, 1)
  58         cc, err := cfg.GetCluster("z1111")
  59         c.Assert(err, check.IsNil)
  60         c.Check(cc.ClusterID, check.Equals, "z1111")
  61         c.Check(cc.API.MaxRequestAmplification, check.Equals, 4)
  62         c.Check(cc.API.MaxItemsPerResponse, check.Equals, 1000)
  63 }
  64
  65 func (s *LoadSuite) TestMungeLegacyConfigArgs(c *check.C) {
  66         f, err := ioutil.TempFile("", "")
  67         c.Check(err, check.IsNil)
  68         defer os.Remove(f.Name())
  69         io.WriteString(f, "Debug: true\n")
  70         oldfile := f.Name()
  71
  72         f, err = ioutil.TempFile("", "")
  73         c.Check(err, check.IsNil)
  74         defer os.Remove(f.Name())
  75         io.WriteString(f, "Clusters: {aaaaa: {}}\n")
  76         newfile := f.Name()
  77
  78         for _, trial := range []struct {
  79                 argsIn  []string
  80                 argsOut []string
  81         }{
  82                 {
  83                         []string{"-config", oldfile},
  84                         []string{"-old-config", oldfile},
  85                 },
  86                 {
  87                         []string{"-config=" + oldfile},
  88                         []string{"-old-config=" + oldfile},
  89                 },
  90                 {
  91                         []string{"-config", newfile},
  92                         []string{"-config", newfile},
  93                 },
  94                 {
  95                         []string{"-config=" + newfile},
  96                         []string{"-config=" + newfile},
  97                 },
  98                 {
  99                         []string{"-foo", oldfile},
 100                         []string{"-foo", oldfile},
 101                 },
 102                 {
 103                         []string{"-foo=" + oldfile},
 104                         []string{"-foo=" + oldfile},
 105                 },
 106                 {
 107                         []string{"-foo", "-config=" + oldfile},
 108                         []string{"-foo", "-old-config=" + oldfile},
 109                 },
 110                 {
 111                         []string{"-foo", "bar", "-config=" + oldfile},
 112                         []string{"-foo", "bar", "-old-config=" + oldfile},
 113                 },
 114                 {
 115                         []string{"-foo=bar", "baz", "-config=" + oldfile},
 116                         []string{"-foo=bar", "baz", "-old-config=" + oldfile},
 117                 },
 118                 {
 119                         []string{"-config=/dev/null"},
 120                         []string{"-config=/dev/null"},
 121                 },
 122                 {
 123                         []string{"-config=-"},
 124                         []string{"-config=-"},
 125                 },
 126                 {
 127                         []string{"-config="},
 128                         []string{"-config="},
 129                 },
 130                 {
 131                         []string{"-foo=bar", "baz", "-config"},
 132                         []string{"-foo=bar", "baz", "-config"},
 133                 },
 134                 {
 135                         []string{},
 136                         nil,
 137                 },
 138         } {
 139                 var logbuf bytes.Buffer
 140                 logger := logrus.New()
 141                 logger.Out = &logbuf
 142
 143                 var ldr Loader
 144                 args := ldr.MungeLegacyConfigArgs(logger, trial.argsIn, "-old-config")
 145                 c.Check(args, check.DeepEquals, trial.argsOut)
 146                 if fmt.Sprintf("%v", trial.argsIn) != fmt.Sprintf("%v", trial.argsOut) {
 147                         c.Check(logbuf.String(), check.Matches, `.*`+oldfile+` is not a cluster config file -- interpreting -config as -old-config.*\n`)
 148                 }
 149         }
 150 }
 151
 152 func (s *LoadSuite) TestSampleKeys(c *check.C) {
 153         for _, yaml := range []string{
 154                 `{"Clusters":{"z1111":{}}}`,
 155                 `{"Clusters":{"z1111":{"InstanceTypes":{"Foo":{"RAM": "12345M"}}}}}`,
 156         } {
 157                 cfg, err := testLoader(c, yaml, nil).Load()
 158                 c.Assert(err, check.IsNil)
 159                 cc, err := cfg.GetCluster("z1111")
 160                 _, hasSample := cc.InstanceTypes["SAMPLE"]
 161                 c.Check(hasSample, check.Equals, false)
 162                 if strings.Contains(yaml, "Foo") {
 163                         c.Check(cc.InstanceTypes["Foo"].RAM, check.Equals, arvados.ByteSize(12345000000))
 164                         c.Check(cc.InstanceTypes["Foo"].Price, check.Equals, 0.0)
 165                 }
 166         }
 167 }
 168
 169 func (s *LoadSuite) TestMultipleClusters(c *check.C) {
 170         cfg, err := testLoader(c, `{"Clusters":{"z1111":{},"z2222":{}}}`, nil).Load()
 171         c.Assert(err, check.IsNil)
 172         c1, err := cfg.GetCluster("z1111")
 173         c.Assert(err, check.IsNil)
 174         c.Check(c1.ClusterID, check.Equals, "z1111")
 175         c2, err := cfg.GetCluster("z2222")
 176         c.Assert(err, check.IsNil)
 177         c.Check(c2.ClusterID, check.Equals, "z2222")
 178 }
 179
 180 func (s *LoadSuite) TestDeprecatedOrUnknownWarning(c *check.C) {
 181         var logbuf bytes.Buffer
 182         _, err := testLoader(c, `
 183 Clusters:
 184   zzzzz:
 185     postgresql: {}
 186     BadKey: {}
 187     Containers: {}
 188     RemoteClusters:
 189       z2222:
 190         Host: z2222.arvadosapi.com
 191         Proxy: true
 192         BadKey: badValue
 193 `, &logbuf).Load()
 194         c.Assert(err, check.IsNil)
 195         logs := strings.Split(strings.TrimSuffix(logbuf.String(), "\n"), "\n")
 196         for _, log := range logs {
 197                 c.Check(log, check.Matches, `.*deprecated or unknown config entry:.*BadKey.*`)
 198         }
 199         c.Check(logs, check.HasLen, 2)
 200 }
 201
 202 func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
 203         var logbuf bytes.Buffer
 204         var supplied map[string]interface{}
 205         yaml.Unmarshal(DefaultYAML, &supplied)
 206
 207         loader := testLoader(c, string(DefaultYAML), &logbuf)
 208         cfg, err := loader.Load()
 209         c.Assert(err, check.IsNil)
 210         var loaded map[string]interface{}
 211         buf, err := yaml.Marshal(cfg)
 212         c.Assert(err, check.IsNil)
 213         err = yaml.Unmarshal(buf, &loaded)
 214         c.Assert(err, check.IsNil)
 215
 216         loader.logExtraKeys(loaded, supplied, "")
 217         c.Check(logbuf.String(), check.Equals, "")
 218 }
 219
 220 func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
 221         var logbuf bytes.Buffer
 222         logger := logrus.New()
 223         logger.Out = &logbuf
 224         cfg, err := testLoader(c, `{"Clusters":{"zzzzz":{}}}`, &logbuf).Load()
 225         c.Assert(err, check.IsNil)
 226         yaml, err := yaml.Marshal(cfg)
 227         c.Assert(err, check.IsNil)
 228         cfgDumped, err := testLoader(c, string(yaml), &logbuf).Load()
 229         c.Assert(err, check.IsNil)
 230         c.Check(cfg, check.DeepEquals, cfgDumped)
 231         c.Check(logbuf.String(), check.Equals, "")
 232 }
 233
 234 func (s *LoadSuite) TestPostgreSQLKeyConflict(c *check.C) {
 235         _, err := testLoader(c, `
 236 Clusters:
 237  zzzzz:
 238   postgresql:
 239    connection:
 240      DBName: dbname
 241      Host: host
 242 `, nil).Load()
 243         c.Check(err, check.ErrorMatches, `Clusters.zzzzz.PostgreSQL.Connection: multiple entries for "(dbname|host)".*`)
 244 }
 245
 246 func (s *LoadSuite) TestBadType(c *check.C) {
 247         for _, data := range []string{`
 248 Clusters:
 249  zzzzz:
 250   PostgreSQL: true
 251 `, `
 252 Clusters:
 253  zzzzz:
 254   PostgreSQL:
 255    ConnectionPool: true
 256 `, `
 257 Clusters:
 258  zzzzz:
 259   PostgreSQL:
 260    ConnectionPool: "foo"
 261 `, `
 262 Clusters:
 263  zzzzz:
 264   PostgreSQL:
 265    ConnectionPool: []
 266 `, `
 267 Clusters:
 268  zzzzz:
 269   PostgreSQL:
 270    ConnectionPool: [] # {foo: bar} isn't caught here; we rely on config-check
 271 `,
 272         } {
 273                 c.Log(data)
 274                 v, err := testLoader(c, data, nil).Load()
 275                 if v != nil {
 276                         c.Logf("%#v", v.Clusters["zzzzz"].PostgreSQL.ConnectionPool)
 277                 }
 278                 c.Check(err, check.ErrorMatches, `.*cannot unmarshal .*PostgreSQL.*`)
 279         }
 280 }
 281
 282 func (s *LoadSuite) TestMovedKeys(c *check.C) {
 283         s.checkEquivalent(c, `# config has old keys only
 284 Clusters:
 285  zzzzz:
 286   RequestLimits:
 287    MultiClusterRequestConcurrency: 3
 288    MaxItemsPerResponse: 999
 289 `, `
 290 Clusters:
 291  zzzzz:
 292   API:
 293    MaxRequestAmplification: 3
 294    MaxItemsPerResponse: 999
 295 `)
 296         s.checkEquivalent(c, `# config has both old and new keys; old values win
 297 Clusters:
 298  zzzzz:
 299   RequestLimits:
 300    MultiClusterRequestConcurrency: 0
 301    MaxItemsPerResponse: 555
 302   API:
 303    MaxRequestAmplification: 3
 304    MaxItemsPerResponse: 999
 305 `, `
 306 Clusters:
 307  zzzzz:
 308   API:
 309    MaxRequestAmplification: 0
 310    MaxItemsPerResponse: 555
 311 `)
 312 }
 313
 314 func (s *LoadSuite) checkEquivalent(c *check.C, goty, expectedy string) {
 315         got, err := testLoader(c, goty, nil).Load()
 316         c.Assert(err, check.IsNil)
 317         expected, err := testLoader(c, expectedy, nil).Load()
 318         c.Assert(err, check.IsNil)
 319         if !c.Check(got, check.DeepEquals, expected) {
 320                 cmd := exec.Command("diff", "-u", "--label", "expected", "--label", "got", "/dev/fd/3", "/dev/fd/4")
 321                 for _, obj := range []interface{}{expected, got} {
 322                         y, _ := yaml.Marshal(obj)
 323                         pr, pw, err := os.Pipe()
 324                         c.Assert(err, check.IsNil)
 325                         defer pr.Close()
 326                         go func() {
 327                                 io.Copy(pw, bytes.NewBuffer(y))
 328                                 pw.Close()
 329                         }()
 330                         cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
 331                 }
 332                 diff, err := cmd.CombinedOutput()
 333                 c.Log(string(diff))
 334                 c.Check(err, check.IsNil)
 335         }
 336 }