1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
18 "git.curoverse.com/arvados.git/sdk/go/arvados"
19 "git.curoverse.com/arvados.git/sdk/go/ctxlog"
20 "github.com/ghodss/yaml"
21 "github.com/sirupsen/logrus"
22 check "gopkg.in/check.v1"
25 // Gocheck boilerplate
26 func Test(t *testing.T) {
30 var _ = check.Suite(&LoadSuite{})
32 // Return a new Loader that reads cluster config from configdata
33 // (instead of the usual default /etc/arvados/config.yml), and logs to
34 // logdst or (if that's nil) c.Log.
35 func testLoader(c *check.C, configdata string, logdst io.Writer) *Loader {
36 logger := ctxlog.TestLogger(c)
42 ldr := NewLoader(bytes.NewBufferString(configdata), logger)
47 type LoadSuite struct{}
49 func (s *LoadSuite) TestEmpty(c *check.C) {
50 cfg, err := testLoader(c, "", nil).Load()
51 c.Check(cfg, check.IsNil)
52 c.Assert(err, check.ErrorMatches, `config does not define any clusters`)
55 func (s *LoadSuite) TestNoConfigs(c *check.C) {
56 cfg, err := testLoader(c, `Clusters: {"z1111": {}}`, nil).Load()
57 c.Assert(err, check.IsNil)
58 c.Assert(cfg.Clusters, check.HasLen, 1)
59 cc, err := cfg.GetCluster("z1111")
60 c.Assert(err, check.IsNil)
61 c.Check(cc.ClusterID, check.Equals, "z1111")
62 c.Check(cc.API.MaxRequestAmplification, check.Equals, 4)
63 c.Check(cc.API.MaxItemsPerResponse, check.Equals, 1000)
66 func (s *LoadSuite) TestMungeLegacyConfigArgs(c *check.C) {
67 f, err := ioutil.TempFile("", "")
68 c.Check(err, check.IsNil)
69 defer os.Remove(f.Name())
70 io.WriteString(f, "Debug: true\n")
73 f, err = ioutil.TempFile("", "")
74 c.Check(err, check.IsNil)
75 defer os.Remove(f.Name())
76 io.WriteString(f, "Clusters: {aaaaa: {}}\n")
79 for _, trial := range []struct {
84 []string{"-config", oldfile},
85 []string{"-old-config", oldfile},
88 []string{"-config=" + oldfile},
89 []string{"-old-config=" + oldfile},
92 []string{"-config", newfile},
93 []string{"-config", newfile},
96 []string{"-config=" + newfile},
97 []string{"-config=" + newfile},
100 []string{"-foo", oldfile},
101 []string{"-foo", oldfile},
104 []string{"-foo=" + oldfile},
105 []string{"-foo=" + oldfile},
108 []string{"-foo", "-config=" + oldfile},
109 []string{"-foo", "-old-config=" + oldfile},
112 []string{"-foo", "bar", "-config=" + oldfile},
113 []string{"-foo", "bar", "-old-config=" + oldfile},
116 []string{"-foo=bar", "baz", "-config=" + oldfile},
117 []string{"-foo=bar", "baz", "-old-config=" + oldfile},
120 []string{"-config=/dev/null"},
121 []string{"-config=/dev/null"},
124 []string{"-config=-"},
125 []string{"-config=-"},
128 []string{"-config="},
129 []string{"-config="},
132 []string{"-foo=bar", "baz", "-config"},
133 []string{"-foo=bar", "baz", "-config"},
140 var logbuf bytes.Buffer
141 logger := logrus.New()
145 args := ldr.MungeLegacyConfigArgs(logger, trial.argsIn, "-old-config")
146 c.Check(args, check.DeepEquals, trial.argsOut)
147 if fmt.Sprintf("%v", trial.argsIn) != fmt.Sprintf("%v", trial.argsOut) {
148 c.Check(logbuf.String(), check.Matches, `.*`+oldfile+` is not a cluster config file -- interpreting -config as -old-config.*\n`)
153 func (s *LoadSuite) TestSampleKeys(c *check.C) {
154 for _, yaml := range []string{
155 `{"Clusters":{"z1111":{}}}`,
156 `{"Clusters":{"z1111":{"InstanceTypes":{"Foo":{"RAM": "12345M"}}}}}`,
158 cfg, err := testLoader(c, yaml, nil).Load()
159 c.Assert(err, check.IsNil)
160 cc, err := cfg.GetCluster("z1111")
161 _, hasSample := cc.InstanceTypes["SAMPLE"]
162 c.Check(hasSample, check.Equals, false)
163 if strings.Contains(yaml, "Foo") {
164 c.Check(cc.InstanceTypes["Foo"].RAM, check.Equals, arvados.ByteSize(12345000000))
165 c.Check(cc.InstanceTypes["Foo"].Price, check.Equals, 0.0)
170 func (s *LoadSuite) TestMultipleClusters(c *check.C) {
171 cfg, err := testLoader(c, `{"Clusters":{"z1111":{},"z2222":{}}}`, nil).Load()
172 c.Assert(err, check.IsNil)
173 c1, err := cfg.GetCluster("z1111")
174 c.Assert(err, check.IsNil)
175 c.Check(c1.ClusterID, check.Equals, "z1111")
176 c2, err := cfg.GetCluster("z2222")
177 c.Assert(err, check.IsNil)
178 c.Check(c2.ClusterID, check.Equals, "z2222")
181 func (s *LoadSuite) TestDeprecatedOrUnknownWarning(c *check.C) {
182 var logbuf bytes.Buffer
183 _, err := testLoader(c, `
191 Host: z2222.arvadosapi.com
195 c.Assert(err, check.IsNil)
196 logs := strings.Split(strings.TrimSuffix(logbuf.String(), "\n"), "\n")
197 for _, log := range logs {
198 c.Check(log, check.Matches, `.*deprecated or unknown config entry:.*BadKey.*`)
200 c.Check(logs, check.HasLen, 2)
203 func (s *LoadSuite) checkSAMPLEKeys(c *check.C, path string, x interface{}) {
204 v := reflect.Indirect(reflect.ValueOf(x))
207 var stringKeys, sampleKey bool
211 if k.Kind() == reflect.String {
213 if k.String() == "SAMPLE" || k.String() == "xxxxx" {
215 s.checkSAMPLEKeys(c, path+"."+k.String(), iter.Value().Interface())
219 if stringKeys && !sampleKey {
220 c.Errorf("%s is a map with string keys (type %T) but config.default.yml has no SAMPLE key", path, x)
224 for i := 0; i < v.NumField(); i++ {
226 if val.CanInterface() {
227 s.checkSAMPLEKeys(c, path+"."+v.Type().Field(i).Name, val.Interface())
233 func (s *LoadSuite) TestDefaultConfigHasAllSAMPLEKeys(c *check.C) {
234 var logbuf bytes.Buffer
235 loader := testLoader(c, string(DefaultYAML), &logbuf)
236 cfg, err := loader.Load()
237 c.Assert(err, check.IsNil)
238 s.checkSAMPLEKeys(c, "", cfg)
241 func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
242 var logbuf bytes.Buffer
243 var supplied map[string]interface{}
244 yaml.Unmarshal(DefaultYAML, &supplied)
246 loader := testLoader(c, string(DefaultYAML), &logbuf)
247 cfg, err := loader.Load()
248 c.Assert(err, check.IsNil)
249 var loaded map[string]interface{}
250 buf, err := yaml.Marshal(cfg)
251 c.Assert(err, check.IsNil)
252 err = yaml.Unmarshal(buf, &loaded)
253 c.Assert(err, check.IsNil)
255 loader.logExtraKeys(loaded, supplied, "")
256 c.Check(logbuf.String(), check.Equals, "")
259 func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
260 var logbuf bytes.Buffer
261 logger := logrus.New()
263 cfg, err := testLoader(c, `{"Clusters":{"zzzzz":{}}}`, &logbuf).Load()
264 c.Assert(err, check.IsNil)
265 yaml, err := yaml.Marshal(cfg)
266 c.Assert(err, check.IsNil)
267 cfgDumped, err := testLoader(c, string(yaml), &logbuf).Load()
268 c.Assert(err, check.IsNil)
269 c.Check(cfg, check.DeepEquals, cfgDumped)
270 c.Check(logbuf.String(), check.Equals, "")
273 func (s *LoadSuite) TestPostgreSQLKeyConflict(c *check.C) {
274 _, err := testLoader(c, `
282 c.Check(err, check.ErrorMatches, `Clusters.zzzzz.PostgreSQL.Connection: multiple entries for "(dbname|host)".*`)
285 func (s *LoadSuite) TestBadType(c *check.C) {
286 for _, data := range []string{`
299 ConnectionPool: "foo"
309 ConnectionPool: [] # {foo: bar} isn't caught here; we rely on config-check
313 v, err := testLoader(c, data, nil).Load()
315 c.Logf("%#v", v.Clusters["zzzzz"].PostgreSQL.ConnectionPool)
317 c.Check(err, check.ErrorMatches, `.*cannot unmarshal .*PostgreSQL.*`)
321 func (s *LoadSuite) TestMovedKeys(c *check.C) {
322 s.checkEquivalent(c, `# config has old keys only
326 MultiClusterRequestConcurrency: 3
327 MaxItemsPerResponse: 999
332 MaxRequestAmplification: 3
333 MaxItemsPerResponse: 999
335 s.checkEquivalent(c, `# config has both old and new keys; old values win
339 MultiClusterRequestConcurrency: 0
340 MaxItemsPerResponse: 555
342 MaxRequestAmplification: 3
343 MaxItemsPerResponse: 999
348 MaxRequestAmplification: 0
349 MaxItemsPerResponse: 555
353 func (s *LoadSuite) checkEquivalent(c *check.C, goty, expectedy string) {
354 got, err := testLoader(c, goty, nil).Load()
355 c.Assert(err, check.IsNil)
356 expected, err := testLoader(c, expectedy, nil).Load()
357 c.Assert(err, check.IsNil)
358 if !c.Check(got, check.DeepEquals, expected) {
359 cmd := exec.Command("diff", "-u", "--label", "expected", "--label", "got", "/dev/fd/3", "/dev/fd/4")
360 for _, obj := range []interface{}{expected, got} {
361 y, _ := yaml.Marshal(obj)
362 pr, pw, err := os.Pipe()
363 c.Assert(err, check.IsNil)
366 io.Copy(pw, bytes.NewBuffer(y))
369 cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
371 diff, err := cmd.CombinedOutput()
373 c.Check(err, check.IsNil)