Merge branch '16265-security-updates' into dependabot/bundler/apps/workbench/loofah...
[arvados.git] / lib / config / load_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package config
6
7 import (
8         "bytes"
9         "fmt"
10         "io"
11         "io/ioutil"
12         "os"
13         "os/exec"
14         "reflect"
15         "strings"
16         "testing"
17
18         "git.arvados.org/arvados.git/sdk/go/arvados"
19         "git.arvados.org/arvados.git/sdk/go/ctxlog"
20         "github.com/ghodss/yaml"
21         "github.com/sirupsen/logrus"
22         check "gopkg.in/check.v1"
23 )
24
25 // Gocheck boilerplate
26 func Test(t *testing.T) {
27         check.TestingT(t)
28 }
29
30 var _ = check.Suite(&LoadSuite{})
31
32 // Return a new Loader that reads cluster config from configdata
33 // (instead of the usual default /etc/arvados/config.yml), and logs to
34 // logdst or (if that's nil) c.Log.
35 func testLoader(c *check.C, configdata string, logdst io.Writer) *Loader {
36         logger := ctxlog.TestLogger(c)
37         if logdst != nil {
38                 lgr := logrus.New()
39                 lgr.Out = logdst
40                 logger = lgr
41         }
42         ldr := NewLoader(bytes.NewBufferString(configdata), logger)
43         ldr.Path = "-"
44         return ldr
45 }
46
47 type LoadSuite struct{}
48
49 func (s *LoadSuite) SetUpSuite(c *check.C) {
50         os.Unsetenv("ARVADOS_API_HOST")
51         os.Unsetenv("ARVADOS_API_HOST_INSECURE")
52         os.Unsetenv("ARVADOS_API_TOKEN")
53 }
54
55 func (s *LoadSuite) TestEmpty(c *check.C) {
56         cfg, err := testLoader(c, "", nil).Load()
57         c.Check(cfg, check.IsNil)
58         c.Assert(err, check.ErrorMatches, `config does not define any clusters`)
59 }
60
61 func (s *LoadSuite) TestNoConfigs(c *check.C) {
62         cfg, err := testLoader(c, `Clusters: {"z1111": {}}`, nil).Load()
63         c.Assert(err, check.IsNil)
64         c.Assert(cfg.Clusters, check.HasLen, 1)
65         cc, err := cfg.GetCluster("z1111")
66         c.Assert(err, check.IsNil)
67         c.Check(cc.ClusterID, check.Equals, "z1111")
68         c.Check(cc.API.MaxRequestAmplification, check.Equals, 4)
69         c.Check(cc.API.MaxItemsPerResponse, check.Equals, 1000)
70 }
71
72 func (s *LoadSuite) TestMungeLegacyConfigArgs(c *check.C) {
73         f, err := ioutil.TempFile("", "")
74         c.Check(err, check.IsNil)
75         defer os.Remove(f.Name())
76         io.WriteString(f, "Debug: true\n")
77         oldfile := f.Name()
78
79         f, err = ioutil.TempFile("", "")
80         c.Check(err, check.IsNil)
81         defer os.Remove(f.Name())
82         io.WriteString(f, "Clusters: {aaaaa: {}}\n")
83         newfile := f.Name()
84
85         for _, trial := range []struct {
86                 argsIn  []string
87                 argsOut []string
88         }{
89                 {
90                         []string{"-config", oldfile},
91                         []string{"-old-config", oldfile},
92                 },
93                 {
94                         []string{"-config=" + oldfile},
95                         []string{"-old-config=" + oldfile},
96                 },
97                 {
98                         []string{"-config", newfile},
99                         []string{"-config", newfile},
100                 },
101                 {
102                         []string{"-config=" + newfile},
103                         []string{"-config=" + newfile},
104                 },
105                 {
106                         []string{"-foo", oldfile},
107                         []string{"-foo", oldfile},
108                 },
109                 {
110                         []string{"-foo=" + oldfile},
111                         []string{"-foo=" + oldfile},
112                 },
113                 {
114                         []string{"-foo", "-config=" + oldfile},
115                         []string{"-foo", "-old-config=" + oldfile},
116                 },
117                 {
118                         []string{"-foo", "bar", "-config=" + oldfile},
119                         []string{"-foo", "bar", "-old-config=" + oldfile},
120                 },
121                 {
122                         []string{"-foo=bar", "baz", "-config=" + oldfile},
123                         []string{"-foo=bar", "baz", "-old-config=" + oldfile},
124                 },
125                 {
126                         []string{"-config=/dev/null"},
127                         []string{"-config=/dev/null"},
128                 },
129                 {
130                         []string{"-config=-"},
131                         []string{"-config=-"},
132                 },
133                 {
134                         []string{"-config="},
135                         []string{"-config="},
136                 },
137                 {
138                         []string{"-foo=bar", "baz", "-config"},
139                         []string{"-foo=bar", "baz", "-config"},
140                 },
141                 {
142                         []string{},
143                         nil,
144                 },
145         } {
146                 var logbuf bytes.Buffer
147                 logger := logrus.New()
148                 logger.Out = &logbuf
149
150                 var ldr Loader
151                 args := ldr.MungeLegacyConfigArgs(logger, trial.argsIn, "-old-config")
152                 c.Check(args, check.DeepEquals, trial.argsOut)
153                 if fmt.Sprintf("%v", trial.argsIn) != fmt.Sprintf("%v", trial.argsOut) {
154                         c.Check(logbuf.String(), check.Matches, `.*`+oldfile+` is not a cluster config file -- interpreting -config as -old-config.*\n`)
155                 }
156         }
157 }
158
159 func (s *LoadSuite) TestSampleKeys(c *check.C) {
160         for _, yaml := range []string{
161                 `{"Clusters":{"z1111":{}}}`,
162                 `{"Clusters":{"z1111":{"InstanceTypes":{"Foo":{"RAM": "12345M"}}}}}`,
163         } {
164                 cfg, err := testLoader(c, yaml, nil).Load()
165                 c.Assert(err, check.IsNil)
166                 cc, err := cfg.GetCluster("z1111")
167                 _, hasSample := cc.InstanceTypes["SAMPLE"]
168                 c.Check(hasSample, check.Equals, false)
169                 if strings.Contains(yaml, "Foo") {
170                         c.Check(cc.InstanceTypes["Foo"].RAM, check.Equals, arvados.ByteSize(12345000000))
171                         c.Check(cc.InstanceTypes["Foo"].Price, check.Equals, 0.0)
172                 }
173         }
174 }
175
176 func (s *LoadSuite) TestMultipleClusters(c *check.C) {
177         ldr := testLoader(c, `{"Clusters":{"z1111":{},"z2222":{}}}`, nil)
178         ldr.SkipDeprecated = true
179         cfg, err := ldr.Load()
180         c.Assert(err, check.IsNil)
181         c1, err := cfg.GetCluster("z1111")
182         c.Assert(err, check.IsNil)
183         c.Check(c1.ClusterID, check.Equals, "z1111")
184         c2, err := cfg.GetCluster("z2222")
185         c.Assert(err, check.IsNil)
186         c.Check(c2.ClusterID, check.Equals, "z2222")
187 }
188
189 func (s *LoadSuite) TestDeprecatedOrUnknownWarning(c *check.C) {
190         var logbuf bytes.Buffer
191         _, err := testLoader(c, `
192 Clusters:
193   zzzzz:
194     postgresql: {}
195     BadKey: {}
196     Containers: {}
197     RemoteClusters:
198       z2222:
199         Host: z2222.arvadosapi.com
200         Proxy: true
201         BadKey: badValue
202 `, &logbuf).Load()
203         c.Assert(err, check.IsNil)
204         logs := strings.Split(strings.TrimSuffix(logbuf.String(), "\n"), "\n")
205         for _, log := range logs {
206                 c.Check(log, check.Matches, `.*deprecated or unknown config entry:.*BadKey.*`)
207         }
208         c.Check(logs, check.HasLen, 2)
209 }
210
211 func (s *LoadSuite) checkSAMPLEKeys(c *check.C, path string, x interface{}) {
212         v := reflect.Indirect(reflect.ValueOf(x))
213         switch v.Kind() {
214         case reflect.Map:
215                 var stringKeys, sampleKey bool
216                 iter := v.MapRange()
217                 for iter.Next() {
218                         k := iter.Key()
219                         if k.Kind() == reflect.String {
220                                 stringKeys = true
221                                 if k.String() == "SAMPLE" || k.String() == "xxxxx" {
222                                         sampleKey = true
223                                         s.checkSAMPLEKeys(c, path+"."+k.String(), iter.Value().Interface())
224                                 }
225                         }
226                 }
227                 if stringKeys && !sampleKey {
228                         c.Errorf("%s is a map with string keys (type %T) but config.default.yml has no SAMPLE key", path, x)
229                 }
230                 return
231         case reflect.Struct:
232                 for i := 0; i < v.NumField(); i++ {
233                         val := v.Field(i)
234                         if val.CanInterface() {
235                                 s.checkSAMPLEKeys(c, path+"."+v.Type().Field(i).Name, val.Interface())
236                         }
237                 }
238         }
239 }
240
241 func (s *LoadSuite) TestDefaultConfigHasAllSAMPLEKeys(c *check.C) {
242         var logbuf bytes.Buffer
243         loader := testLoader(c, string(DefaultYAML), &logbuf)
244         cfg, err := loader.Load()
245         c.Assert(err, check.IsNil)
246         s.checkSAMPLEKeys(c, "", cfg)
247 }
248
249 func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
250         var logbuf bytes.Buffer
251         var supplied map[string]interface{}
252         yaml.Unmarshal(DefaultYAML, &supplied)
253
254         loader := testLoader(c, string(DefaultYAML), &logbuf)
255         cfg, err := loader.Load()
256         c.Assert(err, check.IsNil)
257         var loaded map[string]interface{}
258         buf, err := yaml.Marshal(cfg)
259         c.Assert(err, check.IsNil)
260         err = yaml.Unmarshal(buf, &loaded)
261         c.Assert(err, check.IsNil)
262
263         loader.logExtraKeys(loaded, supplied, "")
264         c.Check(logbuf.String(), check.Equals, "")
265 }
266
267 func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
268         var logbuf bytes.Buffer
269         logger := logrus.New()
270         logger.Out = &logbuf
271         cfg, err := testLoader(c, `{"Clusters":{"zzzzz":{}}}`, &logbuf).Load()
272         c.Assert(err, check.IsNil)
273         yaml, err := yaml.Marshal(cfg)
274         c.Assert(err, check.IsNil)
275         cfgDumped, err := testLoader(c, string(yaml), &logbuf).Load()
276         c.Assert(err, check.IsNil)
277         c.Check(cfg, check.DeepEquals, cfgDumped)
278         c.Check(logbuf.String(), check.Equals, "")
279 }
280
281 func (s *LoadSuite) TestPostgreSQLKeyConflict(c *check.C) {
282         _, err := testLoader(c, `
283 Clusters:
284  zzzzz:
285   postgresql:
286    connection:
287      DBName: dbname
288      Host: host
289 `, nil).Load()
290         c.Check(err, check.ErrorMatches, `Clusters.zzzzz.PostgreSQL.Connection: multiple entries for "(dbname|host)".*`)
291 }
292
293 func (s *LoadSuite) TestBadType(c *check.C) {
294         for _, data := range []string{`
295 Clusters:
296  zzzzz:
297   PostgreSQL: true
298 `, `
299 Clusters:
300  zzzzz:
301   PostgreSQL:
302    ConnectionPool: true
303 `, `
304 Clusters:
305  zzzzz:
306   PostgreSQL:
307    ConnectionPool: "foo"
308 `, `
309 Clusters:
310  zzzzz:
311   PostgreSQL:
312    ConnectionPool: []
313 `, `
314 Clusters:
315  zzzzz:
316   PostgreSQL:
317    ConnectionPool: [] # {foo: bar} isn't caught here; we rely on config-check
318 `,
319         } {
320                 c.Log(data)
321                 v, err := testLoader(c, data, nil).Load()
322                 if v != nil {
323                         c.Logf("%#v", v.Clusters["zzzzz"].PostgreSQL.ConnectionPool)
324                 }
325                 c.Check(err, check.ErrorMatches, `.*cannot unmarshal .*PostgreSQL.*`)
326         }
327 }
328
329 func (s *LoadSuite) TestMovedKeys(c *check.C) {
330         checkEquivalent(c, `# config has old keys only
331 Clusters:
332  zzzzz:
333   RequestLimits:
334    MultiClusterRequestConcurrency: 3
335    MaxItemsPerResponse: 999
336 `, `
337 Clusters:
338  zzzzz:
339   API:
340    MaxRequestAmplification: 3
341    MaxItemsPerResponse: 999
342 `)
343         checkEquivalent(c, `# config has both old and new keys; old values win
344 Clusters:
345  zzzzz:
346   RequestLimits:
347    MultiClusterRequestConcurrency: 0
348    MaxItemsPerResponse: 555
349   API:
350    MaxRequestAmplification: 3
351    MaxItemsPerResponse: 999
352 `, `
353 Clusters:
354  zzzzz:
355   API:
356    MaxRequestAmplification: 0
357    MaxItemsPerResponse: 555
358 `)
359 }
360
361 func checkEquivalent(c *check.C, goty, expectedy string) {
362         gotldr := testLoader(c, goty, nil)
363         expectedldr := testLoader(c, expectedy, nil)
364         checkEquivalentLoaders(c, gotldr, expectedldr)
365 }
366
367 func checkEqualYAML(c *check.C, got, expected interface{}) {
368         expectedyaml, err := yaml.Marshal(expected)
369         c.Assert(err, check.IsNil)
370         gotyaml, err := yaml.Marshal(got)
371         c.Assert(err, check.IsNil)
372         if !bytes.Equal(gotyaml, expectedyaml) {
373                 cmd := exec.Command("diff", "-u", "--label", "expected", "--label", "got", "/dev/fd/3", "/dev/fd/4")
374                 for _, y := range [][]byte{expectedyaml, gotyaml} {
375                         pr, pw, err := os.Pipe()
376                         c.Assert(err, check.IsNil)
377                         defer pr.Close()
378                         go func(data []byte) {
379                                 pw.Write(data)
380                                 pw.Close()
381                         }(y)
382                         cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
383                 }
384                 diff, err := cmd.CombinedOutput()
385                 // diff should report differences and exit non-zero.
386                 c.Check(err, check.NotNil)
387                 c.Log(string(diff))
388                 c.Error("got != expected; see diff (-expected +got) above")
389         }
390 }
391
392 func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) {
393         got, err := gotldr.Load()
394         c.Assert(err, check.IsNil)
395         expected, err := expectedldr.Load()
396         c.Assert(err, check.IsNil)
397         checkEqualYAML(c, got, expected)
398 }
399
400 func checkListKeys(path string, x interface{}) (err error) {
401         v := reflect.Indirect(reflect.ValueOf(x))
402         switch v.Kind() {
403         case reflect.Map:
404                 iter := v.MapRange()
405                 for iter.Next() {
406                         k := iter.Key()
407                         if k.Kind() == reflect.String {
408                                 if err = checkListKeys(path+"."+k.String(), iter.Value().Interface()); err != nil {
409                                         return
410                                 }
411                         }
412                 }
413                 return
414
415         case reflect.Struct:
416                 for i := 0; i < v.NumField(); i++ {
417                         val := v.Field(i)
418                         structField := v.Type().Field(i)
419                         fieldname := structField.Name
420                         endsWithList := strings.HasSuffix(fieldname, "List")
421                         isAnArray := structField.Type.Kind() == reflect.Slice
422                         if endsWithList != isAnArray {
423                                 if endsWithList {
424                                         err = fmt.Errorf("%s.%s ends with 'List' but field is not an array (type %v)", path, fieldname, val.Kind())
425                                         return
426                                 }
427                                 if isAnArray && structField.Type.Elem().Kind() != reflect.Uint8 {
428                                         err = fmt.Errorf("%s.%s is an array but field name does not end in 'List' (slice of %v)", path, fieldname, structField.Type.Elem().Kind())
429                                         return
430                                 }
431                         }
432                         if val.CanInterface() {
433                                 checkListKeys(path+"."+fieldname, val.Interface())
434                         }
435                 }
436         }
437         return
438 }
439
440 func (s *LoadSuite) TestListKeys(c *check.C) {
441         v1 := struct {
442                 EndInList []string
443         }{[]string{"a", "b"}}
444         var m1 = make(map[string]interface{})
445         m1["c"] = &v1
446         if err := checkListKeys("", m1); err != nil {
447                 c.Error(err)
448         }
449
450         v2 := struct {
451                 DoesNot []string
452         }{[]string{"a", "b"}}
453         var m2 = make(map[string]interface{})
454         m2["c"] = &v2
455         if err := checkListKeys("", m2); err == nil {
456                 c.Errorf("Should have produced an error")
457         }
458
459         v3 := struct {
460                 EndInList string
461         }{"a"}
462         var m3 = make(map[string]interface{})
463         m3["c"] = &v3
464         if err := checkListKeys("", m3); err == nil {
465                 c.Errorf("Should have produced an error")
466         }
467
468         var logbuf bytes.Buffer
469         loader := testLoader(c, string(DefaultYAML), &logbuf)
470         cfg, err := loader.Load()
471         c.Assert(err, check.IsNil)
472         if err := checkListKeys("", cfg); err != nil {
473                 c.Error(err)
474         }
475 }