Merge branch '16997-sort-config-for-diff'

fixes #16997

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/go.mod b/go.mod
index 88dcb86c72e35cafcf61905f7d67ebb554026b0d..191d3aa5e5e51b9352b04ac08005e206ed04f5db 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -78,3 +78,4 @@ require (
 )
 
 replace github.com/AdRoll/goamz => github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef
+replace gopkg.in/yaml.v2 => github.com/arvados/yaml v0.0.0-20210427145106-92a1cab0904b

diff --git a/go.sum b/go.sum
index 91b5689eb33424ce130f8c06400c74da002e585e..43645178d7e167eb258f7843d58bf87b18cf68b6 100644 (file)
--- a/go.sum
+++ b/go.sum
@@ -51,6 +51,8 @@ github.com/arvados/cgofuse v1.2.0-arvados1 h1:4Q4vRJ4hbTCcI4gGEaa6hqwj3rqlUuzeFQ
 github.com/arvados/cgofuse v1.2.0-arvados1/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
 github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef h1:cl7DIRbiAYNqaVxg3CZY8qfZoBOKrj06H/x9SPGaxas=
 github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef/go.mod h1:rCtgyMmBGEbjTm37fCuBYbNL0IhztiALzo3OB9HyiOM=
+github.com/arvados/yaml v0.0.0-20210427145106-92a1cab0904b h1:hK0t0aJTTXI64lpXln2A1SripqOym+GVNTnwsLes39Y=
+github.com/arvados/yaml v0.0.0-20210427145106-92a1cab0904b/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.25.30 h1:I9qj6zW3mMfsg91e+GMSN/INcaX9tTFvr/l/BAHKaIY=
 github.com/aws/aws-sdk-go v1.25.30/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=

diff --git a/lib/config/cmd.go b/lib/config/cmd.go
index 347e8519a9717dff33eaefee1a3ed2570a4d013c..8e638e6ecb4cc712a7ba08f68c09ea2c33b189fc 100644 (file)
--- a/lib/config/cmd.go
+++ b/lib/config/cmd.go
@@ -12,7 +12,6 @@ import (
        "os"
        "os/exec"
 
-       "git.arvados.org/arvados.git/sdk/go/arvados"
        "git.arvados.org/arvados.git/sdk/go/ctxlog"
        "github.com/ghodss/yaml"
        "github.com/sirupsen/logrus"
@@ -120,16 +119,15 @@ func (checkCommand) RunCommand(prog string, args []string, stdin io.Reader, stdo
        if err != nil {
                return 1
        }
+       // Reset() to avoid printing the same warnings twice when they
+       // are logged by both without-legacy and with-legacy loads.
+       logbuf.Reset()
        loader.SkipDeprecated = false
        loader.SkipLegacy = false
        withDepr, err := loader.Load()
        if err != nil {
                return 1
        }
-       problems := false
-       if warnAboutProblems(logger, withDepr) {
-               problems = true
-       }
        cmd := exec.Command("diff", "-u", "--label", "without-deprecated-configs", "--label", "relying-on-deprecated-configs", "/dev/fd/3", "/dev/fd/4")
        for _, obj := range []interface{}{withoutDepr, withDepr} {
                y, _ := yaml.Marshal(obj)
@@ -165,28 +163,9 @@ func (checkCommand) RunCommand(prog string, args []string, stdin io.Reader, stdo
                        return 1
                }
        }
-
-       if problems {
-               return 1
-       }
        return 0
 }
 
-func warnAboutProblems(logger logrus.FieldLogger, cfg *arvados.Config) bool {
-       warned := false
-       for id, cc := range cfg.Clusters {
-               if cc.SystemRootToken == "" {
-                       logger.Warnf("Clusters.%s.SystemRootToken is empty; see https://doc.arvados.org/master/install/install-keepstore.html", id)
-                       warned = true
-               }
-               if cc.ManagementToken == "" {
-                       logger.Warnf("Clusters.%s.ManagementToken is empty; see https://doc.arvados.org/admin/management-token.html", id)
-                       warned = true
-               }
-       }
-       return warned
-}
-
 var DumpDefaultsCommand defaultsCommand
 
 type defaultsCommand struct{}

diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index bb8d7dca10236d6d80523a576e0a377040b95c27..241f37683449829169f3e78ecfc5879037ce90ca 100644 (file)
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -134,6 +134,18 @@ Clusters:
        c.Check(stderr.String(), check.Matches, `(?ms).*unexpected object in config entry: Clusters.z1234.PostgreSQL.ConnectionPool"\n.*`)
 }
 
+func (s *CommandSuite) TestCheck_DuplicateWarnings(c *check.C) {
+       var stdout, stderr bytes.Buffer
+       in := `
+Clusters:
+ z1234: {}
+`
+       code := CheckCommand.RunCommand("arvados config-check", []string{"-config", "-"}, bytes.NewBufferString(in), &stdout, &stderr)
+       c.Check(code, check.Equals, 1)
+       c.Check(stderr.String(), check.Matches, `(?ms).*SystemRootToken.*`)
+       c.Check(stderr.String(), check.Not(check.Matches), `(?ms).*SystemRootToken.*SystemRootToken.*`)
+}
+
 func (s *CommandSuite) TestDump_Formatting(c *check.C) {
        var stdout, stderr bytes.Buffer
        in := `
@@ -168,3 +180,54 @@ Clusters:
        c.Check(stdout.String(), check.Matches, `(?ms).*\n *ManagementToken: secret\n.*`)
        c.Check(stdout.String(), check.Not(check.Matches), `(?ms).*UnknownKey.*`)
 }
+
+func (s *CommandSuite) TestDump_KeyOrder(c *check.C) {
+       in := `
+Clusters:
+ z1234:
+  Login:
+   Test:
+    Users:
+     a: {}
+     d: {}
+     c: {}
+     b: {}
+     e: {}
+`
+       for trial := 0; trial < 20; trial++ {
+               var stdout, stderr bytes.Buffer
+               code := DumpCommand.RunCommand("arvados config-dump", []string{"-config", "-"}, bytes.NewBufferString(in), &stdout, &stderr)
+               c.Assert(code, check.Equals, 0)
+               if !c.Check(stdout.String(), check.Matches, `(?ms).*a:.*b:.*c:.*d:.*e:.*`) {
+                       c.Logf("config-dump did not use lexical key order on trial %d", trial)
+                       c.Log("stdout:\n", stdout.String())
+                       c.Log("stderr:\n", stderr.String())
+                       c.FailNow()
+               }
+       }
+}
+
+func (s *CommandSuite) TestCheck_KeyOrder(c *check.C) {
+       in := `
+Clusters:
+ z1234:
+  ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+  SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+  Collections:
+   BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+  InstanceTypes:
+   a32a: {}
+   a48a: {}
+   a4a: {}
+`
+       for trial := 0; trial < 20; trial++ {
+               var stdout, stderr bytes.Buffer
+               code := CheckCommand.RunCommand("arvados config-check", []string{"-config=-", "-strict=true"}, bytes.NewBufferString(in), &stdout, &stderr)
+               if !c.Check(code, check.Equals, 0) || stdout.String() != "" || stderr.String() != "" {
+                       c.Logf("config-check returned error or non-empty output on trial %d", trial)
+                       c.Log("stdout:\n", stdout.String())
+                       c.Log("stderr:\n", stderr.String())
+                       c.FailNow()
+               }
+       }
+}

diff --git a/lib/config/load.go b/lib/config/load.go
index b42e1a3c1e7b3f54734754d2727b00a3c8bdfeed..292e3f6d6f39c107073ef9236585ae77764945f9 100644 (file)
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -313,11 +313,15 @@ var acceptableTokenLength = 32
 
 func (ldr *Loader) checkToken(label, token string) error {
        if token == "" {
-               ldr.Logger.Warnf("%s: secret token is not set (use %d+ random characters from a-z, A-Z, 0-9)", label, acceptableTokenLength)
+               if ldr.Logger != nil {
+                       ldr.Logger.Warnf("%s: secret token is not set (use %d+ random characters from a-z, A-Z, 0-9)", label, acceptableTokenLength)
+               }
        } else if !acceptableTokenRe.MatchString(token) {
                return fmt.Errorf("%s: unacceptable characters in token (only a-z, A-Z, 0-9 are acceptable)", label)
        } else if len(token) < acceptableTokenLength {
-               ldr.Logger.Warnf("%s: token is too short (should be at least %d characters)", label, acceptableTokenLength)
+               if ldr.Logger != nil {
+                       ldr.Logger.Warnf("%s: token is too short (should be at least %d characters)", label, acceptableTokenLength)
+               }
        }
        return nil
 }