18983: Warn if LocalKeepBlobBuffersPerVCPU > 0 but will not be used.

[arvados.git] / lib / config / load_test.go

diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 2d87b906c9b399f49f243d170984a31d712f6ed9..feb05cb9515bb03be84352d550c24bf82fe80320 100644 (file)
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -12,13 +12,18 @@ import (
        "os"
        "os/exec"
        "reflect"
+       "regexp"
        "strings"
        "testing"
+       "time"
 
        "git.arvados.org/arvados.git/sdk/go/arvados"
        "git.arvados.org/arvados.git/sdk/go/ctxlog"
        "github.com/ghodss/yaml"
+       "github.com/prometheus/client_golang/prometheus"
+       "github.com/prometheus/common/expfmt"
        "github.com/sirupsen/logrus"
+       "golang.org/x/sys/unix"
        check "gopkg.in/check.v1"
 )
 
@@ -232,7 +237,7 @@ Clusters:
         InternalURLs:
           "http://host.example:12345": {}
     Volumes:
-      zzzzz-nyw5e-aaaaaaaaaaaaaaa: {}
+      zzzzz-nyw5e-aaaaaaaaaaaaaaa: {Replication: 2}
 `, &logbuf).Load()
        c.Assert(err, check.IsNil)
        c.Log(logbuf.String())
@@ -311,12 +316,24 @@ Clusters:
   ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
   SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
   Collections:
-   BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, &logbuf).Load()
+   BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+  InstanceTypes:
+   abc:
+    IncludedScratch: 123456
+`, &logbuf).Load()
        c.Assert(err, check.IsNil)
        yaml, err := yaml.Marshal(cfg)
        c.Assert(err, check.IsNil)
+       // Well, *nearly* no warnings. SourceTimestamp and
+       // SourceSHA256 are included in a config-dump, but not
+       // expected in a real config file.
+       yaml = regexp.MustCompile(`(^|\n)(Source(Timestamp|SHA256): .*?\n)+`).ReplaceAll(yaml, []byte("$1"))
        cfgDumped, err := testLoader(c, string(yaml), &logbuf).Load()
        c.Assert(err, check.IsNil)
+       // SourceTimestamp and SourceSHA256 aren't expected to be
+       // preserved through dump+load
+       cfgDumped.SourceTimestamp = cfg.SourceTimestamp
+       cfgDumped.SourceSHA256 = cfg.SourceSHA256
        c.Check(cfg, check.DeepEquals, cfgDumped)
        c.Check(logbuf.String(), check.Equals, "")
 }
@@ -499,6 +516,12 @@ func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) {
        c.Assert(err, check.IsNil)
        expected, err := expectedldr.Load()
        c.Assert(err, check.IsNil)
+       // The inputs generally aren't even files, so SourceTimestamp
+       // can't be expected to match.
+       got.SourceTimestamp = expected.SourceTimestamp
+       // Obviously the content isn't identical -- otherwise we
+       // wouldn't need to check that it's equivalent.
+       got.SourceSHA256 = expected.SourceSHA256
        checkEqualYAML(c, got, expected)
 }
 
@@ -578,6 +601,31 @@ func (s *LoadSuite) TestListKeys(c *check.C) {
        }
 }
 
+func (s *LoadSuite) TestWarnUnusedLocalKeep(c *check.C) {
+       var logbuf bytes.Buffer
+       _, err := testLoader(c, `
+Clusters:
+ z1111:
+  Volumes:
+   z:
+    Replication: 1
+`, &logbuf).Load()
+       c.Assert(err, check.IsNil)
+       c.Check(logbuf.String(), check.Matches, `(?ms).*LocalKeepBlobBuffersPerVCPU is 1 but will not be used because at least one volume \(z\) has lower replication than DefaultReplication \(1 < 2\) -- suggest changing to 0.*`)
+
+       logbuf.Reset()
+       _, err = testLoader(c, `
+Clusters:
+ z1111:
+  Volumes:
+   z:
+    AccessViaHosts:
+     "http://0.0.0.0:12345": {}
+`, &logbuf).Load()
+       c.Assert(err, check.IsNil)
+       c.Check(logbuf.String(), check.Matches, `(?ms).*LocalKeepBlobBuffersPerVCPU is 1 but will not be used because at least one volume \(z\) uses AccessViaHosts -- suggest changing to 0.*`)
+}
+
 func (s *LoadSuite) TestImplicitStorageClasses(c *check.C) {
        // If StorageClasses and Volumes.*.StorageClasses are all
        // empty, there is a default storage class named "default".
@@ -758,3 +806,46 @@ Clusters:
        c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*Type2\.preemptible.*`)
        c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*(z1111|z2222)[^\n]*InstanceTypes.*`)
 }
+
+func (s *LoadSuite) TestSourceTimestamp(c *check.C) {
+       conftime, err := time.Parse(time.RFC3339, "2022-03-04T05:06:07-08:00")
+       c.Assert(err, check.IsNil)
+       confdata := `Clusters: {zzzzz: {}}`
+       conffile := c.MkDir() + "/config.yml"
+       ioutil.WriteFile(conffile, []byte(confdata), 0777)
+       tv := unix.NsecToTimeval(conftime.UnixNano())
+       unix.Lutimes(conffile, []unix.Timeval{tv, tv})
+       for _, trial := range []struct {
+               configarg  string
+               expectTime time.Time
+       }{
+               {"-", time.Now()},
+               {conffile, conftime},
+       } {
+               c.Logf("trial: %+v", trial)
+               ldr := NewLoader(strings.NewReader(confdata), ctxlog.TestLogger(c))
+               ldr.Path = trial.configarg
+               cfg, err := ldr.Load()
+               c.Assert(err, check.IsNil)
+               c.Check(cfg.SourceTimestamp, check.Equals, cfg.SourceTimestamp.UTC())
+               c.Check(cfg.SourceTimestamp, check.Equals, ldr.sourceTimestamp)
+               c.Check(int(cfg.SourceTimestamp.Sub(trial.expectTime).Seconds()), check.Equals, 0)
+               c.Check(int(ldr.loadTimestamp.Sub(time.Now()).Seconds()), check.Equals, 0)
+
+               var buf bytes.Buffer
+               reg := prometheus.NewRegistry()
+               ldr.RegisterMetrics(reg)
+               enc := expfmt.NewEncoder(&buf, expfmt.FmtText)
+               got, _ := reg.Gather()
+               for _, mf := range got {
+                       enc.Encode(mf)
+               }
+               c.Check(buf.String(), check.Matches, `# HELP .*
+# TYPE .*
+arvados_config_load_timestamp_seconds{sha256="83aea5d82eb1d53372cd65c936c60acc1c6ef946e61977bbca7cfea709d201a8"} \Q`+fmt.Sprintf("%g", float64(ldr.loadTimestamp.UnixNano())/1e9)+`\E
+# HELP .*
+# TYPE .*
+arvados_config_source_timestamp_seconds{sha256="83aea5d82eb1d53372cd65c936c60acc1c6ef946e61977bbca7cfea709d201a8"} \Q`+fmt.Sprintf("%g", float64(cfg.SourceTimestamp.UnixNano())/1e9)+`\E
+`)
+       }
+}