From cbeb2a6dc91804addea68705579db7ed1d53458a Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Fri, 8 Apr 2022 11:08:14 -0400 Subject: [PATCH] 18794: Add SourceTimestamp and SourceSHA256 to config-dump output. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- lib/config/load.go | 24 ++++++++++++++---- lib/config/load_test.go | 42 ++++++++++++++++++++++++++++++++ sdk/go/arvados/config.go | 3 +++ sdk/go/health/aggregator_test.go | 2 ++ 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/lib/config/load.go b/lib/config/load.go index 5afb51c5ad..28f300e3bb 100644 --- a/lib/config/load.go +++ b/lib/config/load.go @@ -17,6 +17,7 @@ import ( "regexp" "strconv" "strings" + "time" "git.arvados.org/arvados.git/sdk/go/arvados" "github.com/ghodss/yaml" @@ -46,6 +47,10 @@ type Loader struct { KeepBalancePath string configdata []byte + // UTC time for configdata: either the modtime of the file we + // read configdata from, or the time when we read configdata + // from a pipe. + sourceTimestamp time.Time } // NewLoader returns a new Loader with Stdin and Logger set to the @@ -166,25 +171,32 @@ func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string, return munged } -func (ldr *Loader) loadBytes(path string) ([]byte, error) { +func (ldr *Loader) loadBytes(path string) ([]byte, time.Time, error) { if path == "-" { - return ioutil.ReadAll(ldr.Stdin) + buf, err := ioutil.ReadAll(ldr.Stdin) + return buf, time.Now().UTC(), err } f, err := os.Open(path) if err != nil { - return nil, err + return nil, time.Time{}, err } defer f.Close() - return ioutil.ReadAll(f) + fi, err := f.Stat() + if err != nil { + return nil, time.Time{}, err + } + buf, err := ioutil.ReadAll(f) + return buf, fi.ModTime().UTC(), err } func (ldr *Loader) Load() (*arvados.Config, error) { if ldr.configdata == nil { - buf, err := ldr.loadBytes(ldr.Path) + buf, t, err := ldr.loadBytes(ldr.Path) if err != nil { return nil, err } ldr.configdata = buf + ldr.sourceTimestamp = t } // FIXME: We should reject YAML if the same key is used twice @@ -330,6 +342,8 @@ func (ldr *Loader) Load() (*arvados.Config, error) { } } } + cfg.SourceTimestamp = ldr.sourceTimestamp + cfg.SourceSHA256 = fmt.Sprintf("%x", sha256.Sum256(ldr.configdata)) return &cfg, nil } diff --git a/lib/config/load_test.go b/lib/config/load_test.go index 2d87b906c9..cef7ea944b 100644 --- a/lib/config/load_test.go +++ b/lib/config/load_test.go @@ -12,13 +12,16 @@ import ( "os" "os/exec" "reflect" + "regexp" "strings" "testing" + "time" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" "github.com/ghodss/yaml" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" check "gopkg.in/check.v1" ) @@ -315,8 +318,16 @@ Clusters: c.Assert(err, check.IsNil) yaml, err := yaml.Marshal(cfg) c.Assert(err, check.IsNil) + // Well, *nearly* no warnings. SourceTimestamp and + // SourceSHA256 are included in a config-dump, but not + // expected in a real config file. + yaml = regexp.MustCompile(`(^|\n)(Source(Timestamp|SHA256): .*?\n)+`).ReplaceAll(yaml, []byte("$1")) cfgDumped, err := testLoader(c, string(yaml), &logbuf).Load() c.Assert(err, check.IsNil) + // SourceTimestamp and SourceSHA256 aren't expected to be + // preserved through dump+load + cfgDumped.SourceTimestamp = cfg.SourceTimestamp + cfgDumped.SourceSHA256 = cfg.SourceSHA256 c.Check(cfg, check.DeepEquals, cfgDumped) c.Check(logbuf.String(), check.Equals, "") } @@ -499,6 +510,12 @@ func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) { c.Assert(err, check.IsNil) expected, err := expectedldr.Load() c.Assert(err, check.IsNil) + // The inputs generally aren't even files, so SourceTimestamp + // can't be expected to match. + got.SourceTimestamp = expected.SourceTimestamp + // Obviously the content isn't identical -- otherwise we + // wouldn't need to check that it's equivalent. + got.SourceSHA256 = expected.SourceSHA256 checkEqualYAML(c, got, expected) } @@ -758,3 +775,28 @@ Clusters: c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*Type2\.preemptible.*`) c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*(z1111|z2222)[^\n]*InstanceTypes.*`) } + +func (s *LoadSuite) TestSourceTimestamp(c *check.C) { + conftime, err := time.Parse(time.RFC3339, "2022-03-04T05:06:07-08:00") + c.Assert(err, check.IsNil) + confdata := `Clusters: {zzzzz: {}}` + conffile := c.MkDir() + "/config.yml" + ioutil.WriteFile(conffile, []byte(confdata), 0777) + tv := unix.NsecToTimeval(conftime.UnixNano()) + unix.Lutimes(conffile, []unix.Timeval{tv, tv}) + for _, trial := range []struct { + configarg string + expectTime time.Time + }{ + {"-", time.Now()}, + {conffile, conftime}, + } { + c.Logf("trial: %+v", trial) + ldr := NewLoader(strings.NewReader(confdata), ctxlog.TestLogger(c)) + ldr.Path = trial.configarg + cfg, err := ldr.Load() + c.Assert(err, check.IsNil) + c.Check(cfg.SourceTimestamp, check.Equals, cfg.SourceTimestamp.UTC()) + c.Check(int(cfg.SourceTimestamp.Sub(trial.expectTime).Seconds()), check.Equals, 0) + } +} diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 6c9324e478..b508a3f05d 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -10,6 +10,7 @@ import ( "fmt" "net/url" "os" + "time" "git.arvados.org/arvados.git/sdk/go/config" ) @@ -24,6 +25,8 @@ var DefaultConfigFile = func() string { type Config struct { Clusters map[string]Cluster AutoReloadConfig bool + SourceTimestamp time.Time + SourceSHA256 string } // GetConfig returns the current system config, loading it from diff --git a/sdk/go/health/aggregator_test.go b/sdk/go/health/aggregator_test.go index 05f0bdd31b..62eca894b7 100644 --- a/sdk/go/health/aggregator_test.go +++ b/sdk/go/health/aggregator_test.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "regexp" "strings" "time" @@ -142,6 +143,7 @@ func (s *AggregatorSuite) TestCheckCommand(c *check.C) { tmpdir := c.MkDir() confdata, err := yaml.Marshal(arvados.Config{Clusters: map[string]arvados.Cluster{s.handler.Cluster.ClusterID: *s.handler.Cluster}}) c.Assert(err, check.IsNil) + confdata = regexp.MustCompile(`SourceTimestamp: [^\n]+\n`).ReplaceAll(confdata, []byte{}) err = ioutil.WriteFile(tmpdir+"/config.yml", confdata, 0777) c.Assert(err, check.IsNil) var stdout, stderr bytes.Buffer -- 2.30.2