18794: Add SourceTimestamp and SourceSHA256 to config-dump output.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/lib/config/load.go b/lib/config/load.go
index 5afb51c5adcfd6dce89d069670b9ce3aadde2a2e..28f300e3bb632596628ec8cd8e04c154344b7698 100644 (file)
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -17,6 +17,7 @@ import (
        "regexp"
        "strconv"
        "strings"
+       "time"
 
        "git.arvados.org/arvados.git/sdk/go/arvados"
        "github.com/ghodss/yaml"
@@ -46,6 +47,10 @@ type Loader struct {
        KeepBalancePath         string
 
        configdata []byte
+       // UTC time for configdata: either the modtime of the file we
+       // read configdata from, or the time when we read configdata
+       // from a pipe.
+       sourceTimestamp time.Time
 }
 
 // NewLoader returns a new Loader with Stdin and Logger set to the
@@ -166,25 +171,32 @@ func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string,
        return munged
 }
 
-func (ldr *Loader) loadBytes(path string) ([]byte, error) {
+func (ldr *Loader) loadBytes(path string) ([]byte, time.Time, error) {
        if path == "-" {
-               return ioutil.ReadAll(ldr.Stdin)
+               buf, err := ioutil.ReadAll(ldr.Stdin)
+               return buf, time.Now().UTC(), err
        }
        f, err := os.Open(path)
        if err != nil {
-               return nil, err
+               return nil, time.Time{}, err
        }
        defer f.Close()
-       return ioutil.ReadAll(f)
+       fi, err := f.Stat()
+       if err != nil {
+               return nil, time.Time{}, err
+       }
+       buf, err := ioutil.ReadAll(f)
+       return buf, fi.ModTime().UTC(), err
 }
 
 func (ldr *Loader) Load() (*arvados.Config, error) {
        if ldr.configdata == nil {
-               buf, err := ldr.loadBytes(ldr.Path)
+               buf, t, err := ldr.loadBytes(ldr.Path)
                if err != nil {
                        return nil, err
                }
                ldr.configdata = buf
+               ldr.sourceTimestamp = t
        }
 
        // FIXME: We should reject YAML if the same key is used twice
@@ -330,6 +342,8 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                        }
                }
        }
+       cfg.SourceTimestamp = ldr.sourceTimestamp
+       cfg.SourceSHA256 = fmt.Sprintf("%x", sha256.Sum256(ldr.configdata))
        return &cfg, nil
 }
 

diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 2d87b906c9b399f49f243d170984a31d712f6ed9..cef7ea944bb08cd15357f9e4c58341a2135419f2 100644 (file)
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -12,13 +12,16 @@ import (
        "os"
        "os/exec"
        "reflect"
+       "regexp"
        "strings"
        "testing"
+       "time"
 
        "git.arvados.org/arvados.git/sdk/go/arvados"
        "git.arvados.org/arvados.git/sdk/go/ctxlog"
        "github.com/ghodss/yaml"
        "github.com/sirupsen/logrus"
+       "golang.org/x/sys/unix"
        check "gopkg.in/check.v1"
 )
 
@@ -315,8 +318,16 @@ Clusters:
        c.Assert(err, check.IsNil)
        yaml, err := yaml.Marshal(cfg)
        c.Assert(err, check.IsNil)
+       // Well, *nearly* no warnings. SourceTimestamp and
+       // SourceSHA256 are included in a config-dump, but not
+       // expected in a real config file.
+       yaml = regexp.MustCompile(`(^|\n)(Source(Timestamp|SHA256): .*?\n)+`).ReplaceAll(yaml, []byte("$1"))
        cfgDumped, err := testLoader(c, string(yaml), &logbuf).Load()
        c.Assert(err, check.IsNil)
+       // SourceTimestamp and SourceSHA256 aren't expected to be
+       // preserved through dump+load
+       cfgDumped.SourceTimestamp = cfg.SourceTimestamp
+       cfgDumped.SourceSHA256 = cfg.SourceSHA256
        c.Check(cfg, check.DeepEquals, cfgDumped)
        c.Check(logbuf.String(), check.Equals, "")
 }
@@ -499,6 +510,12 @@ func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) {
        c.Assert(err, check.IsNil)
        expected, err := expectedldr.Load()
        c.Assert(err, check.IsNil)
+       // The inputs generally aren't even files, so SourceTimestamp
+       // can't be expected to match.
+       got.SourceTimestamp = expected.SourceTimestamp
+       // Obviously the content isn't identical -- otherwise we
+       // wouldn't need to check that it's equivalent.
+       got.SourceSHA256 = expected.SourceSHA256
        checkEqualYAML(c, got, expected)
 }
 
@@ -758,3 +775,28 @@ Clusters:
        c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*Type2\.preemptible.*`)
        c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*(z1111|z2222)[^\n]*InstanceTypes.*`)
 }
+
+func (s *LoadSuite) TestSourceTimestamp(c *check.C) {
+       conftime, err := time.Parse(time.RFC3339, "2022-03-04T05:06:07-08:00")
+       c.Assert(err, check.IsNil)
+       confdata := `Clusters: {zzzzz: {}}`
+       conffile := c.MkDir() + "/config.yml"
+       ioutil.WriteFile(conffile, []byte(confdata), 0777)
+       tv := unix.NsecToTimeval(conftime.UnixNano())
+       unix.Lutimes(conffile, []unix.Timeval{tv, tv})
+       for _, trial := range []struct {
+               configarg  string
+               expectTime time.Time
+       }{
+               {"-", time.Now()},
+               {conffile, conftime},
+       } {
+               c.Logf("trial: %+v", trial)
+               ldr := NewLoader(strings.NewReader(confdata), ctxlog.TestLogger(c))
+               ldr.Path = trial.configarg
+               cfg, err := ldr.Load()
+               c.Assert(err, check.IsNil)
+               c.Check(cfg.SourceTimestamp, check.Equals, cfg.SourceTimestamp.UTC())
+               c.Check(int(cfg.SourceTimestamp.Sub(trial.expectTime).Seconds()), check.Equals, 0)
+       }
+}

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 6c9324e478a273b68e83dcd7fb4e46150b8e0da1..b508a3f05d2c40a1c25345c3bd2f8b79c907089d 100644 (file)
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -10,6 +10,7 @@ import (
        "fmt"
        "net/url"
        "os"
+       "time"
 
        "git.arvados.org/arvados.git/sdk/go/config"
 )
@@ -24,6 +25,8 @@ var DefaultConfigFile = func() string {
 type Config struct {
        Clusters         map[string]Cluster
        AutoReloadConfig bool
+       SourceTimestamp  time.Time
+       SourceSHA256     string
 }
 
 // GetConfig returns the current system config, loading it from

diff --git a/sdk/go/health/aggregator_test.go b/sdk/go/health/aggregator_test.go
index 05f0bdd31b18e0808e32c5321881bf77fc56e75c..62eca894b7d1719f2c35491f18f811b88c33e466 100644 (file)
--- a/sdk/go/health/aggregator_test.go
+++ b/sdk/go/health/aggregator_test.go
@@ -10,6 +10,7 @@ import (
        "io/ioutil"
        "net/http"
        "net/http/httptest"
+       "regexp"
        "strings"
        "time"
 
@@ -142,6 +143,7 @@ func (s *AggregatorSuite) TestCheckCommand(c *check.C) {
        tmpdir := c.MkDir()
        confdata, err := yaml.Marshal(arvados.Config{Clusters: map[string]arvados.Cluster{s.handler.Cluster.ClusterID: *s.handler.Cluster}})
        c.Assert(err, check.IsNil)
+       confdata = regexp.MustCompile(`SourceTimestamp: [^\n]+\n`).ReplaceAll(confdata, []byte{})
        err = ioutil.WriteFile(tmpdir+"/config.yml", confdata, 0777)
        c.Assert(err, check.IsNil)
        var stdout, stderr bytes.Buffer