X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/437d0dc1dfeea7b81999caa1c540da0903df87f7..86660414472d4ff0d8267f9845a753497bd41692:/lib/config/load_test.go diff --git a/lib/config/load_test.go b/lib/config/load_test.go index 2d87b906c9..a19400c191 100644 --- a/lib/config/load_test.go +++ b/lib/config/load_test.go @@ -12,13 +12,19 @@ import ( "os" "os/exec" "reflect" + "regexp" + "runtime" "strings" "testing" + "time" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" "github.com/ghodss/yaml" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" check "gopkg.in/check.v1" ) @@ -232,7 +238,7 @@ Clusters: InternalURLs: "http://host.example:12345": {} Volumes: - zzzzz-nyw5e-aaaaaaaaaaaaaaa: {} + zzzzz-nyw5e-aaaaaaaaaaaaaaa: {Replication: 2} `, &logbuf).Load() c.Assert(err, check.IsNil) c.Log(logbuf.String()) @@ -311,12 +317,24 @@ Clusters: ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa Collections: - BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, &logbuf).Load() + BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + InstanceTypes: + abc: + IncludedScratch: 123456 +`, &logbuf).Load() c.Assert(err, check.IsNil) yaml, err := yaml.Marshal(cfg) c.Assert(err, check.IsNil) + // Well, *nearly* no warnings. SourceTimestamp and + // SourceSHA256 are included in a config-dump, but not + // expected in a real config file. + yaml = regexp.MustCompile(`(^|\n)(Source(Timestamp|SHA256): .*?\n)+`).ReplaceAll(yaml, []byte("$1")) cfgDumped, err := testLoader(c, string(yaml), &logbuf).Load() c.Assert(err, check.IsNil) + // SourceTimestamp and SourceSHA256 aren't expected to be + // preserved through dump+load + cfgDumped.SourceTimestamp = cfg.SourceTimestamp + cfgDumped.SourceSHA256 = cfg.SourceSHA256 c.Check(cfg, check.DeepEquals, cfgDumped) c.Check(logbuf.String(), check.Equals, "") } @@ -389,7 +407,7 @@ Clusters: if v != nil { c.Logf("%#v", v.Clusters) } - c.Check(err, check.ErrorMatches, `.*cluster ID should be 5 alphanumeric characters.*`) + c.Check(err, check.ErrorMatches, `.*cluster ID should be 5 lowercase alphanumeric characters.*`) } } @@ -499,6 +517,12 @@ func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) { c.Assert(err, check.IsNil) expected, err := expectedldr.Load() c.Assert(err, check.IsNil) + // The inputs generally aren't even files, so SourceTimestamp + // can't be expected to match. + got.SourceTimestamp = expected.SourceTimestamp + // Obviously the content isn't identical -- otherwise we + // wouldn't need to check that it's equivalent. + got.SourceSHA256 = expected.SourceSHA256 checkEqualYAML(c, got, expected) } @@ -578,6 +602,80 @@ func (s *LoadSuite) TestListKeys(c *check.C) { } } +func (s *LoadSuite) TestLoopbackInstanceTypes(c *check.C) { + ldr := testLoader(c, ` +Clusters: + z1111: + Containers: + CloudVMs: + Enable: true + Driver: loopback + InstanceTypes: + a: {} + b: {} +`, nil) + cfg, err := ldr.Load() + c.Check(err, check.ErrorMatches, `Clusters\.z1111\.InstanceTypes: cannot use multiple InstanceTypes with loopback driver`) + + ldr = testLoader(c, ` +Clusters: + z1111: + Containers: + CloudVMs: + Enable: true + Driver: loopback +`, nil) + cfg, err = ldr.Load() + c.Assert(err, check.IsNil) + cc, err := cfg.GetCluster("") + c.Assert(err, check.IsNil) + c.Check(cc.InstanceTypes, check.HasLen, 1) + c.Check(cc.InstanceTypes["localhost"].VCPUs, check.Equals, runtime.NumCPU()) + + ldr = testLoader(c, ` +Clusters: + z1111: + Containers: + CloudVMs: + Enable: true + Driver: loopback + InstanceTypes: + a: + VCPUs: 9 +`, nil) + cfg, err = ldr.Load() + c.Assert(err, check.IsNil) + cc, err = cfg.GetCluster("") + c.Assert(err, check.IsNil) + c.Check(cc.InstanceTypes, check.HasLen, 1) + c.Check(cc.InstanceTypes["a"].VCPUs, check.Equals, 9) +} + +func (s *LoadSuite) TestWarnUnusedLocalKeep(c *check.C) { + var logbuf bytes.Buffer + _, err := testLoader(c, ` +Clusters: + z1111: + Volumes: + z: + Replication: 1 +`, &logbuf).Load() + c.Assert(err, check.IsNil) + c.Check(logbuf.String(), check.Matches, `(?ms).*LocalKeepBlobBuffersPerVCPU is 1 but will not be used because at least one volume \(z\) has lower replication than DefaultReplication \(1 < 2\) -- suggest changing to 0.*`) + + logbuf.Reset() + _, err = testLoader(c, ` +Clusters: + z1111: + Volumes: + z: + AccessViaHosts: + "http://0.0.0.0:12345": {} +`, &logbuf).Load() + c.Assert(err, check.IsNil) + c.Check(logbuf.String(), check.Matches, `(?ms).*LocalKeepBlobBuffersPerVCPU is 1 but will not be used because at least one volume \(z\) uses AccessViaHosts -- suggest changing to 0.*`) +} + func (s *LoadSuite) TestImplicitStorageClasses(c *check.C) { // If StorageClasses and Volumes.*.StorageClasses are all // empty, there is a default storage class named "default". @@ -758,3 +856,59 @@ Clusters: c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*Type2\.preemptible.*`) c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*(z1111|z2222)[^\n]*InstanceTypes.*`) } + +func (s *LoadSuite) TestSourceTimestamp(c *check.C) { + conftime, err := time.Parse(time.RFC3339, "2022-03-04T05:06:07-08:00") + c.Assert(err, check.IsNil) + confdata := `Clusters: {zzzzz: {}}` + conffile := c.MkDir() + "/config.yml" + ioutil.WriteFile(conffile, []byte(confdata), 0777) + tv := unix.NsecToTimeval(conftime.UnixNano()) + unix.Lutimes(conffile, []unix.Timeval{tv, tv}) + for _, trial := range []struct { + configarg string + expectTime time.Time + }{ + {"-", time.Now()}, + {conffile, conftime}, + } { + c.Logf("trial: %+v", trial) + ldr := NewLoader(strings.NewReader(confdata), ctxlog.TestLogger(c)) + ldr.Path = trial.configarg + cfg, err := ldr.Load() + c.Assert(err, check.IsNil) + c.Check(cfg.SourceTimestamp, check.Equals, cfg.SourceTimestamp.UTC()) + c.Check(cfg.SourceTimestamp, check.Equals, ldr.sourceTimestamp) + c.Check(int(cfg.SourceTimestamp.Sub(trial.expectTime).Seconds()), check.Equals, 0) + c.Check(int(ldr.loadTimestamp.Sub(time.Now()).Seconds()), check.Equals, 0) + + var buf bytes.Buffer + reg := prometheus.NewRegistry() + ldr.RegisterMetrics(reg) + enc := expfmt.NewEncoder(&buf, expfmt.FmtText) + got, _ := reg.Gather() + for _, mf := range got { + enc.Encode(mf) + } + c.Check(buf.String(), check.Matches, `# HELP .* +# TYPE .* +arvados_config_load_timestamp_seconds{sha256="83aea5d82eb1d53372cd65c936c60acc1c6ef946e61977bbca7cfea709d201a8"} \Q`+fmt.Sprintf("%g", float64(ldr.loadTimestamp.UnixNano())/1e9)+`\E +# HELP .* +# TYPE .* +arvados_config_source_timestamp_seconds{sha256="83aea5d82eb1d53372cd65c936c60acc1c6ef946e61977bbca7cfea709d201a8"} \Q`+fmt.Sprintf("%g", float64(cfg.SourceTimestamp.UnixNano())/1e9)+`\E +`) + } +} + +func (s *LoadSuite) TestGetHostRAM(c *check.C) { + hostram, err := getHostRAM() + c.Check(err, check.IsNil) + c.Logf("getHostRAM() == %v", hostram) +} + +func (s *LoadSuite) TestGetFilesystemSize(c *check.C) { + path := c.MkDir() + size, err := getFilesystemSize(path) + c.Check(err, check.IsNil) + c.Logf("getFilesystemSize(%q) == %v", path, size) +}