X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/9b9c01638a9b4bd9ff7f32fd186daf77d222eec6..86660414472d4ff0d8267f9845a753497bd41692:/lib/config/load_test.go

diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 4cdebf62bc..a19400c191 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -12,13 +12,19 @@ import (
 	"os"
 	"os/exec"
 	"reflect"
+	"regexp"
+	"runtime"
 	"strings"
 	"testing"
+	"time"
 
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 	"github.com/ghodss/yaml"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/expfmt"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 	check "gopkg.in/check.v1"
 )
 
@@ -29,6 +35,8 @@ func Test(t *testing.T) {
 
 var _ = check.Suite(&LoadSuite{})
 
+var emptyConfigYAML = `Clusters: {"z1111": {}}`
+
 // Return a new Loader that reads cluster config from configdata
 // (instead of the usual default /etc/arvados/config.yml), and logs to
 // logdst or (if that's nil) c.Log.
@@ -59,7 +67,7 @@ func (s *LoadSuite) TestEmpty(c *check.C) {
 }
 
 func (s *LoadSuite) TestNoConfigs(c *check.C) {
-	cfg, err := testLoader(c, `Clusters: {"z1111": {}}`, nil).Load()
+	cfg, err := testLoader(c, emptyConfigYAML, nil).Load()
 	c.Assert(err, check.IsNil)
 	c.Assert(cfg.Clusters, check.HasLen, 1)
 	cc, err := cfg.GetCluster("z1111")
@@ -69,6 +77,18 @@ func (s *LoadSuite) TestNoConfigs(c *check.C) {
 	c.Check(cc.API.MaxItemsPerResponse, check.Equals, 1000)
 }
 
+func (s *LoadSuite) TestNullKeyDoesNotOverrideDefault(c *check.C) {
+	ldr := testLoader(c, `{"Clusters":{"z1111":{"API":}}}`, nil)
+	ldr.SkipDeprecated = true
+	cfg, err := ldr.Load()
+	c.Assert(err, check.IsNil)
+	c1, err := cfg.GetCluster("z1111")
+	c.Assert(err, check.IsNil)
+	c.Check(c1.ClusterID, check.Equals, "z1111")
+	c.Check(c1.API.MaxRequestAmplification, check.Equals, 4)
+	c.Check(c1.API.MaxItemsPerResponse, check.Equals, 1000)
+}
+
 func (s *LoadSuite) TestMungeLegacyConfigArgs(c *check.C) {
 	f, err := ioutil.TempFile("", "")
 	c.Check(err, check.IsNil)
@@ -79,7 +99,7 @@ func (s *LoadSuite) TestMungeLegacyConfigArgs(c *check.C) {
 	f, err = ioutil.TempFile("", "")
 	c.Check(err, check.IsNil)
 	defer os.Remove(f.Name())
-	io.WriteString(f, "Clusters: {aaaaa: {}}\n")
+	io.WriteString(f, emptyConfigYAML)
 	newfile := f.Name()
 
 	for _, trial := range []struct {
@@ -196,21 +216,37 @@ Clusters:
     SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
     Collections:
      BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    postgresql: {}
-    BadKey: {}
-    Containers: {}
+    PostgreSQL: {}
+    BadKey1: {}
+    Containers:
+      RunTimeEngine: abc
     RemoteClusters:
       z2222:
         Host: z2222.arvadosapi.com
         Proxy: true
-        BadKey: badValue
+        BadKey2: badValue
+    Services:
+      KeepStore:
+        InternalURLs:
+          "http://host.example:12345": {}
+      Keepstore:
+        InternalURLs:
+          "http://host.example:12345":
+            RendezVous: x
+    ServiceS:
+      Keepstore:
+        InternalURLs:
+          "http://host.example:12345": {}
+    Volumes:
+      zzzzz-nyw5e-aaaaaaaaaaaaaaa: {Replication: 2}
 `, &logbuf).Load()
 	c.Assert(err, check.IsNil)
+	c.Log(logbuf.String())
 	logs := strings.Split(strings.TrimSuffix(logbuf.String(), "\n"), "\n")
 	for _, log := range logs {
-		c.Check(log, check.Matches, `.*deprecated or unknown config entry:.*BadKey.*`)
+		c.Check(log, check.Matches, `.*deprecated or unknown config entry:.*(RunTimeEngine.*RuntimeEngine|BadKey1|BadKey2|KeepStore|ServiceS|RendezVous).*`)
 	}
-	c.Check(logs, check.HasLen, 2)
+	c.Check(logs, check.HasLen, 6)
 }
 
 func (s *LoadSuite) checkSAMPLEKeys(c *check.C, path string, x interface{}) {
@@ -275,20 +311,30 @@ func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
 
 func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
 	var logbuf bytes.Buffer
-	logger := logrus.New()
-	logger.Out = &logbuf
 	cfg, err := testLoader(c, `
 Clusters:
  zzzzz:
   ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
   SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
   Collections:
-   BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, &logbuf).Load()
+   BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+  InstanceTypes:
+   abc:
+    IncludedScratch: 123456
+`, &logbuf).Load()
 	c.Assert(err, check.IsNil)
 	yaml, err := yaml.Marshal(cfg)
 	c.Assert(err, check.IsNil)
+	// Well, *nearly* no warnings. SourceTimestamp and
+	// SourceSHA256 are included in a config-dump, but not
+	// expected in a real config file.
+	yaml = regexp.MustCompile(`(^|\n)(Source(Timestamp|SHA256): .*?\n)+`).ReplaceAll(yaml, []byte("$1"))
 	cfgDumped, err := testLoader(c, string(yaml), &logbuf).Load()
 	c.Assert(err, check.IsNil)
+	// SourceTimestamp and SourceSHA256 aren't expected to be
+	// preserved through dump+load
+	cfgDumped.SourceTimestamp = cfg.SourceTimestamp
+	cfgDumped.SourceSHA256 = cfg.SourceSHA256
 	c.Check(cfg, check.DeepEquals, cfgDumped)
 	c.Check(logbuf.String(), check.Equals, "")
 }
@@ -310,11 +356,7 @@ func (s *LoadSuite) TestUnacceptableTokens(c *check.C) {
 	} {
 		c.Logf("trying bogus config: %s", trial.example)
 		_, err := testLoader(c, "Clusters:\n zzzzz:\n  "+trial.example, nil).Load()
-		if trial.short {
-			c.Check(err, check.ErrorMatches, `Clusters.zzzzz.`+trial.configPath+`: unacceptable characters in token.*`)
-		} else {
-			c.Check(err, check.ErrorMatches, `Clusters.zzzzz.`+trial.configPath+`: unacceptable characters in token.*`)
-		}
+		c.Check(err, check.ErrorMatches, `Clusters.zzzzz.`+trial.configPath+`: unacceptable characters in token.*`)
 	}
 }
 
@@ -322,8 +364,8 @@ func (s *LoadSuite) TestPostgreSQLKeyConflict(c *check.C) {
 	_, err := testLoader(c, `
 Clusters:
  zzzzz:
-  postgresql:
-   connection:
+  PostgreSQL:
+   Connection:
      DBName: dbname
      Host: host
 `, nil).Load()
@@ -351,7 +393,7 @@ Clusters:
     Proxy: true
 `, `
 Clusters:
- abcdef:
+ abcde:
   ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
   SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
   Collections:
@@ -365,7 +407,7 @@ Clusters:
 		if v != nil {
 			c.Logf("%#v", v.Clusters)
 		}
-		c.Check(err, check.ErrorMatches, `.*cluster ID should be 5 alphanumeric characters.*`)
+		c.Check(err, check.ErrorMatches, `.*cluster ID should be 5 lowercase alphanumeric characters.*`)
 	}
 }
 
@@ -437,10 +479,12 @@ Clusters:
 `)
 }
 
-func checkEquivalent(c *check.C, goty, expectedy string) {
-	gotldr := testLoader(c, goty, nil)
+func checkEquivalent(c *check.C, goty, expectedy string) string {
+	var logbuf bytes.Buffer
+	gotldr := testLoader(c, goty, &logbuf)
 	expectedldr := testLoader(c, expectedy, nil)
 	checkEquivalentLoaders(c, gotldr, expectedldr)
+	return logbuf.String()
 }
 
 func checkEqualYAML(c *check.C, got, expected interface{}) {
@@ -473,6 +517,12 @@ func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) {
 	c.Assert(err, check.IsNil)
 	expected, err := expectedldr.Load()
 	c.Assert(err, check.IsNil)
+	// The inputs generally aren't even files, so SourceTimestamp
+	// can't be expected to match.
+	got.SourceTimestamp = expected.SourceTimestamp
+	// Obviously the content isn't identical -- otherwise we
+	// wouldn't need to check that it's equivalent.
+	got.SourceSHA256 = expected.SourceSHA256
 	checkEqualYAML(c, got, expected)
 }
 
@@ -544,11 +594,321 @@ func (s *LoadSuite) TestListKeys(c *check.C) {
 		c.Errorf("Should have produced an error")
 	}
 
-	var logbuf bytes.Buffer
-	loader := testLoader(c, string(DefaultYAML), &logbuf)
+	loader := testLoader(c, string(DefaultYAML), nil)
 	cfg, err := loader.Load()
 	c.Assert(err, check.IsNil)
 	if err := checkListKeys("", cfg); err != nil {
 		c.Error(err)
 	}
 }
+
+func (s *LoadSuite) TestLoopbackInstanceTypes(c *check.C) {
+	ldr := testLoader(c, `
+Clusters:
+ z1111:
+  Containers:
+   CloudVMs:
+    Enable: true
+    Driver: loopback
+  InstanceTypes:
+   a: {}
+   b: {}
+`, nil)
+	cfg, err := ldr.Load()
+	c.Check(err, check.ErrorMatches, `Clusters\.z1111\.InstanceTypes: cannot use multiple InstanceTypes with loopback driver`)
+
+	ldr = testLoader(c, `
+Clusters:
+ z1111:
+  Containers:
+   CloudVMs:
+    Enable: true
+    Driver: loopback
+`, nil)
+	cfg, err = ldr.Load()
+	c.Assert(err, check.IsNil)
+	cc, err := cfg.GetCluster("")
+	c.Assert(err, check.IsNil)
+	c.Check(cc.InstanceTypes, check.HasLen, 1)
+	c.Check(cc.InstanceTypes["localhost"].VCPUs, check.Equals, runtime.NumCPU())
+
+	ldr = testLoader(c, `
+Clusters:
+ z1111:
+  Containers:
+   CloudVMs:
+    Enable: true
+    Driver: loopback
+  InstanceTypes:
+   a:
+    VCPUs: 9
+`, nil)
+	cfg, err = ldr.Load()
+	c.Assert(err, check.IsNil)
+	cc, err = cfg.GetCluster("")
+	c.Assert(err, check.IsNil)
+	c.Check(cc.InstanceTypes, check.HasLen, 1)
+	c.Check(cc.InstanceTypes["a"].VCPUs, check.Equals, 9)
+}
+
+func (s *LoadSuite) TestWarnUnusedLocalKeep(c *check.C) {
+	var logbuf bytes.Buffer
+	_, err := testLoader(c, `
+Clusters:
+ z1111:
+  Volumes:
+   z:
+    Replication: 1
+`, &logbuf).Load()
+	c.Assert(err, check.IsNil)
+	c.Check(logbuf.String(), check.Matches, `(?ms).*LocalKeepBlobBuffersPerVCPU is 1 but will not be used because at least one volume \(z\) has lower replication than DefaultReplication \(1 < 2\) -- suggest changing to 0.*`)
+
+	logbuf.Reset()
+	_, err = testLoader(c, `
+Clusters:
+ z1111:
+  Volumes:
+   z:
+    AccessViaHosts:
+     "http://0.0.0.0:12345": {}
+`, &logbuf).Load()
+	c.Assert(err, check.IsNil)
+	c.Check(logbuf.String(), check.Matches, `(?ms).*LocalKeepBlobBuffersPerVCPU is 1 but will not be used because at least one volume \(z\) uses AccessViaHosts -- suggest changing to 0.*`)
+}
+
+func (s *LoadSuite) TestImplicitStorageClasses(c *check.C) {
+	// If StorageClasses and Volumes.*.StorageClasses are all
+	// empty, there is a default storage class named "default".
+	ldr := testLoader(c, `{"Clusters":{"z1111":{}}}`, nil)
+	cfg, err := ldr.Load()
+	c.Assert(err, check.IsNil)
+	cc, err := cfg.GetCluster("z1111")
+	c.Assert(err, check.IsNil)
+	c.Check(cc.StorageClasses, check.HasLen, 1)
+	c.Check(cc.StorageClasses["default"].Default, check.Equals, true)
+	c.Check(cc.StorageClasses["default"].Priority, check.Equals, 0)
+
+	// The implicit "default" storage class is used by all
+	// volumes.
+	ldr = testLoader(c, `
+Clusters:
+ z1111:
+  Volumes:
+   z: {}`, nil)
+	cfg, err = ldr.Load()
+	c.Assert(err, check.IsNil)
+	cc, err = cfg.GetCluster("z1111")
+	c.Assert(err, check.IsNil)
+	c.Check(cc.StorageClasses, check.HasLen, 1)
+	c.Check(cc.StorageClasses["default"].Default, check.Equals, true)
+	c.Check(cc.StorageClasses["default"].Priority, check.Equals, 0)
+	c.Check(cc.Volumes["z"].StorageClasses["default"], check.Equals, true)
+
+	// The "default" storage class isn't implicit if any classes
+	// are configured explicitly.
+	ldr = testLoader(c, `
+Clusters:
+ z1111:
+  StorageClasses:
+   local:
+    Default: true
+    Priority: 111
+  Volumes:
+   z:
+    StorageClasses:
+     local: true`, nil)
+	cfg, err = ldr.Load()
+	c.Assert(err, check.IsNil)
+	cc, err = cfg.GetCluster("z1111")
+	c.Assert(err, check.IsNil)
+	c.Check(cc.StorageClasses, check.HasLen, 1)
+	c.Check(cc.StorageClasses["local"].Default, check.Equals, true)
+	c.Check(cc.StorageClasses["local"].Priority, check.Equals, 111)
+
+	// It is an error for a volume to refer to a storage class
+	// that isn't listed in StorageClasses.
+	ldr = testLoader(c, `
+Clusters:
+ z1111:
+  StorageClasses:
+   local:
+    Default: true
+    Priority: 111
+  Volumes:
+   z:
+    StorageClasses:
+     nx: true`, nil)
+	_, err = ldr.Load()
+	c.Assert(err, check.ErrorMatches, `z: volume refers to storage class "nx" that is not defined.*`)
+
+	// It is an error for a volume to refer to a storage class
+	// that isn't listed in StorageClasses ... even if it's
+	// "default", which would exist implicitly if it weren't
+	// referenced explicitly by a volume.
+	ldr = testLoader(c, `
+Clusters:
+ z1111:
+  Volumes:
+   z:
+    StorageClasses:
+     default: true`, nil)
+	_, err = ldr.Load()
+	c.Assert(err, check.ErrorMatches, `z: volume refers to storage class "default" that is not defined.*`)
+
+	// If the "default" storage class is configured explicitly, it
+	// is not used implicitly by any volumes, even if it's the
+	// only storage class.
+	var logbuf bytes.Buffer
+	ldr = testLoader(c, `
+Clusters:
+ z1111:
+  StorageClasses:
+   default:
+    Default: true
+    Priority: 111
+  Volumes:
+   z: {}`, &logbuf)
+	_, err = ldr.Load()
+	c.Assert(err, check.ErrorMatches, `z: volume has no StorageClasses listed`)
+
+	// If StorageClasses are configured explicitly, there must be
+	// at least one with Default: true. (Calling one "default" is
+	// not sufficient.)
+	ldr = testLoader(c, `
+Clusters:
+ z1111:
+  StorageClasses:
+   default:
+    Priority: 111
+  Volumes:
+   z:
+    StorageClasses:
+     default: true`, nil)
+	_, err = ldr.Load()
+	c.Assert(err, check.ErrorMatches, `there is no default storage class.*`)
+}
+
+func (s *LoadSuite) TestPreemptiblePriceFactor(c *check.C) {
+	yaml := `
+Clusters:
+ z1111:
+  InstanceTypes:
+   Type1:
+    RAM: 12345M
+    VCPUs: 8
+    Price: 1.23
+ z2222:
+  Containers:
+   PreemptiblePriceFactor: 0.5
+  InstanceTypes:
+   Type1:
+    RAM: 12345M
+    VCPUs: 8
+    Price: 1.23
+ z3333:
+  Containers:
+   PreemptiblePriceFactor: 0.5
+  InstanceTypes:
+   Type1:
+    RAM: 12345M
+    VCPUs: 8
+    Price: 1.23
+   Type1.preemptible: # higher price than the auto-added variant would use -- should generate warning
+    ProviderType: Type1
+    RAM: 12345M
+    VCPUs: 8
+    Price: 1.23
+    Preemptible: true
+   Type2:
+    RAM: 23456M
+    VCPUs: 16
+    Price: 2.46
+   Type2.preemptible: # identical to the auto-added variant -- so no warning
+    ProviderType: Type2
+    RAM: 23456M
+    VCPUs: 16
+    Price: 1.23
+    Preemptible: true
+`
+	var logbuf bytes.Buffer
+	cfg, err := testLoader(c, yaml, &logbuf).Load()
+	c.Assert(err, check.IsNil)
+	cc, err := cfg.GetCluster("z1111")
+	c.Assert(err, check.IsNil)
+	c.Check(cc.InstanceTypes["Type1"].Price, check.Equals, 1.23)
+	c.Check(cc.InstanceTypes, check.HasLen, 1)
+
+	cc, err = cfg.GetCluster("z2222")
+	c.Assert(err, check.IsNil)
+	c.Check(cc.InstanceTypes["Type1"].Preemptible, check.Equals, false)
+	c.Check(cc.InstanceTypes["Type1"].Price, check.Equals, 1.23)
+	c.Check(cc.InstanceTypes["Type1.preemptible"].Preemptible, check.Equals, true)
+	c.Check(cc.InstanceTypes["Type1.preemptible"].Price, check.Equals, 1.23/2)
+	c.Check(cc.InstanceTypes["Type1.preemptible"].ProviderType, check.Equals, "Type1")
+	c.Check(cc.InstanceTypes, check.HasLen, 2)
+
+	cc, err = cfg.GetCluster("z3333")
+	c.Assert(err, check.IsNil)
+	// Don't overwrite the explicitly configured preemptible variant
+	c.Check(cc.InstanceTypes["Type1.preemptible"].Price, check.Equals, 1.23)
+	c.Check(cc.InstanceTypes, check.HasLen, 4)
+	c.Check(logbuf.String(), check.Matches, `(?ms).*Clusters\.z3333\.InstanceTypes\[Type1\.preemptible\]: already exists, so not automatically adding a preemptible variant of Type1.*`)
+	c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*Type2\.preemptible.*`)
+	c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*(z1111|z2222)[^\n]*InstanceTypes.*`)
+}
+
+func (s *LoadSuite) TestSourceTimestamp(c *check.C) {
+	conftime, err := time.Parse(time.RFC3339, "2022-03-04T05:06:07-08:00")
+	c.Assert(err, check.IsNil)
+	confdata := `Clusters: {zzzzz: {}}`
+	conffile := c.MkDir() + "/config.yml"
+	ioutil.WriteFile(conffile, []byte(confdata), 0777)
+	tv := unix.NsecToTimeval(conftime.UnixNano())
+	unix.Lutimes(conffile, []unix.Timeval{tv, tv})
+	for _, trial := range []struct {
+		configarg  string
+		expectTime time.Time
+	}{
+		{"-", time.Now()},
+		{conffile, conftime},
+	} {
+		c.Logf("trial: %+v", trial)
+		ldr := NewLoader(strings.NewReader(confdata), ctxlog.TestLogger(c))
+		ldr.Path = trial.configarg
+		cfg, err := ldr.Load()
+		c.Assert(err, check.IsNil)
+		c.Check(cfg.SourceTimestamp, check.Equals, cfg.SourceTimestamp.UTC())
+		c.Check(cfg.SourceTimestamp, check.Equals, ldr.sourceTimestamp)
+		c.Check(int(cfg.SourceTimestamp.Sub(trial.expectTime).Seconds()), check.Equals, 0)
+		c.Check(int(ldr.loadTimestamp.Sub(time.Now()).Seconds()), check.Equals, 0)
+
+		var buf bytes.Buffer
+		reg := prometheus.NewRegistry()
+		ldr.RegisterMetrics(reg)
+		enc := expfmt.NewEncoder(&buf, expfmt.FmtText)
+		got, _ := reg.Gather()
+		for _, mf := range got {
+			enc.Encode(mf)
+		}
+		c.Check(buf.String(), check.Matches, `# HELP .*
+# TYPE .*
+arvados_config_load_timestamp_seconds{sha256="83aea5d82eb1d53372cd65c936c60acc1c6ef946e61977bbca7cfea709d201a8"} \Q`+fmt.Sprintf("%g", float64(ldr.loadTimestamp.UnixNano())/1e9)+`\E
+# HELP .*
+# TYPE .*
+arvados_config_source_timestamp_seconds{sha256="83aea5d82eb1d53372cd65c936c60acc1c6ef946e61977bbca7cfea709d201a8"} \Q`+fmt.Sprintf("%g", float64(cfg.SourceTimestamp.UnixNano())/1e9)+`\E
+`)
+	}
+}
+
+func (s *LoadSuite) TestGetHostRAM(c *check.C) {
+	hostram, err := getHostRAM()
+	c.Check(err, check.IsNil)
+	c.Logf("getHostRAM() == %v", hostram)
+}
+
+func (s *LoadSuite) TestGetFilesystemSize(c *check.C) {
+	path := c.MkDir()
+	size, err := getFilesystemSize(path)
+	c.Check(err, check.IsNil)
+	c.Logf("getFilesystemSize(%q) == %v", path, size)
+}