18596: Add PreemptiblePriceFactor config.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 0a8f55244b905cb48a56a4b994733534e8a5fd53..c9232a76d92d7eeee4913b927f18db090655c77d 100644 (file)
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -907,6 +907,13 @@ Clusters:
       # configured, and has no effect on top-level containers.
       AlwaysUsePreemptibleInstances: true
 
+      # Automatically add a preemptible variant for every
+      # non-preemptible entry in InstanceTypes below. The maximum bid
+      # price for the preemptible variant will be the non-preemptible
+      # price multiplied by PreemptiblePriceFactor. If 0, preemptible
+      # variants are not added automatically.
+      PreemptiblePriceFactor: 0
+
       # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
       # cloud dispatcher for executing containers on worker VMs.
       # Begins with "-----BEGIN RSA PRIVATE KEY-----\n"

diff --git a/lib/config/export.go b/lib/config/export.go
index 4e903a8b3d39398b80faf08199b77d9f146165b6..67b7c3fa0ecbd1ce49df2962cfda66ae056ec693 100644 (file)
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -130,6 +130,7 @@ var whitelist = map[string]bool{
        "Containers.MaxDispatchAttempts":           false,
        "Containers.MaxRetryAttempts":              true,
        "Containers.MinRetryPeriod":                true,
+       "Containers.PreemptiblePriceFactor":        false,
        "Containers.ReserveExtraRAM":               true,
        "Containers.RuntimeEngine":                 true,
        "Containers.ShellAccess":                   true,

diff --git a/lib/config/load.go b/lib/config/load.go
index 8d498af170f2180881fac496c82900b1bd764d7f..f5d42c491629fa1943ce56cbf6969e6551de4d0f 100644 (file)
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -285,6 +285,19 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                }
        }
 
+       // Preprocess/automate some configs
+       for id, cc := range cfg.Clusters {
+               ldr.autofillPreemptible(&cc)
+
+               if strings.Count(cc.Users.AnonymousUserToken, "/") == 3 {
+                       // V2 token, strip it to just a secret
+                       tmp := strings.Split(cc.Users.AnonymousUserToken, "/")
+                       cc.Users.AnonymousUserToken = tmp[2]
+               }
+
+               cfg.Clusters[id] = cc
+       }
+
        // Check for known mistakes
        for id, cc := range cfg.Clusters {
                for remote := range cc.RemoteClusters {
@@ -316,11 +329,6 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                                return nil, err
                        }
                }
-               if strings.Count(cc.Users.AnonymousUserToken, "/") == 3 {
-                       // V2 token, strip it to just a secret
-                       tmp := strings.Split(cc.Users.AnonymousUserToken, "/")
-                       cc.Users.AnonymousUserToken = tmp[2]
-               }
        }
        return &cfg, nil
 }
@@ -527,3 +535,17 @@ func (ldr *Loader) logExtraKeys(expected, supplied map[string]interface{}, prefi
                }
        }
 }
+
+func (ldr *Loader) autofillPreemptible(cc *arvados.Cluster) {
+       if factor := cc.Containers.PreemptiblePriceFactor; factor > 0 {
+               for name, it := range cc.InstanceTypes {
+                       if !it.Preemptible {
+                               it.Preemptible = true
+                               it.Price = it.Price * factor
+                               it.Name = name + ".preemptible"
+                               cc.InstanceTypes[it.Name] = it
+                       }
+               }
+       }
+
+}

diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 1ede805b0085a668b40169af458243a304c12e93..3d8045d315a348a91e189dfa3e4052ca714e5e55 100644 (file)
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -695,3 +695,38 @@ Clusters:
        _, err = ldr.Load()
        c.Assert(err, check.ErrorMatches, `there is no default storage class.*`)
 }
+
+func (s *LoadSuite) TestPreemptiblePriceFactor(c *check.C) {
+       yaml := `
+Clusters:
+ z1111:
+  InstanceTypes:
+   Type1:
+    RAM: 12345M
+    VCPUs: 8
+    Price: 1.23
+ z2222:
+  Containers:
+   PreemptiblePriceFactor: 0.5
+  InstanceTypes:
+   Type1:
+    RAM: 12345M
+    VCPUs: 8
+    Price: 1.23
+`
+       cfg, err := testLoader(c, yaml, nil).Load()
+       c.Assert(err, check.IsNil)
+       cc, err := cfg.GetCluster("z1111")
+       c.Assert(err, check.IsNil)
+       c.Check(cc.InstanceTypes["Type1"].Price, check.Equals, 1.23)
+       c.Check(cc.InstanceTypes, check.HasLen, 1)
+
+       cc, err = cfg.GetCluster("z2222")
+       c.Assert(err, check.IsNil)
+       c.Check(cc.InstanceTypes["Type1"].Preemptible, check.Equals, false)
+       c.Check(cc.InstanceTypes["Type1"].Price, check.Equals, 1.23)
+       c.Check(cc.InstanceTypes["Type1.preemptible"].Preemptible, check.Equals, true)
+       c.Check(cc.InstanceTypes["Type1.preemptible"].Price, check.Equals, 1.23/2)
+       c.Check(cc.InstanceTypes["Type1.preemptible"].ProviderType, check.Equals, "Type1")
+       c.Check(cc.InstanceTypes, check.HasLen, 2)
+}

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index b8c8269f12acba74feb00edc07ec7949e0db5fc4..6242f6cd56ccee653c54bf7d6e1df160652cc5fe 100644 (file)
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -445,6 +445,7 @@ type ContainersConfig struct {
        StaleLockTimeout              Duration
        SupportedDockerImageFormats   StringSet
        AlwaysUsePreemptibleInstances bool
+       PreemptiblePriceFactor        float64
        RuntimeEngine                 string
        LocalKeepBlobBuffersPerVCPU   int
        LocalKeepLogsToContainerLog   string