18321: Incorporate CUDA request into picking a node type

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index a84dc5d31673b58b152645f3b73f3efa514ea767..c69cee75b94cddc8ae8aa955e72db43f5f915842 100644 (file)
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -1303,6 +1303,11 @@ Clusters:
         AddedScratch: 0
         Price: 0.1
         Preemptible: false
+        # Include this section if the node type includes GPU (CUDA) support
+        CUDA:
+          DriverVersion: "11.0"
+          HardwareCapability: "9.0"
+          DeviceCount: 1
 
     StorageClasses:
 

diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 567ac30a9b7d855cfabdc10374dcf55b8f9694c7..9294f76968afb00739b17911c8f548230f66892a 100644 (file)
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -1309,6 +1309,11 @@ Clusters:
         AddedScratch: 0
         Price: 0.1
         Preemptible: false
+        # Include this section if the node type includes GPU (CUDA) support
+        CUDA:
+          DriverVersion: "11.0"
+          HardwareCapability: "9.0"
+          DeviceCount: 1
 
     StorageClasses:
 

diff --git a/lib/dispatchcloud/node_size.go b/lib/dispatchcloud/node_size.go
index 1b10826cbb6227a589d0c74217c808c59c220c8b..aa2cd7d5696d46a7184b2edbb39d45b6d4149ead 100644 (file)
--- a/lib/dispatchcloud/node_size.go
+++ b/lib/dispatchcloud/node_size.go
@@ -83,6 +83,19 @@ func EstimateScratchSpace(ctr *arvados.Container) (needScratch int64) {
        return
 }
 
+// compareVersion returns true if vs1 >= vs2, otherwise false
+func compareVersion(vs1 string, vs2 string) bool {
+       v1, err := strconv.ParseFloat(vs1, 64)
+       if err != nil {
+               return false
+       }
+       v2, err := strconv.ParseFloat(vs2, 64)
+       if err != nil {
+               return false
+       }
+       return v1 >= v2
+}
+
 // ChooseInstanceType returns the cheapest available
 // arvados.InstanceType big enough to run ctr.
 func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvados.InstanceType, err error) {
@@ -103,14 +116,19 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
        ok := false
        for _, it := range cc.InstanceTypes {
                switch {
-               case ok && it.Price > best.Price:
-               case int64(it.Scratch) < needScratch:
-               case int64(it.RAM) < needRAM:
-               case it.VCPUs < needVCPUs:
-               case it.Preemptible != ctr.SchedulingParameters.Preemptible:
-               case it.Price == best.Price && (it.RAM < best.RAM || it.VCPUs < best.VCPUs):
-                       // Equal price, but worse specs
+               // reasons to reject a node
+               case ok && it.Price > best.Price: // already selected a node, and this one is more expensive
+               case int64(it.Scratch) < needScratch: // insufficient scratch
+               case int64(it.RAM) < needRAM: // insufficient RAM
+               case it.VCPUs < needVCPUs: // insufficient VCPUs
+               case it.Preemptible != ctr.SchedulingParameters.Preemptible: // wrong preemptable setting
+               case it.Price == best.Price && (it.RAM < best.RAM || it.VCPUs < best.VCPUs): // same price, worse specs
+               case it.CUDA.DeviceCount < ctr.RuntimeConstraints.CUDADeviceCount: // insufficient CUDA devices
+               case it.CUDA.DeviceCount > 0 && !compareVersion(it.CUDA.DriverVersion, ctr.RuntimeConstraints.CUDADriverVersion): // insufficient driver version
+               case it.CUDA.DeviceCount > 0 && !compareVersion(it.CUDA.HardwareCapability, ctr.RuntimeConstraints.CUDAHardwareCapability): // insufficient hardware capability
+                       // Don't select this node
                default:
+                       // Didn't reject the node, so select it
                        // Lower price || (same price && better specs)
                        best = it
                        ok = true

diff --git a/lib/dispatchcloud/node_size_test.go b/lib/dispatchcloud/node_size_test.go
index abd292cbaf1278c7ca147e7da7352d74d0ad39d0..cdcf4033fcc629317ef0f6edddf799778bf50e5b 100644 (file)
--- a/lib/dispatchcloud/node_size_test.go
+++ b/lib/dispatchcloud/node_size_test.go
@@ -147,3 +147,76 @@ func (*NodeSizeSuite) TestScratchForDockerImage(c *check.C) {
        // Short manifest will return 0
        c.Check(n, check.Equals, int64(0))
 }
+
+func (*NodeSizeSuite) TestChooseGPU(c *check.C) {
+       menu := map[string]arvados.InstanceType{
+               "costly":         {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly", CUDA: arvados.CUDAFeatures{DeviceCount: 2, HardwareCapability: "9.0", DriverVersion: "11.0"}},
+               "low_capability": {Price: 2.1, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "low_capability", CUDA: arvados.CUDAFeatures{DeviceCount: 1, HardwareCapability: "8.0", DriverVersion: "11.0"}},
+               "best":           {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best", CUDA: arvados.CUDAFeatures{DeviceCount: 1, HardwareCapability: "9.0", DriverVersion: "11.0"}},
+               "low_driver":     {Price: 2.1, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "low_driver", CUDA: arvados.CUDAFeatures{DeviceCount: 1, HardwareCapability: "9.0", DriverVersion: "10.0"}},
+               "small":          {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
+       }
+       best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
+               Mounts: map[string]arvados.Mount{
+                       "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+               },
+               RuntimeConstraints: arvados.RuntimeConstraints{
+                       VCPUs:                  2,
+                       RAM:                    987654321,
+                       KeepCacheRAM:           123456789,
+                       CUDADeviceCount:        1,
+                       CUDAHardwareCapability: "9.0",
+                       CUDADriverVersion:      "11.0",
+               },
+       })
+       c.Check(err, check.IsNil)
+       c.Check(best.Name, check.Equals, "best")
+       c.Check(best.RAM >= 1234567890, check.Equals, true)
+       c.Check(best.VCPUs >= 2, check.Equals, true)
+       c.Check(best.CUDA.DeviceCount >= 1, check.Equals, true)
+       c.Check(best.CUDA.DriverVersion, check.Equals, "11.0")
+       c.Check(best.CUDA.HardwareCapability, check.Equals, "9.0")
+
+       best, err = ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
+               Mounts: map[string]arvados.Mount{
+                       "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+               },
+               RuntimeConstraints: arvados.RuntimeConstraints{
+                       VCPUs:                  2,
+                       RAM:                    987654321,
+                       KeepCacheRAM:           123456789,
+                       CUDADeviceCount:        2,
+                       CUDAHardwareCapability: "9.0",
+                       CUDADriverVersion:      "11.0",
+               },
+       })
+       c.Check(err, check.IsNil)
+       c.Check(best.Name, check.Equals, "costly")
+       c.Check(best.RAM >= 1234567890, check.Equals, true)
+       c.Check(best.VCPUs >= 2, check.Equals, true)
+       c.Check(best.CUDA.DeviceCount >= 2, check.Equals, true)
+       c.Check(best.CUDA.DriverVersion, check.Equals, "11.0")
+       c.Check(best.CUDA.HardwareCapability, check.Equals, "9.0")
+
+       best, err = ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
+               Mounts: map[string]arvados.Mount{
+                       "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+               },
+               RuntimeConstraints: arvados.RuntimeConstraints{
+                       VCPUs:                  2,
+                       RAM:                    987654321,
+                       KeepCacheRAM:           123456789,
+                       CUDADeviceCount:        1,
+                       CUDAHardwareCapability: "8.0",
+                       CUDADriverVersion:      "11.0",
+               },
+       })
+       c.Check(err, check.IsNil)
+       c.Check(best.Name, check.Equals, "low_capability")
+       c.Check(best.RAM >= 1234567890, check.Equals, true)
+       c.Check(best.VCPUs >= 2, check.Equals, true)
+       c.Check(best.CUDA.DeviceCount >= 1, check.Equals, true)
+       c.Check(best.CUDA.DriverVersion, check.Equals, "11.0")
+       c.Check(best.CUDA.HardwareCapability, check.Equals, "8.0")
+
+}

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 474ce33b0ee4e9b2262bbe1b16f9ed36c0c2af38..3111d61403085c5108f72e901ddbe9fcfc8e4ad3 100644 (file)
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -410,6 +410,12 @@ type RemoteCluster struct {
        ActivateUsers bool
 }
 
+type CUDAFeatures struct {
+       DriverVersion      string
+       HardwareCapability string
+       DeviceCount        int
+}
+
 type InstanceType struct {
        Name            string
        ProviderType    string
@@ -420,6 +426,7 @@ type InstanceType struct {
        AddedScratch    ByteSize
        Price           float64
        Preemptible     bool
+       CUDA            CUDAFeatures
 }
 
 type ContainersConfig struct {