18321: Add CUDARuntimeConstraints matching CUDAFeatures

Adjust CUDA version comparison.  Refactor tests.

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/doc/_includes/_container_runtime_constraints.liquid b/doc/_includes/_container_runtime_constraints.liquid
index dcdc29cf3f26646d80de501a0431e946d177f431..6926b9d3dcd5e7fe851dc1783069e9cda22d0222 100644 (file)
--- a/doc/_includes/_container_runtime_constraints.liquid
+++ b/doc/_includes/_container_runtime_constraints.liquid
@@ -14,6 +14,11 @@ table(table table-bordered table-condensed).
 |vcpus|integer|Number of cores to be used to run this process.|Optional. However, a ContainerRequest that is in "Committed" state must provide this.|
 |keep_cache_ram|integer|Number of keep cache bytes to be used to run this process.|Optional.|
 |API|boolean|When set, ARVADOS_API_HOST and ARVADOS_API_TOKEN will be set, and container will have networking enabled to access the Arvados API server.|Optional.|
-|cuda_driver_version|string|Minimum CUDA driver version.|Optional.|
-|cuda_hardware_capability|string|Minimum CUDA hardware capability.|Optional.|
-|cuda_device_count|int|Number of GPUs to request.|Optional.|
+|cuda|object|Request CUDA GPU support, see below|Optional.|
+
+h3. CUDA GPU support
+
+table(table table-bordered table-condensed).
+|device_count|int|Number of GPUs to request.|Required to request a GPU node.|
+|driver_version|string|Minimum CUDA driver version.|Optional.|
+|hardware_capability|string|Minimum CUDA hardware capability.|Optional.|

diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 7e68dcd3314685bf4132739bb17ff0ad2e9ba7b8..52d9c4b0f1959262f7a57094d55c0991c22315e1 100644 (file)
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -988,11 +988,11 @@ func (runner *ContainerRunner) CreateContainer(imageID string, bindmounts map[st
        runner.executorStderr = stderr
 
        cudaDeviceCount := 0
-       if runner.Container.RuntimeConstraints.CUDADriverVersion != "" ||
-               runner.Container.RuntimeConstraints.CUDAHardwareCapability != "" ||
-               runner.Container.RuntimeConstraints.CUDADeviceCount != 0 {
+       if runner.Container.RuntimeConstraints.CUDA.DriverVersion != "" ||
+               runner.Container.RuntimeConstraints.CUDA.HardwareCapability != "" ||
+               runner.Container.RuntimeConstraints.CUDA.DeviceCount != 0 {
                // if any of these are set, enable CUDA GPU support
-               cudaDeviceCount = runner.Container.RuntimeConstraints.CUDADeviceCount
+               cudaDeviceCount = runner.Container.RuntimeConstraints.CUDA.DeviceCount
                if cudaDeviceCount == 0 {
                        cudaDeviceCount = 1
                }

diff --git a/lib/dispatchcloud/node_size.go b/lib/dispatchcloud/node_size.go
index aa2cd7d5696d46a7184b2edbb39d45b6d4149ead..6831d8d8e9a50268ebc9861fc41b73255040b65d 100644 (file)
--- a/lib/dispatchcloud/node_size.go
+++ b/lib/dispatchcloud/node_size.go
@@ -83,8 +83,8 @@ func EstimateScratchSpace(ctr *arvados.Container) (needScratch int64) {
        return
 }
 
-// compareVersion returns true if vs1 >= vs2, otherwise false
-func compareVersion(vs1 string, vs2 string) bool {
+// compareVersion returns true if vs1 < vs2, otherwise false
+func versionLess(vs1 string, vs2 string) bool {
        v1, err := strconv.ParseFloat(vs1, 64)
        if err != nil {
                return false
@@ -93,7 +93,7 @@ func compareVersion(vs1 string, vs2 string) bool {
        if err != nil {
                return false
        }
-       return v1 >= v2
+       return v1 < v2
 }
 
 // ChooseInstanceType returns the cheapest available
@@ -123,9 +123,9 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
                case it.VCPUs < needVCPUs: // insufficient VCPUs
                case it.Preemptible != ctr.SchedulingParameters.Preemptible: // wrong preemptable setting
                case it.Price == best.Price && (it.RAM < best.RAM || it.VCPUs < best.VCPUs): // same price, worse specs
-               case it.CUDA.DeviceCount < ctr.RuntimeConstraints.CUDADeviceCount: // insufficient CUDA devices
-               case it.CUDA.DeviceCount > 0 && !compareVersion(it.CUDA.DriverVersion, ctr.RuntimeConstraints.CUDADriverVersion): // insufficient driver version
-               case it.CUDA.DeviceCount > 0 && !compareVersion(it.CUDA.HardwareCapability, ctr.RuntimeConstraints.CUDAHardwareCapability): // insufficient hardware capability
+               case it.CUDA.DeviceCount < ctr.RuntimeConstraints.CUDA.DeviceCount: // insufficient CUDA devices
+               case ctr.RuntimeConstraints.CUDA.DeviceCount > 0 && versionLess(it.CUDA.DriverVersion, ctr.RuntimeConstraints.CUDA.DriverVersion): // insufficient driver version
+               case ctr.RuntimeConstraints.CUDA.DeviceCount > 0 && versionLess(it.CUDA.HardwareCapability, ctr.RuntimeConstraints.CUDA.HardwareCapability): // insufficient hardware capability
                        // Don't select this node
                default:
                        // Didn't reject the node, so select it

diff --git a/lib/dispatchcloud/node_size_test.go b/lib/dispatchcloud/node_size_test.go
index cdcf4033fcc629317ef0f6edddf799778bf50e5b..06255462b318621e55f7dc883e74956b5fdfaf73 100644 (file)
--- a/lib/dispatchcloud/node_size_test.go
+++ b/lib/dispatchcloud/node_size_test.go
@@ -154,69 +154,94 @@ func (*NodeSizeSuite) TestChooseGPU(c *check.C) {
                "low_capability": {Price: 2.1, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "low_capability", CUDA: arvados.CUDAFeatures{DeviceCount: 1, HardwareCapability: "8.0", DriverVersion: "11.0"}},
                "best":           {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best", CUDA: arvados.CUDAFeatures{DeviceCount: 1, HardwareCapability: "9.0", DriverVersion: "11.0"}},
                "low_driver":     {Price: 2.1, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "low_driver", CUDA: arvados.CUDAFeatures{DeviceCount: 1, HardwareCapability: "9.0", DriverVersion: "10.0"}},
-               "small":          {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
+               "cheap_gpu":      {Price: 2.0, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "cheap_gpu", CUDA: arvados.CUDAFeatures{DeviceCount: 1, HardwareCapability: "8.0", DriverVersion: "10.0"}},
+               "non_gpu":        {Price: 1.1, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "non_gpu"},
        }
-       best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
-               Mounts: map[string]arvados.Mount{
-                       "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+
+       type GPUTestCase struct {
+               CUDA             arvados.CUDARuntimeConstraints
+               SelectedInstance string
+       }
+       cases := []GPUTestCase{
+               GPUTestCase{
+                       CUDA: arvados.CUDARuntimeConstraints{
+                               DeviceCount:        1,
+                               HardwareCapability: "9.0",
+                               DriverVersion:      "11.0",
+                       },
+                       SelectedInstance: "best",
                },
-               RuntimeConstraints: arvados.RuntimeConstraints{
-                       VCPUs:                  2,
-                       RAM:                    987654321,
-                       KeepCacheRAM:           123456789,
-                       CUDADeviceCount:        1,
-                       CUDAHardwareCapability: "9.0",
-                       CUDADriverVersion:      "11.0",
+               GPUTestCase{
+                       CUDA: arvados.CUDARuntimeConstraints{
+                               DeviceCount:        2,
+                               HardwareCapability: "9.0",
+                               DriverVersion:      "11.0",
+                       },
+                       SelectedInstance: "costly",
                },
-       })
-       c.Check(err, check.IsNil)
-       c.Check(best.Name, check.Equals, "best")
-       c.Check(best.RAM >= 1234567890, check.Equals, true)
-       c.Check(best.VCPUs >= 2, check.Equals, true)
-       c.Check(best.CUDA.DeviceCount >= 1, check.Equals, true)
-       c.Check(best.CUDA.DriverVersion, check.Equals, "11.0")
-       c.Check(best.CUDA.HardwareCapability, check.Equals, "9.0")
-
-       best, err = ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
-               Mounts: map[string]arvados.Mount{
-                       "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+               GPUTestCase{
+                       CUDA: arvados.CUDARuntimeConstraints{
+                               DeviceCount:        1,
+                               HardwareCapability: "8.0",
+                               DriverVersion:      "11.0",
+                       },
+                       SelectedInstance: "low_capability",
                },
-               RuntimeConstraints: arvados.RuntimeConstraints{
-                       VCPUs:                  2,
-                       RAM:                    987654321,
-                       KeepCacheRAM:           123456789,
-                       CUDADeviceCount:        2,
-                       CUDAHardwareCapability: "9.0",
-                       CUDADriverVersion:      "11.0",
+               GPUTestCase{
+                       CUDA: arvados.CUDARuntimeConstraints{
+                               DeviceCount:        1,
+                               HardwareCapability: "9.0",
+                               DriverVersion:      "10.0",
+                       },
+                       SelectedInstance: "low_driver",
                },
-       })
-       c.Check(err, check.IsNil)
-       c.Check(best.Name, check.Equals, "costly")
-       c.Check(best.RAM >= 1234567890, check.Equals, true)
-       c.Check(best.VCPUs >= 2, check.Equals, true)
-       c.Check(best.CUDA.DeviceCount >= 2, check.Equals, true)
-       c.Check(best.CUDA.DriverVersion, check.Equals, "11.0")
-       c.Check(best.CUDA.HardwareCapability, check.Equals, "9.0")
-
-       best, err = ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
-               Mounts: map[string]arvados.Mount{
-                       "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+               GPUTestCase{
+                       CUDA: arvados.CUDARuntimeConstraints{
+                               DeviceCount:        1,
+                               HardwareCapability: "",
+                               DriverVersion:      "10.0",
+                       },
+                       SelectedInstance: "cheap_gpu",
                },
-               RuntimeConstraints: arvados.RuntimeConstraints{
-                       VCPUs:                  2,
-                       RAM:                    987654321,
-                       KeepCacheRAM:           123456789,
-                       CUDADeviceCount:        1,
-                       CUDAHardwareCapability: "8.0",
-                       CUDADriverVersion:      "11.0",
+               GPUTestCase{
+                       CUDA: arvados.CUDARuntimeConstraints{
+                               DeviceCount:        1,
+                               HardwareCapability: "8.0",
+                               DriverVersion:      "",
+                       },
+                       SelectedInstance: "cheap_gpu",
                },
-       })
-       c.Check(err, check.IsNil)
-       c.Check(best.Name, check.Equals, "low_capability")
-       c.Check(best.RAM >= 1234567890, check.Equals, true)
-       c.Check(best.VCPUs >= 2, check.Equals, true)
-       c.Check(best.CUDA.DeviceCount >= 1, check.Equals, true)
-       c.Check(best.CUDA.DriverVersion, check.Equals, "11.0")
-       c.Check(best.CUDA.HardwareCapability, check.Equals, "8.0")
+               GPUTestCase{
+                       CUDA: arvados.CUDARuntimeConstraints{
+                               DeviceCount:        1,
+                               HardwareCapability: "",
+                               DriverVersion:      "",
+                       },
+                       SelectedInstance: "cheap_gpu",
+               },
+               GPUTestCase{
+                       CUDA: arvados.CUDARuntimeConstraints{
+                               DeviceCount:        0,
+                               HardwareCapability: "9.0",
+                               DriverVersion:      "11.0",
+                       },
+                       SelectedInstance: "non_gpu",
+               },
+       }
 
+       for _, tc := range cases {
+               best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
+                       Mounts: map[string]arvados.Mount{
+                               "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+                       },
+                       RuntimeConstraints: arvados.RuntimeConstraints{
+                               VCPUs:        2,
+                               RAM:          987654321,
+                               KeepCacheRAM: 123456789,
+                               CUDA:         tc.CUDA,
+                       },
+               })
+               c.Check(err, check.IsNil)
+               c.Check(best.Name, check.Equals, tc.SelectedInstance)
+       }
 }

diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index 27afc1a3abbb31f380a20fdfb6110a5300168afb..d0b12733340080418039facd27cd0133e650a234 100644 (file)
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -93,16 +93,20 @@ type Mount struct {
        GitURL            string      `json:"git_url"`         // only if kind=="git_tree"
 }
 
+type CUDARuntimeConstraints struct {
+       DriverVersion      string `json:"driver_version,omitempty"`
+       HardwareCapability string `json:"hardware_capability,omitempty"`
+       DeviceCount        int    `json:"device_count,omitempty"`
+}
+
 // RuntimeConstraints specify a container's compute resources (RAM,
 // CPU) and network connectivity.
 type RuntimeConstraints struct {
-       API                    bool   `json:"API"`
-       RAM                    int64  `json:"ram"`
-       VCPUs                  int    `json:"vcpus"`
-       KeepCacheRAM           int64  `json:"keep_cache_ram"`
-       CUDADriverVersion      string `json:"cuda_driver_version,omitempty"`
-       CUDAHardwareCapability string `json:"cuda_hardware_capability,omitempty"`
-       CUDADeviceCount        int    `json:"cuda_device_count,omitempty"`
+       API          bool                   `json:"API"`
+       RAM          int64                  `json:"ram"`
+       VCPUs        int                    `json:"vcpus"`
+       KeepCacheRAM int64                  `json:"keep_cache_ram"`
+       CUDA         CUDARuntimeConstraints `json:"cuda,omitempty"`
 }
 
 // SchedulingParameters specify a container's scheduling parameters