Merge branch '18842-arv-mount-disk-config' refs #18842

[arvados.git] / lib / dispatchcloud / node_size.go

diff --git a/lib/dispatchcloud/node_size.go b/lib/dispatchcloud/node_size.go
index 6fb46b5f46f9d36c2500ee759cee1a3ec19c59f0..0b394f4cfe4f76849fc2eb42541ed613e325921f 100644 (file)
--- a/lib/dispatchcloud/node_size.go
+++ b/lib/dispatchcloud/node_size.go
@@ -10,7 +10,7 @@ import (
        "sort"
        "strconv"
 
-       "git.curoverse.com/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvados"
 )
 
 var ErrInstanceTypesNotConfigured = errors.New("site configuration does not list any instance types")
@@ -56,7 +56,7 @@ func estimateDockerImageSize(collectionPDH string) int64 {
 // EstimateScratchSpace estimates how much available disk space (in
 // bytes) is needed to run the container by summing the capacity
 // requested by 'tmp' mounts plus disk space required to load the
-// Docker image.
+// Docker image plus arv-mount block cache.
 func EstimateScratchSpace(ctr *arvados.Container) (needScratch int64) {
        for _, m := range ctr.Mounts {
                if m.Kind == "tmp" {
@@ -80,9 +80,25 @@ func EstimateScratchSpace(ctr *arvados.Container) (needScratch int64) {
        // Now reserve space for the extracted image on disk.
        needScratch += dockerImageSize
 
+       // Now reserve space the arv-mount disk cache
+       needScratch += ctr.RuntimeConstraints.KeepCacheDisk
+
        return
 }
 
+// compareVersion returns true if vs1 < vs2, otherwise false
+func versionLess(vs1 string, vs2 string) (bool, error) {
+       v1, err := strconv.ParseFloat(vs1, 64)
+       if err != nil {
+               return false, err
+       }
+       v2, err := strconv.ParseFloat(vs2, 64)
+       if err != nil {
+               return false, err
+       }
+       return v1 < v2, nil
+}
+
 // ChooseInstanceType returns the cheapest available
 // arvados.InstanceType big enough to run ctr.
 func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvados.InstanceType, err error) {
@@ -96,19 +112,34 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
        needVCPUs := ctr.RuntimeConstraints.VCPUs
 
        needRAM := ctr.RuntimeConstraints.RAM + ctr.RuntimeConstraints.KeepCacheRAM
+       needRAM += int64(cc.Containers.ReserveExtraRAM)
+       if cc.Containers.LocalKeepBlobBuffersPerVCPU > 0 {
+               // + 200 MiB for keepstore process + 10% for GOGC=10
+               needRAM += 220 << 20
+               // + 64 MiB for each blob buffer + 10% for GOGC=10
+               needRAM += int64(cc.Containers.LocalKeepBlobBuffersPerVCPU * needVCPUs * (1 << 26) * 11 / 10)
+       }
        needRAM = (needRAM * 100) / int64(100-discountConfiguredRAMPercent)
 
        ok := false
        for _, it := range cc.InstanceTypes {
+               driverInsuff, driverErr := versionLess(it.CUDA.DriverVersion, ctr.RuntimeConstraints.CUDA.DriverVersion)
+               capabilityInsuff, capabilityErr := versionLess(it.CUDA.HardwareCapability, ctr.RuntimeConstraints.CUDA.HardwareCapability)
+
                switch {
-               case ok && it.Price > best.Price:
-               case int64(it.Scratch) < needScratch:
-               case int64(it.RAM) < needRAM:
-               case it.VCPUs < needVCPUs:
-               case it.Preemptible != ctr.SchedulingParameters.Preemptible:
-               case it.Price == best.Price && (it.RAM < best.RAM || it.VCPUs < best.VCPUs):
-                       // Equal price, but worse specs
+               // reasons to reject a node
+               case ok && it.Price > best.Price: // already selected a node, and this one is more expensive
+               case int64(it.Scratch) < needScratch: // insufficient scratch
+               case int64(it.RAM) < needRAM: // insufficient RAM
+               case it.VCPUs < needVCPUs: // insufficient VCPUs
+               case it.Preemptible != ctr.SchedulingParameters.Preemptible: // wrong preemptable setting
+               case it.Price == best.Price && (it.RAM < best.RAM || it.VCPUs < best.VCPUs): // same price, worse specs
+               case it.CUDA.DeviceCount < ctr.RuntimeConstraints.CUDA.DeviceCount: // insufficient CUDA devices
+               case ctr.RuntimeConstraints.CUDA.DeviceCount > 0 && (driverInsuff || driverErr != nil): // insufficient driver version
+               case ctr.RuntimeConstraints.CUDA.DeviceCount > 0 && (capabilityInsuff || capabilityErr != nil): // insufficient hardware capability
+                       // Don't select this node
                default:
+                       // Didn't reject the node, so select it
                        // Lower price || (same price && better specs)
                        best = it
                        ok = true