1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
15 "git.curoverse.com/arvados.git/sdk/go/arvados"
19 ErrInstanceTypesNotConfigured = errors.New("site configuration does not list any instance types")
20 discountConfiguredRAMPercent = 5
23 // ConstraintsNotSatisfiableError includes a list of available instance types
24 // to be reported back to the user.
25 type ConstraintsNotSatisfiableError struct {
27 AvailableTypes []arvados.InstanceType
30 // ChooseInstanceType returns the cheapest available
31 // arvados.InstanceType big enough to run ctr.
32 func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvados.InstanceType, err error) {
33 if len(cc.InstanceTypes) == 0 {
34 err = ErrInstanceTypesNotConfigured
38 needScratch := int64(0)
39 for _, m := range ctr.Mounts {
41 needScratch += m.Capacity
45 needVCPUs := ctr.RuntimeConstraints.VCPUs
47 needRAM := ctr.RuntimeConstraints.RAM + ctr.RuntimeConstraints.KeepCacheRAM
48 needRAM = (needRAM * 100) / int64(100-discountConfiguredRAMPercent)
50 availableTypes := make([]arvados.InstanceType, 0, len(cc.InstanceTypes))
51 for _, t := range cc.InstanceTypes {
52 availableTypes = append(availableTypes, t)
54 sort.Slice(availableTypes, func(a, b int) bool {
55 return availableTypes[a].Price < availableTypes[b].Price
57 err = ConstraintsNotSatisfiableError{
58 errors.New("constraints not satisfiable by any configured instance type"),
61 for _, it := range cc.InstanceTypes {
63 case err == nil && it.Price > best.Price:
64 case it.Scratch < needScratch:
65 case it.RAM < needRAM:
66 case it.VCPUs < needVCPUs:
67 case it.Preemptible != ctr.SchedulingParameters.Preemptible:
68 case it.Price == best.Price && (it.RAM < best.RAM || it.VCPUs < best.VCPUs):
69 // Equal price, but worse specs
71 // Lower price || (same price && better specs)
79 // SlurmNodeTypeFeatureKludge ensures SLURM accepts every instance
80 // type name as a valid feature name, even if no instances of that
81 // type have appeared yet.
83 // It takes advantage of some SLURM peculiarities:
85 // (1) A feature is valid after it has been offered by a node, even if
86 // it is no longer offered by any node. So, to make a feature name
87 // valid, we can add it to a dummy node ("compute0"), then remove it.
89 // (2) To test whether a set of feature names are valid without
90 // actually submitting a job, we can call srun --test-only with the
93 // SlurmNodeTypeFeatureKludge does a test-and-fix operation
94 // immediately, and then periodically, in case slurm restarts and
95 // forgets the list of valid features. It never returns (unless there
96 // are no node types configured, in which case it returns
97 // immediately), so it should generally be invoked with "go".
98 func SlurmNodeTypeFeatureKludge(cc *arvados.Cluster) {
99 if len(cc.InstanceTypes) == 0 {
102 var features []string
103 for _, it := range cc.InstanceTypes {
104 features = append(features, "instancetype="+it.Name)
107 slurmKludge(features)
108 time.Sleep(2 * time.Second)
112 const slurmDummyNode = "compute0"
114 func slurmKludge(features []string) {
115 allFeatures := strings.Join(features, ",")
117 cmd := exec.Command("sinfo", "--nodes="+slurmDummyNode, "--format=%f", "--noheader")
118 out, err := cmd.CombinedOutput()
120 log.Printf("running %q %q: %s (output was %q)", cmd.Path, cmd.Args, err, out)
123 if string(out) == allFeatures+"\n" {
124 // Already configured correctly, nothing to do.
128 log.Printf("configuring node %q with all node type features", slurmDummyNode)
129 cmd = exec.Command("scontrol", "update", "NodeName="+slurmDummyNode, "Features="+allFeatures)
130 log.Printf("running: %q %q", cmd.Path, cmd.Args)
131 out, err = cmd.CombinedOutput()
133 log.Printf("error: scontrol: %s (output was %q)", err, out)