1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
15 "git.curoverse.com/arvados.git/sdk/go/arvados"
19 ErrConstraintsNotSatisfiable = errors.New("constraints not satisfiable by any configured instance type")
20 ErrInstanceTypesNotConfigured = errors.New("site configuration does not list any instance types")
23 // ChooseInstanceType returns the cheapest available
24 // arvados.InstanceType big enough to run ctr.
25 func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvados.InstanceType, err error) {
26 needVCPUs := ctr.RuntimeConstraints.VCPUs
27 needRAM := ctr.RuntimeConstraints.RAM + ctr.RuntimeConstraints.KeepCacheRAM
29 if len(cc.InstanceTypes) == 0 {
30 err = ErrInstanceTypesNotConfigured
34 err = ErrConstraintsNotSatisfiable
35 for _, it := range cc.InstanceTypes {
37 case err == nil && it.Price > best.Price:
38 case it.RAM < needRAM:
39 case it.VCPUs < needVCPUs:
40 case it.Price == best.Price && (it.RAM < best.RAM || it.VCPUs < best.VCPUs):
41 // Equal price, but worse specs
43 // Lower price || (same price && better specs)
51 // SlurmNodeTypeFeatureKludge ensures SLURM accepts every instance
52 // type name as a valid feature name, even if no instances of that
53 // type have appeared yet.
55 // It takes advantage of some SLURM peculiarities:
57 // (1) A feature is valid after it has been offered by a node, even if
58 // it is no longer offered by any node. So, to make a feature name
59 // valid, we can add it to a dummy node ("compute0"), then remove it.
61 // (2) when srun is given an invalid --gres argument and an invalid
62 // --constraint argument, the error message mentions "Invalid feature
63 // specification". So, to test whether a feature name is valid without
64 // actually submitting a job, we can call srun with the feature name
65 // and an invalid --gres argument.
67 // SlurmNodeTypeFeatureKludge does a test-and-fix operation
68 // immediately, and then periodically, in case slurm restarts and
69 // forgets the list of valid features. It never returns (unless there
70 // are no node types configured, in which case it returns
71 // immediately), so it should generally be invoked with "go".
72 func SlurmNodeTypeFeatureKludge(cc *arvados.Cluster) {
73 if len(cc.InstanceTypes) == 0 {
77 for _, it := range cc.InstanceTypes {
78 features = append(features, "instancetype="+it.Name)
82 time.Sleep(time.Minute)
87 slurmDummyNode = "compute0"
88 slurmErrBadFeature = "Invalid feature"
89 slurmErrBadGres = "Invalid generic resource"
92 func slurmKludge(features []string) {
93 cmd := exec.Command("srun", "--gres=invalid-gres-specification", "--constraint="+strings.Join(features, "&"), "true")
94 out, err := cmd.CombinedOutput()
97 log.Printf("warning: guaranteed-to-fail srun command did not fail: %q %q", cmd.Path, cmd.Args)
98 log.Printf("output was: %q", out)
100 case bytes.Contains(out, []byte(slurmErrBadFeature)):
101 log.Printf("temporarily configuring node %q with all node type features", slurmDummyNode)
102 for _, nodeFeatures := range []string{strings.Join(features, ","), ""} {
103 cmd = exec.Command("scontrol", "update", "NodeName="+slurmDummyNode, "Features="+nodeFeatures)
104 log.Printf("running: %q %q", cmd.Path, cmd.Args)
105 out, err := cmd.CombinedOutput()
107 log.Printf("error: scontrol: %s (output was %q)", err, out)
111 case bytes.Contains(out, []byte(slurmErrBadGres)):
112 // Evidently our node-type feature names are all valid.
115 log.Printf("warning: expected srun error %q or %q, but output was %q", slurmErrBadFeature, slurmErrBadGres, out)