limit concurrent slurm commands

use a semaphore channel to limit concurrent sbatch/scancel/scontrol commands to 3.
squeue is already limited to one at a time.

fixes 14110

Arvados-DCO-1.1-Signed-off-by: Joshua C. Randall <jcrandall@alum.mit.edu>

diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 36ef264963d760f04501fc25cee6916c62ef4bf2..16d9fd18db2765adb6aa05ab1d742660f51790d8 100644 (file)
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -159,7 +159,7 @@ func (disp *Dispatcher) setup() {
        }
        arv.Retries = 25
 
-       disp.slurm = &slurmCLI{}
+       disp.slurm = NewSlurmCLI()
        disp.sqCheck = &SqueueChecker{
                Period:         time.Duration(disp.PollPeriod),
                PrioritySpread: disp.PrioritySpread,

diff --git a/services/crunch-dispatch-slurm/slurm.go b/services/crunch-dispatch-slurm/slurm.go
index 9e9f45270f82d3450d27e380fe63924177e5501d..782be7d8c4e7226ced947bcd26c983ecb6b31de3 100644 (file)
--- a/services/crunch-dispatch-slurm/slurm.go
+++ b/services/crunch-dispatch-slurm/slurm.go
@@ -20,7 +20,15 @@ type Slurm interface {
        Renice(name string, nice int64) error
 }
 
-type slurmCLI struct{}
+type slurmCLI struct{
+       runSemaphore chan bool
+}
+
+func NewSlurmCLI() *slurmCLI {
+       return &slurmCLI{
+              runSemaphore: make(chan bool, 3),
+       }
+}
 
 func (scli *slurmCLI) Batch(script io.Reader, args []string) error {
        return scli.run(script, "sbatch", args)
@@ -64,6 +72,8 @@ func (scli *slurmCLI) Renice(name string, nice int64) error {
 }
 
 func (scli *slurmCLI) run(stdin io.Reader, prog string, args []string) error {
+       scli.runSemaphore <- true
+       defer func() { <-scli.runSemaphore }()
        cmd := exec.Command(prog, args...)
        cmd.Stdin = stdin
        out, err := cmd.CombinedOutput()