13078: Add test case for "release job held because BadConstraints".

[arvados.git] / services / crunch-dispatch-slurm / squeue.go

diff --git a/services/crunch-dispatch-slurm/squeue.go b/services/crunch-dispatch-slurm/squeue.go
index d15489b278ec6e84c17f944d9e463920e3e17cb0..365b4e85706ccd46f8a77ad30af2700cfaf1ac60 100644 (file)
--- a/services/crunch-dispatch-slurm/squeue.go
+++ b/services/crunch-dispatch-slurm/squeue.go
@@ -51,17 +51,19 @@ func (sqc *SqueueChecker) HasUUID(uuid string) bool {
 // SetPriority sets or updates the desired (Arvados) priority for a
 // container.
 func (sqc *SqueueChecker) SetPriority(uuid string, want int64) {
+       sqc.startOnce.Do(sqc.start)
        sqc.L.Lock()
        defer sqc.L.Unlock()
-       if _, ok := sqc.queue[uuid]; !ok {
+       job, ok := sqc.queue[uuid]
+       if !ok {
                // Wait in case the slurm job was just submitted and
                // will appear in the next squeue update.
                sqc.Wait()
-               if _, ok = sqc.queue[uuid]; !ok {
+               if job, ok = sqc.queue[uuid]; !ok {
                        return
                }
        }
-       sqc.queue[uuid].wantPriority = want
+       job.wantPriority = want
 }
 
 // adjust slurm job nice values as needed to ensure slurm priority
@@ -77,6 +79,14 @@ func (sqc *SqueueChecker) reniceAll() {
                        // (perhaps it's not an Arvados job)
                        continue
                }
+               if j.priority == 0 {
+                       // SLURM <= 15.x implements "hold" by setting
+                       // priority to 0. If we include held jobs
+                       // here, we'll end up trying to push other
+                       // jobs below them using negative priority,
+                       // which won't help anything.
+                       continue
+               }
                jobs = append(jobs, j)
        }
 
@@ -88,7 +98,6 @@ func (sqc *SqueueChecker) reniceAll() {
                if renice[i] == job.nice {
                        continue
                }
-               log.Printf("updating slurm priority for %q: nice %d => %d", job.uuid, job.nice, renice[i])
                sqc.Slurm.Renice(job.uuid, renice[i])
        }
 }
@@ -112,7 +121,7 @@ func (sqc *SqueueChecker) check() {
        sqc.L.Lock()
        defer sqc.L.Unlock()
 
-       cmd := sqc.Slurm.QueueCommand([]string{"--all", "--format=%j %y %Q"})
+       cmd := sqc.Slurm.QueueCommand([]string{"--all", "--noheader", "--format=%j %y %Q %T %r"})
        stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
        cmd.Stdout, cmd.Stderr = stdout, stderr
        if err := cmd.Run(); err != nil {
@@ -126,16 +135,35 @@ func (sqc *SqueueChecker) check() {
                if line == "" {
                        continue
                }
-               var uuid string
+               var uuid, state, reason string
                var n, p int64
-               if _, err := fmt.Sscan(line, &uuid, &n, &p); err != nil {
+               if _, err := fmt.Sscan(line, &uuid, &n, &p, &state, &reason); err != nil {
                        log.Printf("warning: ignoring unparsed line in squeue output: %q", line)
                        continue
                }
-               newq[uuid] = &slurmJob{
-                       uuid:     uuid,
-                       priority: p,
-                       nice:     n,
+               replacing, ok := sqc.queue[uuid]
+               if !ok {
+                       replacing = &slurmJob{uuid: uuid}
+               }
+               replacing.priority = p
+               replacing.nice = n
+               newq[uuid] = replacing
+
+               if state == "PENDING" && reason == "BadConstraints" && p == 0 && replacing.wantPriority > 0 {
+                       // When using SLURM 14.x or 15.x, our queued
+                       // jobs land in this state when "scontrol
+                       // reconfigure" invalidates their feature
+                       // constraints by clearing all node features.
+                       // They stay in this state even after the
+                       // features reappear, until we run "scontrol
+                       // release {jobid}".
+                       //
+                       // "scontrol release" is silent and successful
+                       // regardless of whether the features have
+                       // reappeared, so rather than second-guessing
+                       // whether SLURM is ready, we just keep trying
+                       // this until it works.
+                       sqc.Slurm.Release(uuid)
                }
        }
        sqc.queue = newq