Merge branch '12737-update-rails-api'

[arvados.git] / services / crunch-dispatch-slurm / slurm.go

diff --git a/services/crunch-dispatch-slurm/slurm.go b/services/crunch-dispatch-slurm/slurm.go
index f675f6c4d8dbf65e68332c446270a6f02ab9876e..9e9f45270f82d3450d27e380fe63924177e5501d 100644 (file)
--- a/services/crunch-dispatch-slurm/slurm.go
+++ b/services/crunch-dispatch-slurm/slurm.go
@@ -13,10 +13,11 @@ import (
 )
 
 type Slurm interface {
+       Batch(script io.Reader, args []string) error
        Cancel(name string) error
-       Renice(name string, nice int) error
        QueueCommand(args []string) *exec.Cmd
-       Batch(script io.Reader, args []string) error
+       Release(name string) error
+       Renice(name string, nice int64) error
 }
 
 type slurmCLI struct{}
@@ -26,14 +27,39 @@ func (scli *slurmCLI) Batch(script io.Reader, args []string) error {
 }
 
 func (scli *slurmCLI) Cancel(name string) error {
-       return scli.run(nil, "scancel", []string{"--name=" + name})
+       for _, args := range [][]string{
+               // If the slurm job hasn't started yet, remove it from
+               // the queue.
+               {"--state=pending"},
+               // If the slurm job has started, send SIGTERM. If we
+               // cancel a running job without a --signal argument,
+               // slurm will send SIGTERM and then (after some
+               // site-configured interval) SIGKILL. This would kill
+               // crunch-run without stopping the container, which we
+               // don't want.
+               {"--batch", "--signal=TERM", "--state=running"},
+               {"--batch", "--signal=TERM", "--state=suspended"},
+       } {
+               err := scli.run(nil, "scancel", append([]string{"--name=" + name}, args...))
+               if err != nil {
+                       // scancel exits 0 if no job matches the given
+                       // name and state. Any error from scancel here
+                       // really indicates something is wrong.
+                       return err
+               }
+       }
+       return nil
 }
 
 func (scli *slurmCLI) QueueCommand(args []string) *exec.Cmd {
        return exec.Command("squeue", args...)
 }
 
-func (scli *slurmCLI) Renice(name string, nice int) error {
+func (scli *slurmCLI) Release(name string) error {
+       return scli.run(nil, "scontrol", []string{"release", "Name=" + name})
+}
+
+func (scli *slurmCLI) Renice(name string, nice int64) error {
        return scli.run(nil, "scontrol", []string{"update", "JobName=" + name, fmt.Sprintf("Nice=%d", nice)})
 }