-// If the container is marked as Locked, check if it is already in the slurm
-// queue. If not, submit it.
-//
-// If the container is marked as Running, check if it is in the slurm queue.
-// If not, mark it as Cancelled.
-func monitorSubmitOrCancel(dispatcher *dispatch.Dispatcher, container arvados.Container, monitorDone *bool) {
- submitted := false
- for !*monitorDone {
- if sqCheck.HasUUID(container.UUID) {
- // Found in the queue, so continue monitoring
- submitted = true
- } else if container.State == dispatch.Locked && !submitted {
- // Not in queue but in Locked state and we haven't
- // submitted it yet, so submit it.
-
- log.Printf("About to submit queued container %v", container.UUID)
-
- if err := submit(dispatcher, container, theConfig.CrunchRunCommand); err != nil {
- log.Printf("Error submitting container %s to slurm: %v",
- container.UUID, err)
- // maybe sbatch is broken, put it back to queued
- dispatcher.Unlock(container.UUID)
- }
- submitted = true
- } else {
- // Not in queue and we are not going to submit it.
- // Refresh the container state. If it is
- // Complete/Cancelled, do nothing, if it is Locked then
- // release it back to the Queue, if it is Running then
- // clean up the record.
-
- var con arvados.Container
- err := dispatcher.Arv.Get("containers", container.UUID, nil, &con)
- if err != nil {
- log.Printf("Error getting final container state: %v", err)
- }
+// Submit a container to the slurm queue (or resume monitoring if it's
+// already in the queue). Cancel the slurm job if the container's
+// priority changes to zero or its state indicates it's no longer
+// running.
+func run(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ if ctr.State == dispatch.Locked && !sqCheck.HasUUID(ctr.UUID) {
+ log.Printf("Submitting container %s to slurm", ctr.UUID)
+ if err := submit(disp, ctr, theConfig.CrunchRunCommand); err != nil {
+ text := fmt.Sprintf("Error submitting container %s to slurm: %s", ctr.UUID, err)
+ log.Print(text)
+
+ lr := arvadosclient.Dict{"log": arvadosclient.Dict{
+ "object_uuid": ctr.UUID,
+ "event_type": "dispatch",
+ "properties": map[string]string{"text": text}}}
+ disp.Arv.Create("logs", lr, nil)
+
+ disp.Unlock(ctr.UUID)
+ return
+ }
+ }