13219: Checks for expired run time and cancel container if needed.

Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima@veritasgenetics.com>

diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index 5398d9d74128cd1d941194d37b3ba10eb71a5942..1bdf0890230e96c27d9f77ae0239bfdd3b65b403 100644 (file)
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -20,6 +20,7 @@ type Container struct {
        OutputPath           string               `json:"output_path"`
        Priority             int                  `json:"priority"`
        RuntimeConstraints   RuntimeConstraints   `json:"runtime_constraints"`
+       StartedAt            time.Time            `json:"started_at"`
        State                ContainerState       `json:"state"`
        SchedulingParameters SchedulingParameters `json:"scheduling_parameters"`
 }
@@ -54,6 +55,7 @@ type RuntimeConstraints struct {
 type SchedulingParameters struct {
        Partitions  []string `json:"partitions"`
        Preemptible bool     `json:"preemptible"`
+       MaxRunTime  int      `json:"max_run_time"`
 }
 
 // ContainerList is an arvados#containerList resource.

diff --git a/sdk/go/dispatch/dispatch.go b/sdk/go/dispatch/dispatch.go
index 3289c67b013f37a67ae8ddeaa52d3fd74abe34e5..ca2dbc48d4053f4e3036f0db6bcdaf4e03a0e064 100644 (file)
--- a/sdk/go/dispatch/dispatch.go
+++ b/sdk/go/dispatch/dispatch.go
@@ -195,6 +195,13 @@ func (d *Dispatcher) checkListForUpdates(containers []arvados.Container, todo ma
                        case Queued:
                                tracker.close()
                        case Locked, Running:
+                               if c.SchedulingParameters.MaxRunTime > 0 {
+                                       maxRunTime := time.Duration(c.SchedulingParameters.MaxRunTime) * time.Second
+                                       if time.Since(c.StartedAt) >= maxRunTime {
+                                               // Time's up, schedule container for cancellation
+                                               c.Priority = 0
+                                       }
+                               }
                                tracker.update(c)
                        case Cancelled, Complete:
                                tracker.close()