X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/3facf89bf048487ee718fe15d012b489f2d407b7..ac39afed6cd3de1704d75aecdbc46544b02f02b2:/lib/dispatchcloud/test/queue.go

diff --git a/lib/dispatchcloud/test/queue.go b/lib/dispatchcloud/test/queue.go
index 3598ec6da0..ea2b98236f 100644
--- a/lib/dispatchcloud/test/queue.go
+++ b/lib/dispatchcloud/test/queue.go
@@ -22,17 +22,34 @@ type Queue struct {
 
 	// ChooseType will be called for each entry in Containers. It
 	// must not be nil.
-	ChooseType func(*arvados.Container) (arvados.InstanceType, error)
+	ChooseType func(*arvados.Container) ([]arvados.InstanceType, error)
+
+	// Mimic railsapi implementation of MaxDispatchAttempts config
+	MaxDispatchAttempts int
 
 	Logger logrus.FieldLogger
 
-	entries     map[string]container.QueueEnt
-	updTime     time.Time
-	subscribers map[<-chan struct{}]chan struct{}
+	entries      map[string]container.QueueEnt
+	updTime      time.Time
+	subscribers  map[<-chan struct{}]chan struct{}
+	stateChanges []QueueStateChange
 
 	mtx sync.Mutex
 }
 
+type QueueStateChange struct {
+	UUID string
+	From arvados.ContainerState
+	To   arvados.ContainerState
+}
+
+// All calls to Lock/Unlock/Cancel to date.
+func (q *Queue) StateChanges() []QueueStateChange {
+	q.mtx.Lock()
+	defer q.mtx.Unlock()
+	return q.stateChanges
+}
+
 // Entries returns the containers that were queued when Update was
 // last called.
 func (q *Queue) Entries() (map[string]container.QueueEnt, time.Time) {
@@ -111,6 +128,7 @@ func (q *Queue) notify() {
 // caller must have lock.
 func (q *Queue) changeState(uuid string, from, to arvados.ContainerState) error {
 	ent := q.entries[uuid]
+	q.stateChanges = append(q.stateChanges, QueueStateChange{uuid, from, to})
 	if ent.Container.State != from {
 		return fmt.Errorf("changeState failed: state=%q", ent.Container.State)
 	}
@@ -118,7 +136,15 @@ func (q *Queue) changeState(uuid string, from, to arvados.ContainerState) error
 	q.entries[uuid] = ent
 	for i, ctr := range q.Containers {
 		if ctr.UUID == uuid {
-			q.Containers[i].State = to
+			if max := q.MaxDispatchAttempts; max > 0 && ctr.LockCount >= max && to == arvados.ContainerStateQueued {
+				q.Containers[i].State = arvados.ContainerStateCancelled
+				q.Containers[i].RuntimeStatus = map[string]interface{}{"error": fmt.Sprintf("Failed to start: lock_count == %d", ctr.LockCount)}
+			} else {
+				q.Containers[i].State = to
+				if to == arvados.ContainerStateLocked {
+					q.Containers[i].LockCount++
+				}
+			}
 			break
 		}
 	}
@@ -141,10 +167,12 @@ func (q *Queue) Update() error {
 			ent.Container = ctr
 			upd[ctr.UUID] = ent
 		} else {
-			it, _ := q.ChooseType(&ctr)
+			types, _ := q.ChooseType(&ctr)
+			ctr.Mounts = nil
 			upd[ctr.UUID] = container.QueueEnt{
-				Container:    ctr,
-				InstanceType: it,
+				Container:     ctr,
+				InstanceTypes: types,
+				FirstSeenAt:   time.Now(),
 			}
 		}
 	}