18102: Improve logging.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/lib/dispatchcloud/scheduler/run_queue.go b/lib/dispatchcloud/scheduler/run_queue.go
index 5bd4477ca58923edbd1af0a33df6fb943b51aec6..e9fc5f90215156051fb6de95c123da2c83022700 100644 (file)
--- a/lib/dispatchcloud/scheduler/run_queue.go
+++ b/lib/dispatchcloud/scheduler/run_queue.go
@@ -75,7 +75,7 @@ tryrun:
                                // starve this one by using keeping
                                // idle workers alive on different
                                // instance types.
-                               logger.Debug("overquota")
+                               logger.Trace("overquota")
                                overquota = sorted[i:]
                                break tryrun
                        } else if logger.Info("creating new instance"); sch.pool.Create(it) {
@@ -88,6 +88,7 @@ tryrun:
                                // avoid getting starved here if
                                // instances of a specific type always
                                // fail.
+                               logger.Trace("pool declined to create new instance")
                                continue
                        }
 

diff --git a/lib/dispatchcloud/scheduler/sync.go b/lib/dispatchcloud/scheduler/sync.go
index 038b4c8c5f0cb3a04aa08eddc843ac7564d9913f..4d601d6ae834dfeb242a8328dd4c62d79959d46e 100644 (file)
--- a/lib/dispatchcloud/scheduler/sync.go
+++ b/lib/dispatchcloud/scheduler/sync.go
@@ -66,7 +66,7 @@ func (sch *Scheduler) sync() {
                                // a network outage and is still
                                // preparing to run a container that
                                // has already been unlocked/requeued.
-                               go sch.kill(uuid, fmt.Sprintf("state=%s", ent.Container.State))
+                               go sch.kill(uuid, fmt.Sprintf("pool says running, but queue says state=%s", ent.Container.State))
                        } else if ent.Container.Priority == 0 {
                                sch.logger.WithFields(logrus.Fields{
                                        "ContainerUUID": uuid,
@@ -118,6 +118,10 @@ func (sch *Scheduler) kill(uuid string, reason string) {
                return
        }
        defer sch.uuidUnlock(uuid)
+       sch.logger.WithFields(logrus.Fields{
+               "ContainerUUID": uuid,
+               "reason":        reason,
+       }).Debug("kill")
        sch.pool.KillContainer(uuid, reason)
        sch.pool.ForgetContainer(uuid)
 }