14360: Fix test case.

[arvados.git] / lib / dispatchcloud / dispatcher.go

diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index e422b3963d1d66bcc523271fde25aba3d5a0137f..81ad0ed3fa2b5924fdf1a55a3603dbfc88827fef 100644 (file)
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -28,56 +28,60 @@ import (
 )
 
 const (
-       defaultPollInterval = time.Second
+       defaultPollInterval     = time.Second
+       defaultStaleLockTimeout = time.Minute
 )
 
-type containerQueue interface {
-       scheduler.ContainerQueue
-       Update() error
-}
-
 type pool interface {
        scheduler.WorkerPool
-       View() []worker.View
+       Instances() []worker.InstanceView
+       Stop()
 }
 
 type dispatcher struct {
        Cluster       *arvados.Cluster
        InstanceSetID cloud.InstanceSetID
 
-       logger       logrus.FieldLogger
-       reg          *prometheus.Registry
-       instanceSet  cloud.InstanceSet
-       pool         pool
-       queue        containerQueue
-       httpHandler  http.Handler
-       pollInterval time.Duration
-       sshKey       ssh.Signer
+       logger      logrus.FieldLogger
+       reg         *prometheus.Registry
+       instanceSet cloud.InstanceSet
+       pool        pool
+       queue       scheduler.ContainerQueue
+       httpHandler http.Handler
+       sshKey      ssh.Signer
 
        setupOnce sync.Once
        stop      chan struct{}
+       stopped   chan struct{}
+}
+
+// Start starts the dispatcher. Start can be called multiple times
+// with no ill effect.
+func (disp *dispatcher) Start() {
+       disp.setupOnce.Do(disp.setup)
 }
 
 // ServeHTTP implements service.Handler.
 func (disp *dispatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-       disp.setupOnce.Do(disp.setup)
+       disp.Start()
        disp.httpHandler.ServeHTTP(w, r)
 }
 
 // CheckHealth implements service.Handler.
 func (disp *dispatcher) CheckHealth() error {
-       disp.setupOnce.Do(disp.setup)
+       disp.Start()
        return nil
 }
 
 // Stop dispatching containers and release resources. Typically used
 // in tests.
 func (disp *dispatcher) Close() {
-       disp.setupOnce.Do(disp.setup)
+       disp.Start()
        select {
        case disp.stop <- struct{}{}:
        default:
        }
+       <-disp.stopped
 }
 
 // Make a worker.Executor for the given instance.
@@ -108,6 +112,7 @@ func (disp *dispatcher) initialize() {
                }
        }
        disp.stop = make(chan struct{}, 1)
+       disp.stopped = make(chan struct{})
        disp.logger = logrus.StandardLogger()
 
        if key, err := ssh.ParsePrivateKey(disp.Cluster.Dispatch.PrivateKey); err != nil {
@@ -125,48 +130,41 @@ func (disp *dispatcher) initialize() {
        disp.pool = worker.NewPool(disp.logger, disp.reg, disp.instanceSet, disp.newExecutor, disp.Cluster)
        disp.queue = container.NewQueue(disp.logger, disp.reg, disp.typeChooser, arvClient)
 
-       mux := http.NewServeMux()
-       mux.HandleFunc("/arvados/v1/dispatch/containers", disp.apiContainers)
-       mux.HandleFunc("/arvados/v1/dispatch/instances", disp.apiInstances)
-       metricsH := promhttp.HandlerFor(disp.reg, promhttp.HandlerOpts{
-               ErrorLog: disp.logger,
-       })
-       mux.Handle("/metrics", metricsH)
-       mux.Handle("/metrics.json", metricsH)
-       disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux)
-
-       if d := disp.Cluster.Dispatch.PollInterval; d > 0 {
-               disp.pollInterval = time.Duration(d)
+       if disp.Cluster.ManagementToken == "" {
+               disp.httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+                       http.Error(w, "Management API authentication is not configured", http.StatusForbidden)
+               })
        } else {
-               disp.pollInterval = defaultPollInterval
+               mux := http.NewServeMux()
+               mux.HandleFunc("/arvados/v1/dispatch/containers", disp.apiContainers)
+               mux.HandleFunc("/arvados/v1/dispatch/instances", disp.apiInstances)
+               metricsH := promhttp.HandlerFor(disp.reg, promhttp.HandlerOpts{
+                       ErrorLog: disp.logger,
+               })
+               mux.Handle("/metrics", metricsH)
+               mux.Handle("/metrics.json", metricsH)
+               disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux)
        }
 }
 
 func (disp *dispatcher) run() {
+       defer close(disp.stopped)
        defer disp.instanceSet.Stop()
+       defer disp.pool.Stop()
 
-       t0 := time.Now()
-       disp.logger.Infof("FixStaleLocks starting.")
-       scheduler.FixStaleLocks(disp.logger, disp.queue, disp.pool, time.Duration(disp.Cluster.Dispatch.StaleLockTimeout))
-       disp.logger.Infof("FixStaleLocks finished (%s), starting scheduling.", time.Since(t0))
-
-       wp := disp.pool.Subscribe()
-       defer disp.pool.Unsubscribe(wp)
-       poll := time.NewTicker(disp.pollInterval)
-       for {
-               scheduler.Map(disp.logger, disp.queue, disp.pool)
-               scheduler.Sync(disp.logger, disp.queue, disp.pool)
-               select {
-               case <-disp.stop:
-                       return
-               case <-wp:
-               case <-poll.C:
-                       err := disp.queue.Update()
-                       if err != nil {
-                               disp.logger.Errorf("error updating queue: %s", err)
-                       }
-               }
+       staleLockTimeout := time.Duration(disp.Cluster.Dispatch.StaleLockTimeout)
+       if staleLockTimeout == 0 {
+               staleLockTimeout = defaultStaleLockTimeout
        }
+       pollInterval := time.Duration(disp.Cluster.Dispatch.PollInterval)
+       if pollInterval <= 0 {
+               pollInterval = defaultPollInterval
+       }
+       sched := scheduler.New(disp.logger, disp.queue, disp.pool, staleLockTimeout, pollInterval)
+       sched.Start()
+       defer sched.Stop()
+
+       <-disp.stop
 }
 
 // Management API: all active and queued containers.
@@ -192,8 +190,8 @@ func (disp *dispatcher) apiInstances(w http.ResponseWriter, r *http.Request) {
                return
        }
        var resp struct {
-               Items []worker.View
+               Items []worker.InstanceView
        }
-       resp.Items = disp.pool.View()
+       resp.Items = disp.pool.Instances()
        json.NewEncoder(w).Encode(resp)
 }