Merge branch '14482-rubysdk-empty-dir'

[arvados.git] / lib / dispatchcloud / dispatcher.go

diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 707c31f249e50ae176708499cefff6013d6b7aa1..2415094ac00290625794d8204449052ad749ad85 100644 (file)
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -21,41 +21,38 @@ import (
        "git.curoverse.com/arvados.git/sdk/go/arvados"
        "git.curoverse.com/arvados.git/sdk/go/auth"
        "git.curoverse.com/arvados.git/sdk/go/httpserver"
-       "github.com/Sirupsen/logrus"
        "github.com/prometheus/client_golang/prometheus"
        "github.com/prometheus/client_golang/prometheus/promhttp"
+       "github.com/sirupsen/logrus"
        "golang.org/x/crypto/ssh"
 )
 
 const (
-       defaultPollInterval = time.Second
+       defaultPollInterval     = time.Second
+       defaultStaleLockTimeout = time.Minute
 )
 
-type containerQueue interface {
-       scheduler.ContainerQueue
-       Update() error
-}
-
 type pool interface {
        scheduler.WorkerPool
-       View() []worker.View
+       Instances() []worker.InstanceView
+       Stop()
 }
 
 type dispatcher struct {
        Cluster       *arvados.Cluster
        InstanceSetID cloud.InstanceSetID
 
-       logger       logrus.FieldLogger
-       reg          *prometheus.Registry
-       instanceSet  cloud.InstanceSet
-       pool         pool
-       queue        containerQueue
-       httpHandler  http.Handler
-       pollInterval time.Duration
-       sshKey       ssh.Signer
+       logger      logrus.FieldLogger
+       reg         *prometheus.Registry
+       instanceSet cloud.InstanceSet
+       pool        pool
+       queue       scheduler.ContainerQueue
+       httpHandler http.Handler
+       sshKey      ssh.Signer
 
        setupOnce sync.Once
        stop      chan struct{}
+       stopped   chan struct{}
 }
 
 // Start starts the dispatcher. Start can be called multiple times
@@ -84,6 +81,7 @@ func (disp *dispatcher) Close() {
        case disp.stop <- struct{}{}:
        default:
        }
+       <-disp.stopped
 }
 
 // Make a worker.Executor for the given instance.
@@ -114,6 +112,7 @@ func (disp *dispatcher) initialize() {
                }
        }
        disp.stop = make(chan struct{}, 1)
+       disp.stopped = make(chan struct{})
        disp.logger = logrus.StandardLogger()
 
        if key, err := ssh.ParsePrivateKey(disp.Cluster.Dispatch.PrivateKey); err != nil {
@@ -122,7 +121,7 @@ func (disp *dispatcher) initialize() {
                disp.sshKey = key
        }
 
-       instanceSet, err := newInstanceSet(disp.Cluster, disp.InstanceSetID)
+       instanceSet, err := newInstanceSet(disp.Cluster, disp.InstanceSetID, disp.logger)
        if err != nil {
                disp.logger.Fatalf("error initializing driver: %s", err)
        }
@@ -131,48 +130,41 @@ func (disp *dispatcher) initialize() {
        disp.pool = worker.NewPool(disp.logger, disp.reg, disp.instanceSet, disp.newExecutor, disp.Cluster)
        disp.queue = container.NewQueue(disp.logger, disp.reg, disp.typeChooser, arvClient)
 
-       mux := http.NewServeMux()
-       mux.HandleFunc("/arvados/v1/dispatch/containers", disp.apiContainers)
-       mux.HandleFunc("/arvados/v1/dispatch/instances", disp.apiInstances)
-       metricsH := promhttp.HandlerFor(disp.reg, promhttp.HandlerOpts{
-               ErrorLog: disp.logger,
-       })
-       mux.Handle("/metrics", metricsH)
-       mux.Handle("/metrics.json", metricsH)
-       disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux)
-
-       if d := disp.Cluster.Dispatch.PollInterval; d > 0 {
-               disp.pollInterval = time.Duration(d)
+       if disp.Cluster.ManagementToken == "" {
+               disp.httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+                       http.Error(w, "Management API authentication is not configured", http.StatusForbidden)
+               })
        } else {
-               disp.pollInterval = defaultPollInterval
+               mux := http.NewServeMux()
+               mux.HandleFunc("/arvados/v1/dispatch/containers", disp.apiContainers)
+               mux.HandleFunc("/arvados/v1/dispatch/instances", disp.apiInstances)
+               metricsH := promhttp.HandlerFor(disp.reg, promhttp.HandlerOpts{
+                       ErrorLog: disp.logger,
+               })
+               mux.Handle("/metrics", metricsH)
+               mux.Handle("/metrics.json", metricsH)
+               disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux)
        }
 }
 
 func (disp *dispatcher) run() {
+       defer close(disp.stopped)
        defer disp.instanceSet.Stop()
+       defer disp.pool.Stop()
 
-       t0 := time.Now()
-       disp.logger.Infof("FixStaleLocks starting.")
-       scheduler.FixStaleLocks(disp.logger, disp.queue, disp.pool, time.Duration(disp.Cluster.Dispatch.StaleLockTimeout))
-       disp.logger.Infof("FixStaleLocks finished (%s), starting scheduling.", time.Since(t0))
-
-       wp := disp.pool.Subscribe()
-       defer disp.pool.Unsubscribe(wp)
-       poll := time.NewTicker(disp.pollInterval)
-       for {
-               scheduler.Map(disp.logger, disp.queue, disp.pool)
-               scheduler.Sync(disp.logger, disp.queue, disp.pool)
-               select {
-               case <-disp.stop:
-                       return
-               case <-wp:
-               case <-poll.C:
-                       err := disp.queue.Update()
-                       if err != nil {
-                               disp.logger.Errorf("error updating queue: %s", err)
-                       }
-               }
+       staleLockTimeout := time.Duration(disp.Cluster.Dispatch.StaleLockTimeout)
+       if staleLockTimeout == 0 {
+               staleLockTimeout = defaultStaleLockTimeout
        }
+       pollInterval := time.Duration(disp.Cluster.Dispatch.PollInterval)
+       if pollInterval <= 0 {
+               pollInterval = defaultPollInterval
+       }
+       sched := scheduler.New(disp.logger, disp.queue, disp.pool, staleLockTimeout, pollInterval)
+       sched.Start()
+       defer sched.Stop()
+
+       <-disp.stop
 }
 
 // Management API: all active and queued containers.
@@ -198,8 +190,8 @@ func (disp *dispatcher) apiInstances(w http.ResponseWriter, r *http.Request) {
                return
        }
        var resp struct {
-               Items []worker.View
+               Items []worker.InstanceView
        }
-       resp.Items = disp.pool.View()
+       resp.Items = disp.pool.Instances()
        json.NewEncoder(w).Encode(resp)
 }