X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a3c7d9e03062e3246b0857fbae05f45d22e39169..4d095628f05fae2d2609b91dc12d0f44434b8aa2:/lib/dispatchcloud/dispatcher.go

diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 19c8f6e0b4..217ee39504 100644
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -15,13 +15,17 @@ import (
 	"time"
 
 	"git.arvados.org/arvados.git/lib/cloud"
+	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/lib/controller/dblock"
+	"git.arvados.org/arvados.git/lib/ctrlctx"
 	"git.arvados.org/arvados.git/lib/dispatchcloud/container"
 	"git.arvados.org/arvados.git/lib/dispatchcloud/scheduler"
-	"git.arvados.org/arvados.git/lib/dispatchcloud/ssh_executor"
+	"git.arvados.org/arvados.git/lib/dispatchcloud/sshexecutor"
 	"git.arvados.org/arvados.git/lib/dispatchcloud/worker"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/auth"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"git.arvados.org/arvados.git/sdk/go/health"
 	"git.arvados.org/arvados.git/sdk/go/httpserver"
 	"github.com/julienschmidt/httprouter"
 	"github.com/prometheus/client_golang/prometheus"
@@ -52,6 +56,7 @@ type dispatcher struct {
 	Registry      *prometheus.Registry
 	InstanceSetID cloud.InstanceSetID
 
+	dbConnector ctrlctx.DBConnector
 	logger      logrus.FieldLogger
 	instanceSet cloud.InstanceSet
 	pool        pool
@@ -82,6 +87,11 @@ func (disp *dispatcher) CheckHealth() error {
 	return disp.pool.CheckHealth()
 }
 
+// Done implements service.Handler.
+func (disp *dispatcher) Done() <-chan struct{} {
+	return disp.stopped
+}
+
 // Stop dispatching containers and release resources. Typically used
 // in tests.
 func (disp *dispatcher) Close() {
@@ -95,7 +105,7 @@ func (disp *dispatcher) Close() {
 
 // Make a worker.Executor for the given instance.
 func (disp *dispatcher) newExecutor(inst cloud.Instance) worker.Executor {
-	exr := ssh_executor.New(inst)
+	exr := sshexecutor.New(inst)
 	exr.SetTargetPort(disp.Cluster.Containers.CloudVMs.SSHPort)
 	exr.SetSigners(disp.sshKey)
 	return exr
@@ -112,6 +122,7 @@ func (disp *dispatcher) setup() {
 
 func (disp *dispatcher) initialize() {
 	disp.logger = ctxlog.FromContext(disp.Context)
+	disp.dbConnector = ctrlctx.DBConnector{PostgreSQL: disp.Cluster.PostgreSQL}
 
 	disp.ArvClient.AuthToken = disp.AuthToken
 
@@ -127,7 +138,7 @@ func (disp *dispatcher) initialize() {
 	disp.stop = make(chan struct{}, 1)
 	disp.stopped = make(chan struct{})
 
-	if key, err := ssh.ParsePrivateKey([]byte(disp.Cluster.Containers.DispatchPrivateKey)); err != nil {
+	if key, err := config.LoadSSHKey(disp.Cluster.Containers.DispatchPrivateKey); err != nil {
 		disp.logger.Fatalf("error parsing configured Containers.DispatchPrivateKey: %s", err)
 	} else {
 		disp.sshKey = key
@@ -137,6 +148,7 @@ func (disp *dispatcher) initialize() {
 	if err != nil {
 		disp.logger.Fatalf("error initializing driver: %s", err)
 	}
+	dblock.Dispatch.Lock(disp.Context, disp.dbConnector.GetDB)
 	disp.instanceSet = instanceSet
 	disp.pool = worker.NewPool(disp.logger, disp.ArvClient, disp.Registry, disp.InstanceSetID, disp.instanceSet, disp.newExecutor, disp.sshKey.PublicKey(), disp.Cluster)
 	disp.queue = container.NewQueue(disp.logger, disp.Registry, disp.typeChooser, disp.ArvClient)
@@ -148,7 +160,7 @@ func (disp *dispatcher) initialize() {
 	} else {
 		mux := httprouter.New()
 		mux.HandlerFunc("GET", "/arvados/v1/dispatch/containers", disp.apiContainers)
-		mux.HandlerFunc("POST", "/arvados/v1/dispatch/containers/kill", disp.apiInstanceKill)
+		mux.HandlerFunc("POST", "/arvados/v1/dispatch/containers/kill", disp.apiContainerKill)
 		mux.HandlerFunc("GET", "/arvados/v1/dispatch/instances", disp.apiInstances)
 		mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/hold", disp.apiInstanceHold)
 		mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/drain", disp.apiInstanceDrain)
@@ -159,11 +171,17 @@ func (disp *dispatcher) initialize() {
 		})
 		mux.Handler("GET", "/metrics", metricsH)
 		mux.Handler("GET", "/metrics.json", metricsH)
+		mux.Handler("GET", "/_health/:check", &health.Handler{
+			Token:  disp.Cluster.ManagementToken,
+			Prefix: "/_health/",
+			Routes: health.Routes{"ping": disp.CheckHealth},
+		})
 		disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux)
 	}
 }
 
 func (disp *dispatcher) run() {
+	defer dblock.Dispatch.Unlock()
 	defer close(disp.stopped)
 	defer disp.instanceSet.Stop()
 	defer disp.pool.Stop()
@@ -176,7 +194,11 @@ func (disp *dispatcher) run() {
 	if pollInterval <= 0 {
 		pollInterval = defaultPollInterval
 	}
-	sched := scheduler.New(disp.Context, disp.queue, disp.pool, staleLockTimeout, pollInterval)
+	maxSupervisors := int(float64(disp.Cluster.Containers.CloudVMs.MaxInstances) * disp.Cluster.Containers.CloudVMs.SupervisorFraction)
+	if maxSupervisors == 0 && disp.Cluster.Containers.CloudVMs.SupervisorFraction > 0 {
+		maxSupervisors = 1
+	}
+	sched := scheduler.New(disp.Context, disp.ArvClient, disp.queue, disp.pool, disp.Registry, staleLockTimeout, pollInterval, maxSupervisors)
 	sched.Start()
 	defer sched.Stop()