16217: Exit service command if the service handler fails.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index e3869261a160b4e42d147574410f15b9641b4e9d..3929a1103fb074b35234af17926aa71cb98df76d 100644 (file)
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -67,6 +67,10 @@ func (h *Handler) CheckHealth() error {
        return err
 }
 
+func (h *Handler) Done() <-chan struct{} {
+       return nil
+}
+
 func neverRedirect(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
 
 func (h *Handler) setup() {

diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 4023896f7933dbbd489387405419097dc083434e..02b6c976aec825f810eab3cca43488c808d5cc4e 100644 (file)
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -82,6 +82,11 @@ func (disp *dispatcher) CheckHealth() error {
        return disp.pool.CheckHealth()
 }
 
+// Done implements service.Handler.
+func (disp *dispatcher) Done() <-chan struct{} {
+       return disp.stopped
+}
+
 // Stop dispatching containers and release resources. Typically used
 // in tests.
 func (disp *dispatcher) Close() {

diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index 7f2f78ee9a9f7224aac4aacba94148497f292a5e..1e7a9a36edd3a8142192d14bfcfbf12885e1e857 100644 (file)
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -29,6 +29,7 @@ import (
 type Handler interface {
        http.Handler
        CheckHealth() error
+       Done() <-chan struct{}
 }
 
 type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, token string, registry *prometheus.Registry) Handler
@@ -148,9 +149,15 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
                logger.WithError(err).Errorf("error notifying init daemon")
        }
        go func() {
+               // Shut down server if caller cancels context
                <-ctx.Done()
                srv.Close()
        }()
+       go func() {
+               // Shut down server if handler dies
+               <-handler.Done()
+               srv.Close()
+       }()
        err = srv.Wait()
        if err != nil {
                return 1

diff --git a/lib/service/error.go b/lib/service/error.go
index c4049f7064d70a5a88c654be10cff478bb0f42f3..a4d7370d1b8d0d6a3b630025a181a56c029ed3b0 100644 (file)
--- a/lib/service/error.go
+++ b/lib/service/error.go
@@ -36,3 +36,15 @@ func (eh errorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 func (eh errorHandler) CheckHealth() error {
        return eh.err
 }
+
+// Done returns a closed channel to indicate the service has
+// stopped/failed.
+func (eh errorHandler) Done() <-chan struct{} {
+       return doneChannel
+}
+
+var doneChannel = func() <-chan struct{} {
+       done := make(chan struct{})
+       close(done)
+       return done
+}()

diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index a0284e8f247a60f8d2fd57b752f37a800d54c222..794adabdd3926b6b04036a6c62b1044f2e8f13d5 100644 (file)
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -46,6 +46,10 @@ func (agg *Aggregator) CheckHealth() error {
        return nil
 }
 
+func (agg *Aggregator) Done() <-chan struct{} {
+       return nil
+}
+
 func (agg *Aggregator) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
        agg.setupOnce.Do(agg.setup)
        sendErr := func(statusCode int, err error) {

diff --git a/services/keep-balance/server.go b/services/keep-balance/server.go
index 05658b5e5d7f17a4dcd9bd099d81d68950f97a8f..9801a3fd45d5d13ec40bf661c59b4de5156cfeed 100644 (file)
--- a/services/keep-balance/server.go
+++ b/services/keep-balance/server.go
@@ -53,6 +53,11 @@ func (srv *Server) CheckHealth() error {
        return nil
 }
 
+// Done implements service.Handler.
+func (srv *Server) Done() <-chan struct{} {
+       return nil
+}
+
 func (srv *Server) run() {
        var err error
        if srv.RunOptions.Once {

diff --git a/services/keepstore/command.go b/services/keepstore/command.go
index e0509393cff077e119b6ea7b975d10f90115d536..be2639773650fe5312ef80759cc427bd5cf0c14c 100644 (file)
--- a/services/keepstore/command.go
+++ b/services/keepstore/command.go
@@ -132,6 +132,10 @@ func (h *handler) CheckHealth() error {
        return h.err
 }
 
+func (h *handler) Done() <-chan struct{} {
+       return nil
+}
+
 func newHandlerOrErrorHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
        var h handler
        serviceURL, ok := service.URLFromContext(ctx)

diff --git a/services/ws/router.go b/services/ws/router.go
index b1764c156cad44c5954ca542fc8178b1cc182e1b..b3403dabd00a3740d87f3136a21076ac4640fdc1 100644 (file)
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -37,6 +37,7 @@ type router struct {
        handler   *handler
        mux       *http.ServeMux
        setupOnce sync.Once
+       done      chan struct{}
 
        lastReqID  int64
        lastReqMtx sync.Mutex
@@ -165,3 +166,7 @@ func (rtr *router) CheckHealth() error {
        rtr.setupOnce.Do(rtr.setup)
        return rtr.eventSource.DBHealth()
 }
+
+func (rtr *router) Done() <-chan struct{} {
+       return rtr.done
+}

diff --git a/services/ws/service.go b/services/ws/service.go
index fb313bb799be2ccfcd68d323f5828ac12c79ba8f..c38dcf59e68aac5a975f3362e7b258310341aef5 100644 (file)
--- a/services/ws/service.go
+++ b/services/ws/service.go
@@ -7,7 +7,6 @@ package ws
 import (
        "context"
        "fmt"
-       "os"
 
        "git.arvados.org/arvados.git/lib/cmd"
        "git.arvados.org/arvados.git/lib/service"
@@ -31,12 +30,11 @@ func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg
                QueueSize:    cluster.API.WebsocketServerEventQueue,
                Logger:       ctxlog.FromContext(ctx),
        }
+       done := make(chan struct{})
        go func() {
                eventSource.Run()
                ctxlog.FromContext(ctx).Error("event source stopped")
-               if !testMode {
-                       os.Exit(1)
-               }
+               close(done)
        }()
        eventSource.WaitReady()
        if err := eventSource.DBHealth(); err != nil {
@@ -47,6 +45,7 @@ func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg
                client:         client,
                eventSource:    eventSource,
                newPermChecker: func() permChecker { return newPermChecker(*client) },
+               done:           done,
        }
        return rtr
 }