15954: Don't return from Stop() until child processes end.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index b11cb89e43bbfd6c2b578796a8749a464a31ccb7..5f5bb1ee77b8c5fc8c209f447d6e44acddaab3a0 100644 (file)
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -97,7 +97,8 @@ func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou
        defer boot.Stop()
        if url, ok := boot.WaitReady(); ok {
                fmt.Fprintln(stdout, url)
-               <-ctx.Done() // wait for signal
+               // Wait for signal/crash + orderly shutdown
+               <-boot.done
                return 0
        } else {
                return 1
@@ -121,6 +122,7 @@ type Booter struct {
        done          chan struct{}
        healthChecker *health.Aggregator
        tasksReady    map[string]chan bool
+       waitShutdown  sync.WaitGroup
 
        tempdir    string
        configfile string
@@ -288,6 +290,7 @@ func (boot *Booter) run(cfg *arvados.Config) error {
        boot.healthChecker = &health.Aggregator{Cluster: boot.cluster}
        <-boot.ctx.Done()
        boot.logger.Info("shutting down")
+       boot.waitShutdown.Wait()
        return boot.ctx.Err()
 }
 
@@ -315,9 +318,12 @@ func (boot *Booter) Stop() {
 }
 
 func (boot *Booter) WaitReady() (*arvados.URL, bool) {
+       ticker := time.NewTicker(time.Second)
+       defer ticker.Stop()
        for waiting := true; waiting; {
-               time.Sleep(time.Second)
-               if boot.ctx.Err() != nil {
+               select {
+               case <-ticker.C:
+               case <-boot.ctx.Done():
                        return nil, false
                }
                if boot.healthChecker == nil {

diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 5c1954c838d83b45ae5419fc0fa50d67c4e64161..a06d3a70084cd7aa8391580ead78f886eaee7cbc 100644 (file)
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -77,7 +77,9 @@ func (runNginx) Run(ctx context.Context, fail func(error), boot *Booter) error {
                        }
                }
        }
+       boot.waitShutdown.Add(1)
        go func() {
+               defer boot.waitShutdown.Done()
                fail(boot.RunProgram(ctx, ".", nil, nil, nginx,
                        "-g", "error_log stderr info;",
                        "-g", "pid "+filepath.Join(boot.tempdir, "nginx.pid")+";",

diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 822e737cb972777466c12d0ed1eb9cda9a314148..21834dab26da3d1fc075d21ed42a7942806b868b 100644 (file)
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -90,7 +90,9 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), boot *Boot
        if err != nil {
                return fmt.Errorf("bug: no InternalURLs for component %q: %v", runner, runner.svc.InternalURLs)
        }
+       boot.waitShutdown.Add(1)
        go func() {
+               defer boot.waitShutdown.Done()
                err = boot.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec",
                        "passenger", "start",
                        "-p", port,

diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index 48e24ffaec673e0f0cfc718ea10af5c3f6dfbf94..ed5aecd09faaa1dd2562cd8330d28b3b7427cce5 100644 (file)
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -56,7 +56,9 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), boot *Booter) er
 
        port := boot.cluster.PostgreSQL.Connection["port"]
 
+       boot.waitShutdown.Add(1)
        go func() {
+               defer boot.waitShutdown.Done()
                fail(boot.RunProgram(ctx, boot.tempdir, nil, nil, filepath.Join(bindir, "postgres"),
                        "-l",          // enable ssl
                        "-D", datadir, // data dir

diff --git a/lib/boot/service.go b/lib/boot/service.go
index 6edf78b3ce136ec6defd2134d7640acb28769df4..4b35e13768c9169de8df004be19cf6099f3cb072 100644 (file)
--- a/lib/boot/service.go
+++ b/lib/boot/service.go
@@ -45,7 +45,10 @@ func (runner runGoProgram) String() string {
 
 func (runner runGoProgram) Run(ctx context.Context, fail func(error), boot *Booter) error {
        boot.wait(ctx, runner.depends...)
-       boot.RunProgram(ctx, runner.src, nil, nil, "go", "install")
+       err := boot.RunProgram(ctx, runner.src, nil, nil, "go", "install")
+       if err != nil {
+               return err
+       }
        if ctx.Err() != nil {
                return ctx.Err()
        }
@@ -54,13 +57,17 @@ func (runner runGoProgram) Run(ctx context.Context, fail func(error), boot *Boot
                // Run one for each URL
                for u := range runner.svc.InternalURLs {
                        u := u
+                       boot.waitShutdown.Add(1)
                        go func() {
+                               defer boot.waitShutdown.Done()
                                fail(boot.RunProgram(ctx, boot.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, basename))
                        }()
                }
        } else {
                // Just run one
+               boot.waitShutdown.Add(1)
                go func() {
+                       defer boot.waitShutdown.Done()
                        fail(boot.RunProgram(ctx, boot.tempdir, nil, nil, basename))
                }()
        }