15954: Control listen addresses, improve logging.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index c288bab7515be56261a5e68f15f80ff713084a01..6cf44837cafb291227efa9b7c6954a749b0ddffd 100644 (file)
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -25,8 +25,6 @@ import (
 
        "git.arvados.org/arvados.git/lib/cmd"
        "git.arvados.org/arvados.git/lib/config"
-       "git.arvados.org/arvados.git/lib/controller"
-       "git.arvados.org/arvados.git/lib/dispatchcloud"
        "git.arvados.org/arvados.git/sdk/go/arvados"
        "git.arvados.org/arvados.git/sdk/go/ctxlog"
        "git.arvados.org/arvados.git/sdk/go/health"
@@ -81,6 +79,7 @@ func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou
        flags.StringVar(&boot.SourcePath, "source", ".", "arvados source tree `directory`")
        flags.StringVar(&boot.LibPath, "lib", "/var/lib/arvados", "`directory` to install dependencies and library files")
        flags.StringVar(&boot.ClusterType, "type", "production", "cluster `type`: development, test, or production")
+       flags.StringVar(&boot.ListenHost, "listen-host", "localhost", "host name or interface address for service listeners")
        flags.StringVar(&boot.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
        flags.BoolVar(&boot.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
        err = flags.Parse(args)
@@ -111,6 +110,7 @@ type Booter struct {
        SourcePath           string // e.g., /home/username/src/arvados
        LibPath              string // e.g., /var/lib/arvados
        ClusterType          string // e.g., production
+       ListenHost           string // e.g., localhost
        ControllerAddr       string // e.g., 127.0.0.1:8000
        OwnTemporaryDatabase bool
        Stderr               io.Writer
@@ -202,7 +202,11 @@ func (boot *Booter) run(loader *config.Loader) error {
        }
        // Now that we have the config, replace the bootstrap logger
        // with a new one according to the logging config.
-       boot.logger = ctxlog.New(boot.Stderr, boot.cluster.SystemLogs.Format, boot.cluster.SystemLogs.LogLevel).WithFields(logrus.Fields{
+       loglevel := boot.cluster.SystemLogs.LogLevel
+       if s := os.Getenv("ARVADOS_DEBUG"); s != "" && s != "0" {
+               loglevel = "debug"
+       }
+       boot.logger = ctxlog.New(boot.Stderr, boot.cluster.SystemLogs.Format, loglevel).WithFields(logrus.Fields{
                "PID": os.Getpid(),
        })
        boot.healthChecker = &health.Aggregator{Cluster: boot.cluster}
@@ -230,22 +234,22 @@ func (boot *Booter) run(loader *config.Loader) error {
                createCertificates{},
                runPostgreSQL{},
                runNginx{},
-               runServiceCommand{name: "controller", command: controller.Command, depends: []bootTask{runPostgreSQL{}}},
+               runServiceCommand{name: "controller", svc: boot.cluster.Services.Controller, depends: []bootTask{runPostgreSQL{}}},
                runGoProgram{src: "services/arv-git-httpd"},
                runGoProgram{src: "services/health"},
-               runGoProgram{src: "services/keepproxy"},
+               runGoProgram{src: "services/keepproxy", depends: []bootTask{runPassenger{src: "services/api"}}},
                runGoProgram{src: "services/keepstore", svc: boot.cluster.Services.Keepstore},
                runGoProgram{src: "services/keep-web"},
                runGoProgram{src: "services/ws", depends: []bootTask{runPostgreSQL{}}},
                installPassenger{src: "services/api"},
                runPassenger{src: "services/api", svc: boot.cluster.Services.RailsAPI, depends: []bootTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
-               installPassenger{src: "apps/workbench"},
+               installPassenger{src: "apps/workbench", depends: []bootTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
                runPassenger{src: "apps/workbench", svc: boot.cluster.Services.Workbench1, depends: []bootTask{installPassenger{src: "apps/workbench"}}},
                seedDatabase{},
        }
        if boot.ClusterType != "test" {
                tasks = append(tasks,
-                       runServiceCommand{name: "dispatchcloud", command: dispatchcloud.Command},
+                       runServiceCommand{name: "dispatch-cloud", svc: boot.cluster.Services.Controller},
                        runGoProgram{src: "services/keep-balance"},
                )
        }
@@ -260,10 +264,10 @@ func (boot *Booter) run(loader *config.Loader) error {
                                return
                        }
                        boot.cancel()
-                       boot.logger.WithField("task", task).WithError(err).Error("task failed")
+                       boot.logger.WithField("task", task.String()).WithError(err).Error("task failed")
                }
                go func() {
-                       boot.logger.WithField("task", task).Info("starting")
+                       boot.logger.WithField("task", task.String()).Info("starting")
                        err := task.Run(boot.ctx, fail, boot)
                        if err != nil {
                                fail(err)
@@ -272,7 +276,12 @@ func (boot *Booter) run(loader *config.Loader) error {
                        close(boot.tasksReady[task.String()])
                }()
        }
-       return boot.wait(boot.ctx, tasks...)
+       err = boot.wait(boot.ctx, tasks...)
+       if err != nil {
+               return err
+       }
+       <-boot.ctx.Done()
+       return boot.ctx.Err()
 }
 
 func (boot *Booter) wait(ctx context.Context, tasks ...bootTask) error {
@@ -281,7 +290,7 @@ func (boot *Booter) wait(ctx context.Context, tasks ...bootTask) error {
                if !ok {
                        return fmt.Errorf("no such task: %s", task)
                }
-               boot.logger.WithField("task", task).Info("waiting")
+               boot.logger.WithField("task", task.String()).Info("waiting")
                select {
                case <-ch:
                case <-ctx.Done():
@@ -313,9 +322,10 @@ func (boot *Booter) WaitReady() bool {
                // instead we wait for all configured components to
                // pass.
                waiting = false
-               for _, check := range resp.Checks {
+               for target, check := range resp.Checks {
                        if check.Health != "OK" {
                                waiting = true
+                               boot.logger.WithField("target", target).Debug("waiting")
                        }
                }
        }
@@ -385,13 +395,23 @@ func (boot *Booter) lookPath(prog string) string {
 func (boot *Booter) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
        cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
        fmt.Fprintf(boot.Stderr, "%s executing in %s\n", cmdline, dir)
+
+       logprefix := prog
+       if prog == "bundle" && len(args) > 2 && args[0] == "exec" {
+               logprefix = args[1]
+       }
+       if !strings.HasPrefix(dir, "/") {
+               logprefix = dir + ": " + logprefix
+       }
+       stderr := &logPrefixer{Writer: boot.Stderr, Prefix: []byte("[" + logprefix + "] ")}
+
        cmd := exec.Command(boot.lookPath(prog), args...)
        if output == nil {
-               cmd.Stdout = boot.Stderr
+               cmd.Stdout = stderr
        } else {
                cmd.Stdout = output
        }
-       cmd.Stderr = boot.Stderr
+       cmd.Stderr = stderr
        if strings.HasPrefix(dir, "/") {
                cmd.Dir = dir
        } else {
@@ -452,7 +472,7 @@ func (boot *Booter) autofillConfig(cfg *arvados.Config, log logrus.FieldLogger)
                        return err
                }
                if h == "" {
-                       h = "localhost"
+                       h = boot.ListenHost
                }
                if p == "0" {
                        p, err = availablePort(":0")
@@ -486,11 +506,11 @@ func (boot *Booter) autofillConfig(cfg *arvados.Config, log logrus.FieldLogger)
                        svc == &cluster.Services.WebDAVDownload ||
                        svc == &cluster.Services.Websocket ||
                        svc == &cluster.Services.Workbench1) {
-                       svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("localhost:%s", nextPort())}
+                       svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", boot.ListenHost, nextPort())}
                }
                if len(svc.InternalURLs) == 0 {
                        svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
-                               arvados.URL{Scheme: "http", Host: fmt.Sprintf("localhost:%s", nextPort())}: arvados.ServiceInstance{},
+                               arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", boot.ListenHost, nextPort())}: arvados.ServiceInstance{},
                        }
                }
        }
@@ -518,7 +538,7 @@ func (boot *Booter) autofillConfig(cfg *arvados.Config, log logrus.FieldLogger)
        }
        if boot.ClusterType == "test" {
                // Add a second keepstore process.
-               cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("localhost:%s", nextPort())}] = arvados.ServiceInstance{}
+               cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", boot.ListenHost, nextPort())}] = arvados.ServiceInstance{}
 
                // Create a directory-backed volume for each keepstore
                // process.
@@ -532,7 +552,7 @@ func (boot *Booter) autofillConfig(cfg *arvados.Config, log logrus.FieldLogger)
                        } else if err = os.Mkdir(datadir, 0777); err != nil {
                                return err
                        }
-                       cluster.Volumes[fmt.Sprintf("zzzzz-nyw5e-%015d", volnum)] = arvados.Volume{
+                       cluster.Volumes[fmt.Sprintf(cluster.ClusterID+"-nyw5e-%015d", volnum)] = arvados.Volume{
                                Driver:           "Directory",
                                DriverParameters: json.RawMessage(fmt.Sprintf(`{"Root":%q}`, datadir)),
                                AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{

diff --git a/lib/boot/log.go b/lib/boot/log.go
new file mode 100644 (file)
index 0000000..062a854
--- /dev/null
+++ b/lib/boot/log.go
@@ -0,0 +1,32 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+       "bytes"
+       "io"
+)
+
+type logPrefixer struct {
+       io.Writer
+       Prefix []byte
+       did    bool
+}
+
+func (lp *logPrefixer) Write(p []byte) (int, error) {
+       if len(p) == 0 {
+               return 0, nil
+       }
+       if !lp.did {
+               lp.Writer.Write(lp.Prefix)
+               lp.did = p[len(p)-1] != '\n'
+       }
+       out := append(bytes.Replace(p[:len(p)-1], []byte("\n"), append([]byte("\n"), lp.Prefix...), -1), p[len(p)-1])
+       _, err := lp.Writer.Write(out)
+       if err != nil {
+               return 0, err
+       }
+       return len(p), nil
+}

diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index c5bfd605afac4485bd391c6d87cc5016eddfc0de..5c1954c838d83b45ae5419fc0fa50d67c4e64161 100644 (file)
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -24,11 +24,12 @@ func (runNginx) String() string {
 
 func (runNginx) Run(ctx context.Context, fail func(error), boot *Booter) error {
        vars := map[string]string{
-               "SSLCERT":   filepath.Join(boot.SourcePath, "services", "api", "tmp", "self-signed.pem"), // TODO: root ca
-               "SSLKEY":    filepath.Join(boot.SourcePath, "services", "api", "tmp", "self-signed.key"), // TODO: root ca
-               "ACCESSLOG": filepath.Join(boot.tempdir, "nginx_access.log"),
-               "ERRORLOG":  filepath.Join(boot.tempdir, "nginx_error.log"),
-               "TMPDIR":    boot.tempdir,
+               "LISTENHOST": boot.ListenHost,
+               "SSLCERT":    filepath.Join(boot.SourcePath, "services", "api", "tmp", "self-signed.pem"), // TODO: root ca
+               "SSLKEY":     filepath.Join(boot.SourcePath, "services", "api", "tmp", "self-signed.key"), // TODO: root ca
+               "ACCESSLOG":  filepath.Join(boot.tempdir, "nginx_access.log"),
+               "ERRORLOG":   filepath.Join(boot.tempdir, "nginx_error.log"),
+               "TMPDIR":     boot.tempdir,
        }
        var err error
        for _, cmpt := range []struct {

diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index b67370e1ec0e0eaf71f488ec197631a55dcb2edf..4aada107bcf569028675b923c5e418479fb7291b 100644 (file)
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -21,7 +21,7 @@ type installPassenger struct {
 }
 
 func (runner installPassenger) String() string {
-       return "install " + runner.src
+       return "installPassenger:" + runner.src
 }
 
 func (runner installPassenger) Run(ctx context.Context, fail func(error), boot *Booter) error {
@@ -69,8 +69,7 @@ type runPassenger struct {
 }
 
 func (runner runPassenger) String() string {
-       _, basename := filepath.Split(runner.src)
-       return basename
+       return "runPassenger:" + runner.src
 }
 
 func (runner runPassenger) Run(ctx context.Context, fail func(error), boot *Booter) error {
@@ -82,9 +81,9 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), boot *Boot
        if err != nil {
                return fmt.Errorf("bug: no InternalURLs for component %q: %v", runner, runner.svc.InternalURLs)
        }
-       err = boot.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger", "start", "-p", port)
-       if err != nil {
-               return err
-       }
+       go func() {
+               err = boot.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger", "start", "-p", port)
+               fail(err)
+       }()
        return nil
 }

diff --git a/lib/boot/seed.go b/lib/boot/seed.go
index f915764c5873ba8b9292e1945dbd61a9ffbebfc8..9f086d54451bd5c4297995ca639880eb773cd581 100644 (file)
--- a/lib/boot/seed.go
+++ b/lib/boot/seed.go
@@ -15,7 +15,7 @@ func (seedDatabase) String() string {
 }
 
 func (seedDatabase) Run(ctx context.Context, fail func(error), boot *Booter) error {
-       err := boot.wait(ctx, runPostgreSQL{})
+       err := boot.wait(ctx, runPostgreSQL{}, installPassenger{src: "services/api"})
        if err != nil {
                return err
        }

diff --git a/lib/boot/service.go b/lib/boot/service.go
index 9672dccc4865619bde28219790859f30e0d229aa..6edf78b3ce136ec6defd2134d7640acb28769df4 100644 (file)
--- a/lib/boot/service.go
+++ b/lib/boot/service.go
@@ -5,18 +5,15 @@
 package boot
 
 import (
-       "bytes"
        "context"
-       "fmt"
        "path/filepath"
 
-       "git.arvados.org/arvados.git/lib/cmd"
        "git.arvados.org/arvados.git/sdk/go/arvados"
 )
 
 type runServiceCommand struct {
        name    string
-       command cmd.Handler
+       svc     arvados.Service
        depends []bootTask
 }
 
@@ -27,11 +24,10 @@ func (runner runServiceCommand) String() string {
 func (runner runServiceCommand) Run(ctx context.Context, fail func(error), boot *Booter) error {
        boot.wait(ctx, runner.depends...)
        go func() {
-               // runner.command.RunCommand() doesn't have access to
-               // ctx, so it can't shut down by itself when the
-               // caller cancels. We just abandon it.
-               exitcode := runner.command.RunCommand(runner.name, []string{"-config", boot.configfile}, bytes.NewBuffer(nil), boot.Stderr, boot.Stderr)
-               fail(fmt.Errorf("exit code %d", exitcode))
+               var u arvados.URL
+               for u = range runner.svc.InternalURLs {
+               }
+               fail(boot.RunProgram(ctx, boot.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, "arvados-server", runner.name, "-config", boot.configfile))
        }()
        return nil
 }

diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index b42090fed14287218ff3232b3fc44cdb05cabeab..b10b3f00825f682f9cbf4c4a498b5f56f8824945 100644 (file)
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -17,7 +17,7 @@ http {
   uwsgi_temp_path "{{TMPDIR}}";
   scgi_temp_path "{{TMPDIR}}";
   upstream arv-git-http {
-    server localhost:{{GITPORT}};
+    server {{LISTENHOST}}:{{GITPORT}};
   }
   server {
     listen *:{{GITSSLPORT}} ssl default_server;
@@ -33,7 +33,7 @@ http {
     }
   }
   upstream keepproxy {
-    server localhost:{{KEEPPROXYPORT}};
+    server {{LISTENHOST}}:{{KEEPPROXYPORT}};
   }
   server {
     listen *:{{KEEPPROXYSSLPORT}} ssl default_server;
@@ -52,7 +52,7 @@ http {
     }
   }
   upstream keep-web {
-    server localhost:{{KEEPWEBPORT}};
+    server {{LISTENHOST}}:{{KEEPWEBPORT}};
   }
   server {
     listen *:{{KEEPWEBSSLPORT}} ssl default_server;
@@ -89,7 +89,7 @@ http {
     }
   }
   upstream ws {
-    server localhost:{{WSPORT}};
+    server {{LISTENHOST}}:{{WSPORT}};
   }
   server {
     listen *:{{WSSSLPORT}} ssl default_server;
@@ -107,7 +107,7 @@ http {
     }
   }
   upstream workbench1 {
-    server localhost:{{WORKBENCH1PORT}};
+    server {{LISTENHOST}}:{{WORKBENCH1PORT}};
   }
   server {
     listen *:{{WORKBENCH1SSLPORT}} ssl default_server;
@@ -123,7 +123,7 @@ http {
     }
   }
   upstream controller {
-    server localhost:{{CONTROLLERPORT}};
+    server {{LISTENHOST}}:{{CONTROLLERPORT}};
   }
   server {
     listen *:{{CONTROLLERSSLPORT}} ssl default_server;

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 005c75edf01591558047a164c1aae722d4e3ed55..639eb87fddfa4df96b9591bac5764ebe9110ad32 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -606,6 +606,7 @@ def run_nginx():
         return
     stop_nginx()
     nginxconf = {}
+    nginxconf['LISTENHOST'] = 'localhost'
     nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
     nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")
     nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV")