16552: Avoid port collision due to port assignment races in tests.

[arvados.git] / lib / boot / nginx.go

diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 5826e5c0136cf7707753a6593c8fb756c1252c5d..b391c4dc8c93f61f50d221bb116cf71da3c10960 100644 (file)
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -12,9 +12,9 @@ import (
        "net/url"
        "os"
        "os/exec"
-       "os/user"
        "path/filepath"
        "regexp"
+       "strings"
 
        "git.arvados.org/arvados.git/sdk/go/arvados"
 )
@@ -32,13 +32,36 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
        if err != nil {
                return err
        }
+       extListenHost := "0.0.0.0"
+       if super.ClusterType == "test" {
+               // Our dynamic port number assignment strategy (choose
+               // an available port, write it in a config file, and
+               // have another process/goroutine bind to it) is prone
+               // to races when used by concurrent supervisors. In
+               // test mode we don't accept remote connections, so we
+               // can avoid collisions by using the per-cluster
+               // loopback address instead of 0.0.0.0.
+               extListenHost = super.ListenHost
+       }
        vars := map[string]string{
-               "LISTENHOST": super.ListenHost,
-               "SSLCERT":    filepath.Join(super.tempdir, "server.crt"),
-               "SSLKEY":     filepath.Join(super.tempdir, "server.key"),
-               "ACCESSLOG":  filepath.Join(super.tempdir, "nginx_access.log"),
-               "ERRORLOG":   filepath.Join(super.tempdir, "nginx_error.log"),
-               "TMPDIR":     super.wwwtempdir,
+               "LISTENHOST":       extListenHost,
+               "UPSTREAMHOST":     super.ListenHost,
+               "SSLCERT":          filepath.Join(super.tempdir, "server.crt"),
+               "SSLKEY":           filepath.Join(super.tempdir, "server.key"),
+               "ACCESSLOG":        filepath.Join(super.tempdir, "nginx_access.log"),
+               "ERRORLOG":         filepath.Join(super.tempdir, "nginx_error.log"),
+               "TMPDIR":           super.wwwtempdir,
+               "ARVADOS_API_HOST": super.cluster.Services.Controller.ExternalURL.Host,
+       }
+       u := url.URL(super.cluster.Services.Controller.ExternalURL)
+       ctrlHost := u.Hostname()
+       if strings.HasPrefix(super.cluster.TLS.Certificate, "file:/") && strings.HasPrefix(super.cluster.TLS.Key, "file:/") {
+               vars["SSLCERT"] = filepath.Clean(super.cluster.TLS.Certificate[5:])
+               vars["SSLKEY"] = filepath.Clean(super.cluster.TLS.Key[5:])
+       } else if f, err := os.Open("/var/lib/acme/live/" + ctrlHost + "/privkey"); err == nil {
+               f.Close()
+               vars["SSLCERT"] = "/var/lib/acme/live/" + ctrlHost + "/cert"
+               vars["SSLKEY"] = "/var/lib/acme/live/" + ctrlHost + "/privkey"
        }
        for _, cmpt := range []struct {
                varname string
@@ -51,6 +74,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
                {"GIT", super.cluster.Services.GitHTTP},
                {"HEALTH", super.cluster.Services.Health},
                {"WORKBENCH1", super.cluster.Services.Workbench1},
+               {"WORKBENCH2", super.cluster.Services.Workbench2},
                {"WS", super.cluster.Services.Websocket},
        } {
                var host, port string
@@ -108,28 +132,18 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
                }
        }
 
-       args := []string{
-               "-g", "error_log stderr info;",
-               "-g", "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + ";",
-               "-c", conffile,
-       }
-       // Nginx ignores "user www-data;" when running as a non-root
-       // user... except that it causes it to ignore our other -g
-       // options. So we still have to decide for ourselves whether
-       // it's needed.
-       if u, err := user.Current(); err != nil {
-               return fmt.Errorf("user.Current(): %w", err)
-       } else if u.Uid == "0" {
-               args = append([]string{"-g", "user www-data;"}, args...)
-       }
+       configs := "error_log stderr info; "
+       configs += "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + "; "
+       configs += "user www-data; "
 
        super.waitShutdown.Add(1)
        go func() {
                defer super.waitShutdown.Done()
-               fail(super.RunProgram(ctx, ".", runOptions{}, nginx, args...))
+               fail(super.RunProgram(ctx, ".", runOptions{}, nginx, "-g", configs, "-c", conffile))
        }()
        // Choose one of the ports where Nginx should listen, and wait
-       // here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
+       // here until we can connect. If ExternalURL is https://foo
+       // (with no port) then we connect to "foo:https"
        testurl := url.URL(super.cluster.Services.Controller.ExternalURL)
        if testurl.Port() == "" {
                testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme)