package boot
import (
+ "bytes"
"context"
"fmt"
"io/ioutil"
"net"
+ "net/url"
"os"
"os/exec"
"path/filepath"
"regexp"
+ "strings"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
)
// Run an Nginx process that proxies the supervisor's configured
}
func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ err := super.wait(ctx, createCertificates{})
+ if err != nil {
+ return err
+ }
+ extListenHost := "0.0.0.0"
+ if super.ClusterType == "test" {
+ // Our dynamic port number assignment strategy (choose
+ // an available port, write it in a config file, and
+ // have another process/goroutine bind to it) is prone
+ // to races when used by concurrent supervisors. In
+ // test mode we don't accept remote connections, so we
+ // can avoid collisions by using the per-cluster
+ // loopback address instead of 0.0.0.0.
+ extListenHost = super.ListenHost
+ }
vars := map[string]string{
- "LISTENHOST": super.ListenHost,
- "SSLCERT": filepath.Join(super.SourcePath, "services", "api", "tmp", "self-signed.pem"), // TODO: root ca
- "SSLKEY": filepath.Join(super.SourcePath, "services", "api", "tmp", "self-signed.key"), // TODO: root ca
- "ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
- "ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
- "TMPDIR": super.tempdir,
+ "LISTENHOST": extListenHost,
+ "UPSTREAMHOST": super.ListenHost,
+ "INTERNALSUBNETS": internalSubnets(super.logger),
+ "SSLCERT": filepath.Join(super.tempdir, "server.crt"),
+ "SSLKEY": filepath.Join(super.tempdir, "server.key"),
+ "ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
+ "ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
+ "TMPDIR": super.wwwtempdir,
+ "ARVADOS_API_HOST": super.cluster.Services.Controller.ExternalURL.Host,
+ }
+ u := url.URL(super.cluster.Services.Controller.ExternalURL)
+ ctrlHost := u.Hostname()
+ if strings.HasPrefix(super.cluster.TLS.Certificate, "file:/") && strings.HasPrefix(super.cluster.TLS.Key, "file:/") {
+ vars["SSLCERT"] = filepath.Clean(super.cluster.TLS.Certificate[5:])
+ vars["SSLKEY"] = filepath.Clean(super.cluster.TLS.Key[5:])
+ } else if f, err := os.Open("/var/lib/acme/live/" + ctrlHost + "/privkey"); err == nil {
+ f.Close()
+ vars["SSLCERT"] = "/var/lib/acme/live/" + ctrlHost + "/cert"
+ vars["SSLKEY"] = "/var/lib/acme/live/" + ctrlHost + "/privkey"
}
- var err error
for _, cmpt := range []struct {
varname string
svc arvados.Service
{"KEEPWEB", super.cluster.Services.WebDAV},
{"KEEPWEBDL", super.cluster.Services.WebDAVDownload},
{"KEEPPROXY", super.cluster.Services.Keepproxy},
- {"GIT", super.cluster.Services.GitHTTP},
+ {"HEALTH", super.cluster.Services.Health},
{"WORKBENCH1", super.cluster.Services.Workbench1},
+ {"WORKBENCH2", super.cluster.Services.Workbench2},
{"WS", super.cluster.Services.Websocket},
} {
- port, err := internalPort(cmpt.svc)
- if err != nil {
- return fmt.Errorf("%s internal port: %s (%v)", cmpt.varname, err, cmpt.svc)
- }
- if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
- return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
+ var host, port string
+ if len(cmpt.svc.InternalURLs) == 0 {
+ // We won't run this service, but we need an
+ // upstream port to write in our templated
+ // nginx config. Choose a port that will
+ // return 502 Bad Gateway.
+ port = "9"
+ } else if host, port, err = internalPort(cmpt.svc); err != nil {
+ return fmt.Errorf("%s internal port: %w (%v)", cmpt.varname, err, cmpt.svc)
+ } else if ok, err := addrIsLocal(net.JoinHostPort(host, port)); !ok || err != nil {
+ return fmt.Errorf("%s addrIsLocal() failed for host %q port %q: %v", cmpt.varname, host, port, err)
}
vars[cmpt.varname+"PORT"] = port
port, err = externalPort(cmpt.svc)
if err != nil {
- return fmt.Errorf("%s external port: %s (%v)", cmpt.varname, err, cmpt.svc)
+ return fmt.Errorf("%s external port: %w (%v)", cmpt.varname, err, cmpt.svc)
}
- if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
- return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
+ listenAddr := net.JoinHostPort(super.ListenHost, port)
+ if ok, err := addrIsLocal(listenAddr); !ok || err != nil {
+ return fmt.Errorf("%s addrIsLocal(%q) failed: %w", cmpt.varname, listenAddr, err)
}
vars[cmpt.varname+"SSLPORT"] = port
}
- tmpl, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf"))
+ var conftemplate string
+ if super.ClusterType == "production" {
+ conftemplate = "/var/lib/arvados/share/nginx.conf"
+ } else {
+ conftemplate = filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf")
+ }
+ tmpl, err := ioutil.ReadFile(conftemplate)
if err != nil {
return err
}
}
}
}
+
+ configs := "error_log stderr warn; "
+ configs += "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + "; "
+ configs += "user www-data; "
+
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, ".", nil, nil, nginx,
- "-g", "error_log stderr info;",
- "-g", "pid "+filepath.Join(super.tempdir, "nginx.pid")+";",
- "-c", conffile))
+ fail(super.RunProgram(ctx, ".", runOptions{}, nginx, "-g", configs, "-c", conffile))
}()
- return waitForConnect(ctx, super.cluster.Services.Controller.ExternalURL.Host)
+ // Choose one of the ports where Nginx should listen, and wait
+ // here until we can connect. If ExternalURL is https://foo
+ // (with no port) then we connect to "foo:https"
+ testurl := url.URL(super.cluster.Services.Controller.ExternalURL)
+ if testurl.Port() == "" {
+ testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme)
+ }
+ return waitForConnect(ctx, testurl.Host)
+}
+
+// Return 0 or more local subnets as "geo" fragments for Nginx config,
+// e.g., "1.2.3.0/24 0; 10.1.0.0/16 0;".
+func internalSubnets(logger logrus.FieldLogger) string {
+ iproutes, err := exec.Command("ip", "route").CombinedOutput()
+ if err != nil {
+ logger.Warnf("treating all clients as external because `ip route` failed: %s (%q)", err, iproutes)
+ return ""
+ }
+ subnets := ""
+ for _, line := range bytes.Split(iproutes, []byte("\n")) {
+ fields := strings.Fields(string(line))
+ if len(fields) > 2 && fields[1] == "dev" {
+ // lan example:
+ // 192.168.86.0/24 dev ens3 proto kernel scope link src 192.168.86.196
+ // gcp example (private subnet):
+ // 10.47.0.0/24 dev eth0 proto kernel scope link src 10.47.0.5
+ // gcp example (no private subnet):
+ // 10.128.0.1 dev ens4 scope link
+ subnets += fields[0] + " 0; "
+ }
+ }
+ return subnets
}
projects / arvados.git / blobdiff