lib/boot/nginx.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package boot
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "io/ioutil"
  12         "net"
  13         "net/url"
  14         "os"
  15         "os/exec"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.arvados.org/arvados.git/sdk/go/arvados"
  21         "github.com/sirupsen/logrus"
  22 )
  23
  24 // Run an Nginx process that proxies the supervisor's configured
  25 // ExternalURLs to the appropriate InternalURLs.
  26 type runNginx struct{}
  27
  28 func (runNginx) String() string {
  29         return "nginx"
  30 }
  31
  32 func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) error {
  33         err := super.wait(ctx, createCertificates{})
  34         if err != nil {
  35                 return err
  36         }
  37         extListenHost := "0.0.0.0"
  38         if super.ClusterType == "test" {
  39                 // Our dynamic port number assignment strategy (choose
  40                 // an available port, write it in a config file, and
  41                 // have another process/goroutine bind to it) is prone
  42                 // to races when used by concurrent supervisors. In
  43                 // test mode we don't accept remote connections, so we
  44                 // can avoid collisions by using the per-cluster
  45                 // loopback address instead of 0.0.0.0.
  46                 extListenHost = super.ListenHost
  47         }
  48         vars := map[string]string{
  49                 "LISTENHOST":       extListenHost,
  50                 "UPSTREAMHOST":     super.ListenHost,
  51                 "INTERNALSUBNETS":  internalSubnets(super.logger),
  52                 "SSLCERT":          filepath.Join(super.tempdir, "server.crt"),
  53                 "SSLKEY":           filepath.Join(super.tempdir, "server.key"),
  54                 "ACCESSLOG":        filepath.Join(super.tempdir, "nginx_access.log"),
  55                 "ERRORLOG":         filepath.Join(super.tempdir, "nginx_error.log"),
  56                 "TMPDIR":           super.wwwtempdir,
  57                 "ARVADOS_API_HOST": super.cluster.Services.Controller.ExternalURL.Host,
  58         }
  59         u := url.URL(super.cluster.Services.Controller.ExternalURL)
  60         ctrlHost := u.Hostname()
  61         if strings.HasPrefix(super.cluster.TLS.Certificate, "file:/") && strings.HasPrefix(super.cluster.TLS.Key, "file:/") {
  62                 vars["SSLCERT"] = filepath.Clean(super.cluster.TLS.Certificate[5:])
  63                 vars["SSLKEY"] = filepath.Clean(super.cluster.TLS.Key[5:])
  64         } else if f, err := os.Open("/var/lib/acme/live/" + ctrlHost + "/privkey"); err == nil {
  65                 f.Close()
  66                 vars["SSLCERT"] = "/var/lib/acme/live/" + ctrlHost + "/cert"
  67                 vars["SSLKEY"] = "/var/lib/acme/live/" + ctrlHost + "/privkey"
  68         }
  69         for _, cmpt := range []struct {
  70                 varname string
  71                 svc     arvados.Service
  72         }{
  73                 {"CONTROLLER", super.cluster.Services.Controller},
  74                 {"KEEPWEB", super.cluster.Services.WebDAV},
  75                 {"KEEPWEBDL", super.cluster.Services.WebDAVDownload},
  76                 {"KEEPPROXY", super.cluster.Services.Keepproxy},
  77                 {"HEALTH", super.cluster.Services.Health},
  78                 {"WORKBENCH1", super.cluster.Services.Workbench1},
  79                 {"WORKBENCH2", super.cluster.Services.Workbench2},
  80                 {"WS", super.cluster.Services.Websocket},
  81         } {
  82                 var host, port string
  83                 if len(cmpt.svc.InternalURLs) == 0 {
  84                         // We won't run this service, but we need an
  85                         // upstream port to write in our templated
  86                         // nginx config. Choose a port that will
  87                         // return 502 Bad Gateway.
  88                         port = "9"
  89                 } else if host, port, err = internalPort(cmpt.svc); err != nil {
  90                         return fmt.Errorf("%s internal port: %w (%v)", cmpt.varname, err, cmpt.svc)
  91                 } else if ok, err := addrIsLocal(net.JoinHostPort(host, port)); !ok || err != nil {
  92                         return fmt.Errorf("%s addrIsLocal() failed for host %q port %q: %v", cmpt.varname, host, port, err)
  93                 }
  94                 vars[cmpt.varname+"PORT"] = port
  95
  96                 port, err = externalPort(cmpt.svc)
  97                 if err != nil {
  98                         return fmt.Errorf("%s external port: %w (%v)", cmpt.varname, err, cmpt.svc)
  99                 }
 100                 listenAddr := net.JoinHostPort(super.ListenHost, port)
 101                 if ok, err := addrIsLocal(listenAddr); !ok || err != nil {
 102                         return fmt.Errorf("%s addrIsLocal(%q) failed: %w", cmpt.varname, listenAddr, err)
 103                 }
 104                 vars[cmpt.varname+"SSLPORT"] = port
 105         }
 106         if portmin, portmax := super.cluster.Services.ContainerWebServices.ExternalPortMin,
 107                 super.cluster.Services.ContainerWebServices.ExternalPortMax; 0 < portmin && portmin < portmax {
 108                 vars["CONTROLLERLISTENEXTRA"] = fmt.Sprintf("listen %s:%d-%d ssl;", extListenHost, portmin, portmax)
 109         }
 110         var conftemplate string
 111         if super.ClusterType == "production" {
 112                 conftemplate = "/var/lib/arvados/share/nginx.conf"
 113         } else {
 114                 conftemplate = filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf")
 115         }
 116         tmpl, err := ioutil.ReadFile(conftemplate)
 117         if err != nil {
 118                 return err
 119         }
 120         conf := regexp.MustCompile(`{{.*?}}`).ReplaceAllStringFunc(string(tmpl), func(src string) string {
 121                 if len(src) < 4 {
 122                         return src
 123                 }
 124                 return vars[src[2:len(src)-2]]
 125         })
 126         conffile := filepath.Join(super.tempdir, "nginx.conf")
 127         err = ioutil.WriteFile(conffile, []byte(conf), 0755)
 128         if err != nil {
 129                 return err
 130         }
 131         nginx := "nginx"
 132         if _, err := exec.LookPath(nginx); err != nil {
 133                 for _, dir := range []string{"/sbin", "/usr/sbin", "/usr/local/sbin"} {
 134                         if _, err = os.Stat(dir + "/nginx"); err == nil {
 135                                 nginx = dir + "/nginx"
 136                                 break
 137                         }
 138                 }
 139         }
 140
 141         configs := "error_log stderr warn; "
 142         configs += "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + "; "
 143         configs += "user www-data; "
 144
 145         super.waitShutdown.Add(1)
 146         go func() {
 147                 defer super.waitShutdown.Done()
 148                 fail(super.RunProgram(ctx, ".", runOptions{}, nginx, "-g", configs, "-c", conffile))
 149         }()
 150         // Choose one of the ports where Nginx should listen, and wait
 151         // here until we can connect. If ExternalURL is https://foo
 152         // (with no port) then we connect to "foo:https"
 153         testurl := url.URL(super.cluster.Services.Controller.ExternalURL)
 154         if testurl.Port() == "" {
 155                 testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme)
 156         }
 157         return waitForConnect(ctx, testurl.Host)
 158 }
 159
 160 // Return 0 or more local subnets as "geo" fragments for Nginx config,
 161 // e.g., "1.2.3.0/24 0; 10.1.0.0/16 0;".
 162 func internalSubnets(logger logrus.FieldLogger) string {
 163         iproutes, err := exec.Command("ip", "route").CombinedOutput()
 164         if err != nil {
 165                 logger.Warnf("treating all clients as external because `ip route` failed: %s (%q)", err, iproutes)
 166                 return ""
 167         }
 168         subnets := ""
 169         for _, line := range bytes.Split(iproutes, []byte("\n")) {
 170                 fields := strings.Fields(string(line))
 171                 if len(fields) > 2 && fields[1] == "dev" {
 172                         // lan example:
 173                         // 192.168.86.0/24 dev ens3 proto kernel scope link src 192.168.86.196
 174                         // gcp example (private subnet):
 175                         // 10.47.0.0/24 dev eth0 proto kernel scope link src 10.47.0.5
 176                         // gcp example (no private subnet):
 177                         // 10.128.0.1 dev ens4 scope link
 178                         subnets += fields[0] + " 0; "
 179                 }
 180         }
 181         return subnets
 182 }