From: Tom Clegg Date: Tue, 13 Jul 2021 20:16:36 +0000 (-0400) Subject: 16561: Add ListenAddress, explain InternalURLs/ExternalURL better. X-Git-Tag: 2.5.0~128^2~7 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/2c61bf045e35d8971412880031412c4f1a9660f6?ds=sidebyside 16561: Add ListenAddress, explain InternalURLs/ExternalURL better. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml index a9bbf4eee9..c321434cb1 100644 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@ -22,34 +22,67 @@ Clusters: Services: - # In each of the service sections below, the keys under - # InternalURLs are the endpoints where the service should be - # listening, and reachable from other hosts in the - # cluster. Example: + # Each of the service sections below specifies ListenAddress, + # InternalURLs, and ExternalURL. + # + # InternalURLs specify how other Arvados service processes will + # connect to the service. Typically these use internal hostnames + # and high port numbers. Example: # # InternalURLs: - # "http://host1.example:12345": {} - # "http://host2.example:12345": {} + # "http://host1.internal.example:12345": {} + # "http://host2.internal.example:12345": {} + # + # ListenAddress specifies the address and port the service + # process's HTTP server should listen on. Example: + # + # ListenAddress: "0.0.0.0:12345" + # + # If ListenAddress is blank, the service will try listening on + # the host:port part of each InternalURLs entry until one + # works. This approach only works if the host names resolve (via + # /etc/hosts, DNS, etc) to the IP addresses of the host's + # network interfaces. + # + # ExternalURL specifies how applications/clients will connect to + # the service, regardless of whether they are inside or outside + # the cluster. Example: + # + # ExternalURL: "https://keep.zzzzz.example.com/" + # + # To avoid routing internal traffic through external networks, + # use split-horizon DNS for ExternalURL host names: inside the + # cluster's private network "host.zzzzz.example.com" resolves to + # the host's private IP address, while outside the cluster + # "host.zzzzz.example.com" resolves to the host's public IP + # address (or its external gateway or load balancer). RailsAPI: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" Controller: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" Websocket: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" Keepbalance: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" GitHTTP: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" GitSSH: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" DispatchCloud: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" DispatchLSF: @@ -59,9 +92,11 @@ Clusters: InternalURLs: {SAMPLE: {}} ExternalURL: "" Keepproxy: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" WebDAV: + ListenAddress: "" InternalURLs: {SAMPLE: {}} # Base URL for Workbench inline preview. If blank, use # WebDAVDownload instead, and disable inline preview. @@ -101,6 +136,7 @@ Clusters: ExternalURL: "" WebDAVDownload: + ListenAddress: "" InternalURLs: {SAMPLE: {}} # Base URL for download links. If blank, serve links to WebDAV # with disposition=attachment query param. Unlike preview links, @@ -115,6 +151,7 @@ Clusters: ExternalURL: "" Keepstore: + ListenAddress: "" InternalURLs: SAMPLE: # Rendezvous is normally empty/omitted. When changing the @@ -124,9 +161,11 @@ Clusters: Rendezvous: "" ExternalURL: "" Composer: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" WebShell: + ListenAddress: "" InternalURLs: {SAMPLE: {}} # ShellInABox service endpoint URL for a given VM. If empty, do not # offer web shell logins. @@ -138,12 +177,15 @@ Clusters: # https://*.webshell.uuid_prefix.arvadosapi.com ExternalURL: "" Workbench1: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" Workbench2: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" Health: + ListenAddress: "" InternalURLs: {SAMPLE: {}} ExternalURL: "" diff --git a/lib/config/export.go b/lib/config/export.go index a55295d126..56090afec9 100644 --- a/lib/config/export.go +++ b/lib/config/export.go @@ -214,6 +214,7 @@ var whitelist = map[string]bool{ "Services.*": true, "Services.*.ExternalURL": true, "Services.*.InternalURLs": false, + "Services.*.ListenAddress": false, "StorageClasses": true, "StorageClasses.*": true, "StorageClasses.*.Default": true, diff --git a/lib/service/cmd.go b/lib/service/cmd.go index 679cbede13..a9d5731eac 100644 --- a/lib/service/cmd.go +++ b/lib/service/cmd.go @@ -121,11 +121,11 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout }) ctx := ctxlog.Context(c.ctx, logger) - listenURL, err := getListenAddr(cluster.Services, c.svcName, log) + listenURL, internalURL, err := getListenAddr(cluster.Services, c.svcName, log) if err != nil { return 1 } - ctx = context.WithValue(ctx, contextKeyURL{}, listenURL) + ctx = context.WithValue(ctx, contextKeyURL{}, internalURL) reg := prometheus.NewRegistry() loader.RegisterMetrics(reg) @@ -157,7 +157,7 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout }, Addr: listenURL.Host, } - if listenURL.Scheme == "https" { + if listenURL.Scheme == "https" || listenURL.Scheme == "wss" { tlsconfig, err := tlsConfigWithCertUpdater(cluster, logger) if err != nil { logger.WithError(err).Errorf("cannot start %s service on %s", c.svcName, listenURL.String()) @@ -223,28 +223,41 @@ func interceptHealthReqs(mgtToken string, checkHealth func() error, next http.Ha return ifCollectionInHost(next, mux) } -func getListenAddr(svcs arvados.Services, prog arvados.ServiceName, log logrus.FieldLogger) (arvados.URL, error) { +func getListenAddr(svcs arvados.Services, prog arvados.ServiceName, log logrus.FieldLogger) (arvados.URL, arvados.URL, error) { svc, ok := svcs.Map()[prog] if !ok { - return arvados.URL{}, fmt.Errorf("unknown service name %q", prog) + return arvados.URL{}, arvados.URL{}, fmt.Errorf("unknown service name %q", prog) } if want := os.Getenv("ARVADOS_SERVICE_INTERNAL_URL"); want == "" { } else if url, err := url.Parse(want); err != nil { - return arvados.URL{}, fmt.Errorf("$ARVADOS_SERVICE_INTERNAL_URL (%q): %s", want, err) + return arvados.URL{}, arvados.URL{}, fmt.Errorf("$ARVADOS_SERVICE_INTERNAL_URL (%q): %s", want, err) } else { if url.Path == "" { url.Path = "/" } - return arvados.URL(*url), nil + internalURL := arvados.URL(*url) + listenURL := arvados.URL(*url) + if svc.ListenAddress != "" { + listenURL.Host = svc.ListenAddress + } + return listenURL, internalURL, nil + } + + if svc.ListenAddress != "" { + for internalURL := range svc.InternalURLs { + listenURL := internalURL + listenURL.Host = svc.ListenAddress + return listenURL, internalURL, nil + } } errors := []string{} - for url := range svc.InternalURLs { - listener, err := net.Listen("tcp", url.Host) + for internalURL := range svc.InternalURLs { + listener, err := net.Listen("tcp", internalURL.Host) if err == nil { listener.Close() - return url, nil + return internalURL, internalURL, nil } else if strings.Contains(err.Error(), "cannot assign requested address") { // If 'Host' specifies a different server than // the current one, it'll resolve the hostname @@ -252,13 +265,13 @@ func getListenAddr(svcs arvados.Services, prog arvados.ServiceName, log logrus.F // can't bind an IP address it doesn't own. continue } else { - errors = append(errors, fmt.Sprintf("tried %v, got %v", url, err)) + errors = append(errors, fmt.Sprintf("tried %v, got %v", internalURL, err)) } } if len(errors) > 0 { - return arvados.URL{}, fmt.Errorf("could not enable the %q service on this host: %s", prog, strings.Join(errors, "; ")) + return arvados.URL{}, arvados.URL{}, fmt.Errorf("could not enable the %q service on this host: %s", prog, strings.Join(errors, "; ")) } - return arvados.URL{}, fmt.Errorf("configuration does not enable the %q service on this host", prog) + return arvados.URL{}, arvados.URL{}, fmt.Errorf("configuration does not enable the %q service on this host", prog) } type contextKeyURL struct{} diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 0d8f293124..ad663b23ef 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -366,8 +366,9 @@ type Services struct { } type Service struct { - InternalURLs map[URL]ServiceInstance - ExternalURL URL + ListenAddress string + InternalURLs map[URL]ServiceInstance + ExternalURL URL } type TestUser struct {