From 48a3b3a3c28a6590fdf3d2b750192706cb751fae Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Fri, 24 May 2019 13:22:12 -0400 Subject: [PATCH] 15003: Remove NodeProfiles section from cluster config. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- .../install-controller.html.textile.liquid | 11 +-- ...install-dispatch-cloud.html.textile.liquid | 15 +--- lib/config/deprecated.go | 24 +++++- lib/controller/cmd.go | 4 +- lib/controller/federation_test.go | 62 ++++----------- lib/controller/handler.go | 38 +++++----- lib/controller/handler_test.go | 12 +-- lib/controller/server_test.go | 15 ++-- lib/dispatchcloud/cmd.go | 4 +- lib/service/cmd.go | 48 ++++++++---- lib/service/cmd_test.go | 2 +- lib/service/error.go | 2 +- sdk/go/arvados/config.go | 75 +++++-------------- sdk/go/arvadostest/stub.go | 15 ++++ sdk/go/health/aggregator.go | 47 ++++++------ 15 files changed, 165 insertions(+), 209 deletions(-) diff --git a/doc/install/install-controller.html.textile.liquid b/doc/install/install-controller.html.textile.liquid index 3e94b290d5..394aa0fdf7 100644 --- a/doc/install/install-controller.html.textile.liquid +++ b/doc/install/install-controller.html.textile.liquid @@ -92,12 +92,13 @@ Create the cluster configuration file @/etc/arvados/config.yml@ using the follow 
Clusters:
   uuid_prefix:
-    NodeProfiles:
-      apiserver:
-        arvados-controller:
-          Listen: ":9004" # must match the "upstream controller" section of your Nginx config
+    Services:
+      Controller:
+        InternalURLs:
+          "http://localhost:9004": {} # must match the "upstream controller" section of your Nginx config
+      RailsAPI:
         arvados-api-server:
-          Listen: ":8000" # must match the "upstream api" section of your Nginx config
+          "http://localhost:8000": {} # must match the "upstream api" section of your Nginx config
     PostgreSQL:
       ConnectionPool: 128
       Connection:
diff --git a/doc/install/install-dispatch-cloud.html.textile.liquid b/doc/install/install-dispatch-cloud.html.textile.liquid
index 42c814b879..2ecf3c3774 100644
--- a/doc/install/install-dispatch-cloud.html.textile.liquid
+++ b/doc/install/install-dispatch-cloud.html.textile.liquid
@@ -66,14 +66,12 @@ Add or update the following portions of your cluster configuration file, @/etc/a
   uuid_prefix:
     ManagementToken: xyzzy
     SystemRootToken: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-    NodeProfiles:
-      # The key "apiserver" corresponds to ARVADOS_NODE_PROFILE in environment file (see below).
-      apiserver:
-        arvados-dispatch-cloud:
-          Listen: ":9006"
     Services:
       Controller:
         ExternalURL: "https://uuid_prefix.arvadosapi.com"
+      DispatchCloud:
+        InternalURLs:
+          "http://localhost:9006": {}
     CloudVMs:
       # BootProbeCommand is a shell command that succeeds when an instance is ready for service
       BootProbeCommand: "sudo systemctl status docker"
@@ -153,13 +151,6 @@ Minimal configuration example for Azure:
-Create the host configuration file @/etc/arvados/environment@. - - -
ARVADOS_NODE_PROFILE=apiserver
-
- - h2. Install the dispatcher First, "add the appropriate package repository for your distribution":{{ site.baseurl }}/install/install-manual-prerequisites.html#repos. diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go index c8f943f3cc..8ffa2a5834 100644 --- a/lib/config/deprecated.go +++ b/lib/config/deprecated.go @@ -20,13 +20,33 @@ type deprRequestLimits struct { type deprCluster struct { RequestLimits deprRequestLimits - NodeProfiles map[string]arvados.NodeProfile + NodeProfiles map[string]nodeProfile } type deprecatedConfig struct { Clusters map[string]deprCluster } +type nodeProfile struct { + Controller systemServiceInstance `json:"arvados-controller"` + Health systemServiceInstance `json:"arvados-health"` + Keepbalance systemServiceInstance `json:"keep-balance"` + Keepproxy systemServiceInstance `json:"keepproxy"` + Keepstore systemServiceInstance `json:"keepstore"` + Keepweb systemServiceInstance `json:"keep-web"` + Nodemanager systemServiceInstance `json:"arvados-node-manager"` + DispatchCloud systemServiceInstance `json:"arvados-dispatch-cloud"` + RailsAPI systemServiceInstance `json:"arvados-api-server"` + Websocket systemServiceInstance `json:"arvados-ws"` + Workbench1 systemServiceInstance `json:"arvados-workbench"` +} + +type systemServiceInstance struct { + Listen string + TLS bool + Insecure bool +} + func applyDeprecatedConfig(cfg *arvados.Config, configdata []byte, log logger) error { var dc deprecatedConfig err := yaml.Unmarshal(configdata, &dc) @@ -63,7 +83,7 @@ func applyDeprecatedConfig(cfg *arvados.Config, configdata []byte, log logger) e return nil } -func applyDeprecatedNodeProfile(hostname string, ssi arvados.SystemServiceInstance, svc *arvados.Service) { +func applyDeprecatedNodeProfile(hostname string, ssi systemServiceInstance, svc *arvados.Service) { scheme := "https" if !ssi.TLS { scheme = "http" diff --git a/lib/controller/cmd.go b/lib/controller/cmd.go index f0268091be..4345370469 100644 --- a/lib/controller/cmd.go +++ b/lib/controller/cmd.go @@ -14,6 +14,6 @@ import ( var Command cmd.Handler = service.Command(arvados.ServiceNameController, newHandler) -func newHandler(_ context.Context, cluster *arvados.Cluster, np *arvados.NodeProfile, _ string) service.Handler { - return &Handler{Cluster: cluster, NodeProfile: np} +func newHandler(_ context.Context, cluster *arvados.Cluster, _ string) service.Handler { + return &Handler{Cluster: cluster} } diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go index c4aa33c15e..1c859cfc51 100644 --- a/lib/controller/federation_test.go +++ b/lib/controller/federation_test.go @@ -54,25 +54,22 @@ func (s *FederationSuite) SetUpTest(c *check.C) { s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler) c.Assert(s.remoteMock.Start(), check.IsNil) - nodeProfile := arvados.NodeProfile{ - Controller: arvados.SystemServiceInstance{Listen: ":"}, - RailsAPI: arvados.SystemServiceInstance{Listen: ":1"}, // local reqs will error "connection refused" - } - s.testHandler = &Handler{Cluster: &arvados.Cluster{ + cluster := &arvados.Cluster{ ClusterID: "zhome", PostgreSQL: integrationTestCluster().PostgreSQL, - NodeProfiles: map[string]arvados.NodeProfile{ - "*": nodeProfile, - }, + TLS: arvados.TLS{Insecure: true}, API: arvados.API{ MaxItemsPerResponse: 1000, MaxRequestAmplification: 4, }, - }, NodeProfile: &nodeProfile} + } + arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/") + arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/") + s.testHandler = &Handler{Cluster: cluster} s.testServer = newServerFromIntegrationTestEnv(c) s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler)) - s.testHandler.Cluster.RemoteClusters = map[string]arvados.RemoteCluster{ + cluster.RemoteClusters = map[string]arvados.RemoteCluster{ "zzzzz": { Host: s.remoteServer.Addr, Proxy: true, @@ -318,16 +315,8 @@ func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *https Handler: h, }, } - c.Assert(srv.Start(), check.IsNil) - - np := arvados.NodeProfile{ - Controller: arvados.SystemServiceInstance{Listen: ":"}, - RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr, - TLS: false, Insecure: true}} - s.testHandler.Cluster.NodeProfiles["*"] = np - s.testHandler.NodeProfile = &np - + arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "http://"+srv.Addr) return srv } @@ -338,13 +327,8 @@ func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server } func (s *FederationSuite) TestGetLocalCollection(c *check.C) { - np := arvados.NodeProfile{ - Controller: arvados.SystemServiceInstance{Listen: ":"}, - RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), - TLS: true, Insecure: true}} s.testHandler.Cluster.ClusterID = "zzzzz" - s.testHandler.Cluster.NodeProfiles["*"] = np - s.testHandler.NodeProfile = &np + arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST")) // HTTP GET @@ -416,12 +400,7 @@ func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) { } func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) { - np := arvados.NodeProfile{ - Controller: arvados.SystemServiceInstance{Listen: ":"}, - RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), - TLS: true, Insecure: true}} - s.testHandler.Cluster.NodeProfiles["*"] = np - s.testHandler.NodeProfile = &np + arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST")) req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil) req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken) @@ -505,12 +484,7 @@ func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) { } func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) { - np := arvados.NodeProfile{ - Controller: arvados.SystemServiceInstance{Listen: ":"}, - RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), - TLS: true, Insecure: true}} - s.testHandler.Cluster.NodeProfiles["*"] = np - s.testHandler.NodeProfile = &np + arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST")) req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil) req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065") @@ -526,12 +500,7 @@ func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) { } func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) { - np := arvados.NodeProfile{ - Controller: arvados.SystemServiceInstance{Listen: ":"}, - RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), - TLS: true, Insecure: true}} - s.testHandler.Cluster.NodeProfiles["*"] = np - s.testHandler.NodeProfile = &np + arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST")) req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil) req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065") @@ -616,13 +585,8 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *c req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2) req.Header.Set("Content-type", "application/json") - np := arvados.NodeProfile{ - Controller: arvados.SystemServiceInstance{Listen: ":"}, - RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), - TLS: true, Insecure: true}} + arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST")) s.testHandler.Cluster.ClusterID = "zzzzz" - s.testHandler.Cluster.NodeProfiles["*"] = np - s.testHandler.NodeProfile = &np resp := s.testRequest(req) c.Check(resp.StatusCode, check.Equals, http.StatusOK) diff --git a/lib/controller/handler.go b/lib/controller/handler.go index 35734d780c..2c3ce1d4f2 100644 --- a/lib/controller/handler.go +++ b/lib/controller/handler.go @@ -8,7 +8,7 @@ import ( "context" "database/sql" "errors" - "net" + "fmt" "net/http" "net/url" "strings" @@ -22,8 +22,7 @@ import ( ) type Handler struct { - Cluster *arvados.Cluster - NodeProfile *arvados.NodeProfile + Cluster *arvados.Cluster setupOnce sync.Once handlerStack http.Handler @@ -61,7 +60,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { func (h *Handler) CheckHealth() error { h.setupOnce.Do(h.setup) - _, _, err := findRailsAPI(h.Cluster, h.NodeProfile) + _, _, err := findRailsAPI(h.Cluster) return err } @@ -127,7 +126,7 @@ func prepend(next http.Handler, middleware middlewareFunc) http.Handler { } func (h *Handler) localClusterRequest(req *http.Request) (*http.Response, error) { - urlOut, insecure, err := findRailsAPI(h.Cluster, h.NodeProfile) + urlOut, insecure, err := findRailsAPI(h.Cluster) if err != nil { return nil, err } @@ -153,22 +152,19 @@ func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next h } } -// For now, findRailsAPI always uses the rails API running on this -// node. -func findRailsAPI(cluster *arvados.Cluster, np *arvados.NodeProfile) (*url.URL, bool, error) { - hostport := np.RailsAPI.Listen - if len(hostport) > 1 && hostport[0] == ':' && strings.TrimRight(hostport[1:], "0123456789") == "" { - // ":12345" => connect to indicated port on localhost - hostport = "localhost" + hostport - } else if _, _, err := net.SplitHostPort(hostport); err == nil { - // "[::1]:12345" => connect to indicated address & port - } else { - return nil, false, err +// Use a localhost entry from Services.RailsAPI.InternalURLs if one is +// present, otherwise choose an arbitrary entry. +func findRailsAPI(cluster *arvados.Cluster) (*url.URL, bool, error) { + var best *url.URL + for target := range cluster.Services.RailsAPI.InternalURLs { + target := url.URL(target) + best = &target + if strings.HasPrefix(target.Host, "localhost:") || strings.HasPrefix(target.Host, "127.0.0.1:") || strings.HasPrefix(target.Host, "[::1]:") { + break + } } - proto := "http" - if np.RailsAPI.TLS { - proto = "https" + if best == nil { + return nil, false, fmt.Errorf("Services.RailsAPI.InternalURLs is empty") } - url, err := url.Parse(proto + "://" + hostport) - return url, np.RailsAPI.Insecure, err + return best, cluster.TLS.Insecure, nil } diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go index 01544a2b0b..a1efaacddf 100644 --- a/lib/controller/handler_test.go +++ b/lib/controller/handler_test.go @@ -42,15 +42,11 @@ func (s *HandlerSuite) SetUpTest(c *check.C) { s.cluster = &arvados.Cluster{ ClusterID: "zzzzz", PostgreSQL: integrationTestCluster().PostgreSQL, - NodeProfiles: map[string]arvados.NodeProfile{ - "*": { - Controller: arvados.SystemServiceInstance{Listen: ":"}, - RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), TLS: true, Insecure: true}, - }, - }, + TLS: arvados.TLS{Insecure: true}, } - node := s.cluster.NodeProfiles["*"] - s.handler = newHandler(s.ctx, s.cluster, &node, "") + arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST")) + arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/") + s.handler = newHandler(s.ctx, s.cluster, "") } func (s *HandlerSuite) TearDownTest(c *check.C) { diff --git a/lib/controller/server_test.go b/lib/controller/server_test.go index ae89c3d7ea..a398af97b2 100644 --- a/lib/controller/server_test.go +++ b/lib/controller/server_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "git.curoverse.com/arvados.git/sdk/go/arvados" + "git.curoverse.com/arvados.git/sdk/go/arvadostest" "git.curoverse.com/arvados.git/sdk/go/ctxlog" "git.curoverse.com/arvados.git/sdk/go/httpserver" check "gopkg.in/check.v1" @@ -32,23 +33,19 @@ func integrationTestCluster() *arvados.Cluster { func newServerFromIntegrationTestEnv(c *check.C) *httpserver.Server { log := ctxlog.TestLogger(c) - nodeProfile := arvados.NodeProfile{ - Controller: arvados.SystemServiceInstance{Listen: ":"}, - RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), TLS: true, Insecure: true}, - } handler := &Handler{Cluster: &arvados.Cluster{ ClusterID: "zzzzz", PostgreSQL: integrationTestCluster().PostgreSQL, - NodeProfiles: map[string]arvados.NodeProfile{ - "*": nodeProfile, - }, - }, NodeProfile: &nodeProfile} + TLS: arvados.TLS{Insecure: true}, + }} + arvadostest.SetServiceURL(&handler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST")) + arvadostest.SetServiceURL(&handler.Cluster.Services.Controller, "http://localhost:/") srv := &httpserver.Server{ Server: http.Server{ Handler: httpserver.AddRequestIDs(httpserver.LogRequests(log, handler)), }, - Addr: nodeProfile.Controller.Listen, + Addr: ":", } return srv } diff --git a/lib/dispatchcloud/cmd.go b/lib/dispatchcloud/cmd.go index 22ceb8aebe..ae6ac70e96 100644 --- a/lib/dispatchcloud/cmd.go +++ b/lib/dispatchcloud/cmd.go @@ -15,10 +15,10 @@ import ( var Command cmd.Handler = service.Command(arvados.ServiceNameDispatchCloud, newHandler) -func newHandler(ctx context.Context, cluster *arvados.Cluster, np *arvados.NodeProfile, token string) service.Handler { +func newHandler(ctx context.Context, cluster *arvados.Cluster, token string) service.Handler { ac, err := arvados.NewClientFromConfig(cluster) if err != nil { - return service.ErrorHandler(ctx, cluster, np, fmt.Errorf("error initializing client from cluster config: %s", err)) + return service.ErrorHandler(ctx, cluster, fmt.Errorf("error initializing client from cluster config: %s", err)) } d := &dispatcher{ Cluster: cluster, diff --git a/lib/service/cmd.go b/lib/service/cmd.go index 024459ca06..cfc40778fe 100644 --- a/lib/service/cmd.go +++ b/lib/service/cmd.go @@ -10,9 +10,12 @@ import ( "flag" "fmt" "io" + "log" + "net" "net/http" "net/url" "os" + "strings" "git.curoverse.com/arvados.git/lib/cmd" "git.curoverse.com/arvados.git/lib/config" @@ -28,7 +31,7 @@ type Handler interface { CheckHealth() error } -type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, _ *arvados.NodeProfile, token string) Handler +type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, token string) Handler type command struct { newHandler NewHandlerFunc @@ -62,7 +65,6 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout flags := flag.NewFlagSet("", flag.ContinueOnError) flags.SetOutput(stderr) configFile := flags.String("config", arvados.DefaultConfigFile, "Site configuration `file`") - nodeProfile := flags.String("node-profile", "", "`Name` of NodeProfiles config entry to use (if blank, use $ARVADOS_NODE_PROFILE or hostname reported by OS)") err = flags.Parse(args) if err == flag.ErrHelp { err = nil @@ -83,19 +85,10 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout }) ctx := ctxlog.Context(c.ctx, log) - profileName := *nodeProfile - if profileName == "" { - profileName = os.Getenv("ARVADOS_NODE_PROFILE") - } - profile, err := cluster.GetNodeProfile(profileName) + listen, err := getListenAddr(cluster.Services, c.svcName) if err != nil { return 1 } - listen := profile.ServicePorts()[c.svcName] - if listen == "" { - err = fmt.Errorf("configuration does not enable the %s service on this host", c.svcName) - return 1 - } if cluster.SystemRootToken == "" { log.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable") @@ -114,7 +107,7 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout } } - handler := c.newHandler(ctx, cluster, profile, cluster.SystemRootToken) + handler := c.newHandler(ctx, cluster, cluster.SystemRootToken) if err = handler.CheckHealth(); err != nil { return 1 } @@ -147,3 +140,32 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout } const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" + +func getListenAddr(svcs arvados.Services, prog arvados.ServiceName) (string, error) { + svc, ok := map[arvados.ServiceName]arvados.Service{ + arvados.ServiceNameController: svcs.Controller, + arvados.ServiceNameDispatchCloud: svcs.DispatchCloud, + arvados.ServiceNameHealth: svcs.Health, + arvados.ServiceNameKeepbalance: svcs.Keepbalance, + arvados.ServiceNameKeepproxy: svcs.Keepproxy, + arvados.ServiceNameKeepstore: svcs.Keepstore, + arvados.ServiceNameKeepweb: svcs.WebDAV, + arvados.ServiceNameWebsocket: svcs.Websocket, + }[prog] + if !ok { + return "", fmt.Errorf("unknown service name %q", prog) + } + for url := range svc.InternalURLs { + if strings.HasPrefix(url.Host, "localhost:") { + return url.Host, nil + } + listener, err := net.Listen("tcp", url.Host) + if err == nil { + listener.Close() + return url.Host, nil + } + log.Print(err) + + } + return "", fmt.Errorf("configuration does not enable the %s service on this host", prog) +} diff --git a/lib/service/cmd_test.go b/lib/service/cmd_test.go index 62960dc31c..bb7c5c51da 100644 --- a/lib/service/cmd_test.go +++ b/lib/service/cmd_test.go @@ -38,7 +38,7 @@ func (*Suite) TestCommand(c *check.C) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, _ *arvados.NodeProfile, token string) Handler { + cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, token string) Handler { c.Check(ctx.Value("foo"), check.Equals, "bar") c.Check(token, check.Equals, "abcde") return &testHandler{ctx: ctx, healthCheck: healthCheck} diff --git a/lib/service/error.go b/lib/service/error.go index 8955210913..1ca5c5f446 100644 --- a/lib/service/error.go +++ b/lib/service/error.go @@ -17,7 +17,7 @@ import ( // responds 500 to all requests. ErrorHandler itself logs the given // error once, and the handler logs it again for each incoming // request. -func ErrorHandler(ctx context.Context, _ *arvados.Cluster, _ *arvados.NodeProfile, err error) Handler { +func ErrorHandler(ctx context.Context, _ *arvados.Cluster, err error) Handler { logger := ctxlog.FromContext(ctx) logger.WithError(err).Error("unhealthy service") return errorHandler{err, logger} diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index b25164c3d1..4936aa270b 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "net/url" - "os" "git.curoverse.com/arvados.git/sdk/go/config" ) @@ -62,7 +61,6 @@ type Cluster struct { ManagementToken string SystemRootToken string Services Services - NodeProfiles map[string]NodeProfile InstanceTypes InstanceTypeMap Containers ContainersConfig RemoteClusters map[string]RemoteCluster @@ -234,51 +232,16 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { return nil } -// GetNodeProfile returns a NodeProfile for the given hostname. An -// error is returned if the appropriate configuration can't be -// determined (e.g., this does not appear to be a system node). If -// node is empty, use the OS-reported hostname. -func (cc *Cluster) GetNodeProfile(node string) (*NodeProfile, error) { - if node == "" { - hostname, err := os.Hostname() - if err != nil { - return nil, err - } - node = hostname - } - if cfg, ok := cc.NodeProfiles[node]; ok { - return &cfg, nil - } - // If node is not listed, but "*" gives a default system node - // config, use the default config. - if cfg, ok := cc.NodeProfiles["*"]; ok { - return &cfg, nil - } - return nil, fmt.Errorf("config does not provision host %q as a system node", node) -} - -type NodeProfile struct { - Controller SystemServiceInstance `json:"arvados-controller"` - Health SystemServiceInstance `json:"arvados-health"` - Keepbalance SystemServiceInstance `json:"keep-balance"` - Keepproxy SystemServiceInstance `json:"keepproxy"` - Keepstore SystemServiceInstance `json:"keepstore"` - Keepweb SystemServiceInstance `json:"keep-web"` - Nodemanager SystemServiceInstance `json:"arvados-node-manager"` - DispatchCloud SystemServiceInstance `json:"arvados-dispatch-cloud"` - RailsAPI SystemServiceInstance `json:"arvados-api-server"` - Websocket SystemServiceInstance `json:"arvados-ws"` - Workbench SystemServiceInstance `json:"arvados-workbench"` -} - type ServiceName string const ( ServiceNameRailsAPI ServiceName = "arvados-api-server" ServiceNameController ServiceName = "arvados-controller" ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud" + ServiceNameHealth ServiceName = "arvados-health" ServiceNameNodemanager ServiceName = "arvados-node-manager" - ServiceNameWorkbench ServiceName = "arvados-workbench" + ServiceNameWorkbench1 ServiceName = "arvados-workbench1" + ServiceNameWorkbench2 ServiceName = "arvados-workbench2" ServiceNameWebsocket ServiceName = "arvados-ws" ServiceNameKeepbalance ServiceName = "keep-balance" ServiceNameKeepweb ServiceName = "keep-web" @@ -288,27 +251,23 @@ const ( // ServicePorts returns the configured listening address (or "" if // disabled) for each service on the node. -func (np *NodeProfile) ServicePorts() map[ServiceName]string { - return map[ServiceName]string{ - ServiceNameRailsAPI: np.RailsAPI.Listen, - ServiceNameController: np.Controller.Listen, - ServiceNameDispatchCloud: np.DispatchCloud.Listen, - ServiceNameNodemanager: np.Nodemanager.Listen, - ServiceNameWorkbench: np.Workbench.Listen, - ServiceNameWebsocket: np.Websocket.Listen, - ServiceNameKeepbalance: np.Keepbalance.Listen, - ServiceNameKeepweb: np.Keepweb.Listen, - ServiceNameKeepproxy: np.Keepproxy.Listen, - ServiceNameKeepstore: np.Keepstore.Listen, +func (svcs Services) Map() map[ServiceName]Service { + return map[ServiceName]Service{ + ServiceNameRailsAPI: svcs.RailsAPI, + ServiceNameController: svcs.Controller, + ServiceNameDispatchCloud: svcs.DispatchCloud, + ServiceNameHealth: svcs.Health, + ServiceNameNodemanager: svcs.Nodemanager, + ServiceNameWorkbench1: svcs.Workbench1, + ServiceNameWorkbench2: svcs.Workbench2, + ServiceNameWebsocket: svcs.Websocket, + ServiceNameKeepbalance: svcs.Keepbalance, + ServiceNameKeepweb: svcs.WebDAV, + ServiceNameKeepproxy: svcs.Keepproxy, + ServiceNameKeepstore: svcs.Keepstore, } } -type SystemServiceInstance struct { - Listen string - TLS bool - Insecure bool -} - type TLS struct { Certificate string Key string diff --git a/sdk/go/arvadostest/stub.go b/sdk/go/arvadostest/stub.go index 89925a957d..6b24a38fd7 100644 --- a/sdk/go/arvadostest/stub.go +++ b/sdk/go/arvadostest/stub.go @@ -6,6 +6,9 @@ package arvadostest import ( "net/http" + "net/url" + + "git.curoverse.com/arvados.git/sdk/go/arvados" ) // StubResponse struct with response status and body @@ -37,3 +40,15 @@ func (stub *ServerStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.Write([]byte(``)) } } + +// SetServiceURL overrides the given service config/discovery with the +// given internalURL. +// +// SetServiceURL panics on errors. +func SetServiceURL(service *arvados.Service, internalURL string) { + u, err := url.Parse(internalURL) + if err != nil { + panic(err) + } + service.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL(*u): {}} +} diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go index 564331327a..d8c0a4abfb 100644 --- a/sdk/go/health/aggregator.go +++ b/sdk/go/health/aggregator.go @@ -9,8 +9,8 @@ import ( "encoding/json" "errors" "fmt" - "net" "net/http" + "net/url" "sync" "time" @@ -113,46 +113,41 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp mtx := sync.Mutex{} wg := sync.WaitGroup{} - for profileName, profile := range cluster.NodeProfiles { - for svc, addr := range profile.ServicePorts() { - // Ensure svc is listed in resp.Services. - mtx.Lock() - if _, ok := resp.Services[svc]; !ok { - resp.Services[svc] = ServiceHealth{Health: "ERROR"} - } - mtx.Unlock() - - if addr == "" { - // svc is not expected on this node. - continue - } + for svcName, svc := range cluster.Services.Map() { + // Ensure svc is listed in resp.Services. + mtx.Lock() + if _, ok := resp.Services[svcName]; !ok { + resp.Services[svcName] = ServiceHealth{Health: "ERROR"} + } + mtx.Unlock() + for addr := range svc.InternalURLs { wg.Add(1) - go func(profileName string, svc arvados.ServiceName, addr string) { + go func(svcName arvados.ServiceName, addr arvados.URL) { defer wg.Done() var result CheckResult - url, err := agg.pingURL(profileName, addr) + pingURL, err := agg.pingURL(addr) if err != nil { result = CheckResult{ Health: "ERROR", Error: err.Error(), } } else { - result = agg.ping(url, cluster) + result = agg.ping(pingURL, cluster) } mtx.Lock() defer mtx.Unlock() - resp.Checks[fmt.Sprintf("%s+%s", svc, url)] = result + resp.Checks[fmt.Sprintf("%s+%s", svcName, pingURL)] = result if result.Health == "OK" { - h := resp.Services[svc] + h := resp.Services[svcName] h.N++ h.Health = "OK" - resp.Services[svc] = h + resp.Services[svcName] = h } else { resp.Health = "ERROR" } - }(profileName, svc, addr) + }(svcName, addr) } } wg.Wait() @@ -168,12 +163,12 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp return resp } -func (agg *Aggregator) pingURL(node, addr string) (string, error) { - _, port, err := net.SplitHostPort(addr) - return "http://" + node + ":" + port + "/_health/ping", err +func (agg *Aggregator) pingURL(svcURL arvados.URL) (*url.URL, error) { + base := url.URL(svcURL) + return base.Parse("/_health/ping") } -func (agg *Aggregator) ping(url string, cluster *arvados.Cluster) (result CheckResult) { +func (agg *Aggregator) ping(target *url.URL, cluster *arvados.Cluster) (result CheckResult) { t0 := time.Now() var err error @@ -186,7 +181,7 @@ func (agg *Aggregator) ping(url string, cluster *arvados.Cluster) (result CheckR } }() - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequest("GET", target.String(), nil) if err != nil { return } -- 2.30.2