X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/36f8e449321e4fa02d88fee1fded14aa8ff81723..db118cca358662e57a3dd0c1186dce1f0a62ca52:/lib/controller/handler.go diff --git a/lib/controller/handler.go b/lib/controller/handler.go index a1b3848e5c..5d6bd9570c 100644 --- a/lib/controller/handler.go +++ b/lib/controller/handler.go @@ -5,106 +5,226 @@ package controller import ( - "io" - "net" + "context" + "fmt" "net/http" + "net/http/httptest" "net/url" "strings" "sync" - "git.curoverse.com/arvados.git/sdk/go/arvados" - "git.curoverse.com/arvados.git/sdk/go/health" - "git.curoverse.com/arvados.git/sdk/go/httpserver" + "git.arvados.org/arvados.git/lib/controller/api" + "git.arvados.org/arvados.git/lib/controller/federation" + "git.arvados.org/arvados.git/lib/controller/localdb" + "git.arvados.org/arvados.git/lib/controller/railsproxy" + "git.arvados.org/arvados.git/lib/controller/router" + "git.arvados.org/arvados.git/lib/ctrlctx" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/health" + "git.arvados.org/arvados.git/sdk/go/httpserver" + + // sqlx needs lib/pq to talk to PostgreSQL + _ "github.com/lib/pq" ) type Handler struct { - Cluster *arvados.Cluster - Node *arvados.SystemNode + Cluster *arvados.Cluster + BackgroundContext context.Context - setupOnce sync.Once - handlerStack http.Handler - proxyClient *arvados.Client + setupOnce sync.Once + federation *federation.Conn + handlerStack http.Handler + proxy *proxy + secureClient *http.Client + insecureClient *http.Client + dbConnector ctrlctx.DBConnector + limitLogCreate chan struct{} } func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { h.setupOnce.Do(h.setup) + if req.Method != "GET" && req.Method != "HEAD" { + // http.ServeMux returns 301 with a cleaned path if + // the incoming request has a double slash. Some + // clients (including the Go standard library) change + // the request method to GET when following a 301 + // redirect if the original method was not HEAD + // (RFC7231 6.4.2 specifically allows this in the case + // of POST). Thus "POST //foo" gets misdirected to + // "GET /foo". To avoid this, eliminate double slashes + // before passing the request to ServeMux. + for strings.Contains(req.URL.Path, "//") { + req.URL.Path = strings.Replace(req.URL.Path, "//", "/", -1) + } + } h.handlerStack.ServeHTTP(w, req) } +func (h *Handler) CheckHealth() error { + h.setupOnce.Do(h.setup) + _, err := h.dbConnector.GetDB(context.TODO()) + if err != nil { + return err + } + _, _, err = railsproxy.FindRailsAPI(h.Cluster) + if err != nil { + return err + } + if h.Cluster.API.VocabularyPath != "" { + req, err := http.NewRequest("GET", "/arvados/v1/vocabulary", nil) + if err != nil { + return err + } + var resp httptest.ResponseRecorder + h.handlerStack.ServeHTTP(&resp, req) + if resp.Result().StatusCode != http.StatusOK { + return fmt.Errorf("%d %s", resp.Result().StatusCode, resp.Result().Status) + } + } + return nil +} + +func (h *Handler) Done() <-chan struct{} { + return nil +} + +func neverRedirect(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + func (h *Handler) setup() { mux := http.NewServeMux() + healthFuncs := make(map[string]health.Func) + + h.dbConnector = ctrlctx.DBConnector{PostgreSQL: h.Cluster.PostgreSQL} + oidcAuthorizer := localdb.OIDCAccessTokenAuthorizer(h.Cluster, h.dbConnector.GetDB) + h.federation = federation.New(h.Cluster, &healthFuncs) + rtr := router.New(h.federation, router.Config{ + MaxRequestSize: h.Cluster.API.MaxRequestSize, + WrapCalls: api.ComposeWrappers( + ctrlctx.WrapCallsInTransactions(h.dbConnector.GetDB), + oidcAuthorizer.WrapCalls, + ctrlctx.WrapCallsWithAuth(h.Cluster)), + }) + + healthRoutes := health.Routes{"ping": func() error { _, err := h.dbConnector.GetDB(context.TODO()); return err }} + for name, f := range healthFuncs { + healthRoutes[name] = f + } mux.Handle("/_health/", &health.Handler{ Token: h.Cluster.ManagementToken, Prefix: "/_health/", + Routes: healthRoutes, }) - mux.Handle("/", http.HandlerFunc(h.proxyRailsAPI)) + mux.Handle("/arvados/v1/config", rtr) + mux.Handle("/arvados/v1/vocabulary", rtr) + mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr) // must come before .../users/ + mux.Handle("/arvados/v1/collections", rtr) + mux.Handle("/arvados/v1/collections/", rtr) + mux.Handle("/arvados/v1/users", rtr) + mux.Handle("/arvados/v1/users/", rtr) + mux.Handle("/arvados/v1/connect/", rtr) + mux.Handle("/arvados/v1/container_requests", rtr) + mux.Handle("/arvados/v1/container_requests/", rtr) + mux.Handle("/arvados/v1/groups", rtr) + mux.Handle("/arvados/v1/groups/", rtr) + mux.Handle("/arvados/v1/links", rtr) + mux.Handle("/arvados/v1/links/", rtr) + mux.Handle("/login", rtr) + mux.Handle("/logout", rtr) + mux.Handle("/arvados/v1/api_client_authorizations", rtr) + mux.Handle("/arvados/v1/api_client_authorizations/", rtr) + + hs := http.NotFoundHandler() + hs = prepend(hs, h.proxyRailsAPI) + hs = prepend(hs, h.limitLogCreateRequests) + hs = h.setupProxyRemoteCluster(hs) + hs = prepend(hs, oidcAuthorizer.Middleware) + mux.Handle("/", hs) h.handlerStack = mux + + sc := *arvados.DefaultSecureClient + sc.CheckRedirect = neverRedirect + h.secureClient = &sc + + ic := *arvados.InsecureHTTPClient + ic.CheckRedirect = neverRedirect + h.insecureClient = &ic + + logCreateLimit := int(float64(h.Cluster.API.MaxConcurrentRequests) * h.Cluster.API.LogCreateRequestFraction) + if logCreateLimit == 0 && h.Cluster.API.LogCreateRequestFraction > 0 { + logCreateLimit = 1 + } + h.limitLogCreate = make(chan struct{}, logCreateLimit) + + h.proxy = &proxy{ + Name: "arvados-controller", + } + + go h.trashSweepWorker() + go h.containerLogSweepWorker() +} + +type middlewareFunc func(http.ResponseWriter, *http.Request, http.Handler) + +func prepend(next http.Handler, middleware middlewareFunc) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + middleware(w, req, next) + }) } -func (h *Handler) proxyRailsAPI(w http.ResponseWriter, reqIn *http.Request) { - urlOut, err := findRailsAPI(h.Cluster, h.Node) +func (h *Handler) localClusterRequest(req *http.Request) (*http.Response, error) { + urlOut, insecure, err := railsproxy.FindRailsAPI(h.Cluster) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + return nil, err } urlOut = &url.URL{ Scheme: urlOut.Scheme, Host: urlOut.Host, - Path: reqIn.URL.Path, - RawPath: reqIn.URL.RawPath, - RawQuery: reqIn.URL.RawQuery, + Path: req.URL.Path, + RawPath: req.URL.RawPath, + RawQuery: req.URL.RawQuery, } - - // Copy headers from incoming request, then add/replace proxy - // headers like Via and X-Forwarded-For. - hdrOut := http.Header{} - for k, v := range reqIn.Header { - hdrOut[k] = v + client := h.secureClient + if insecure { + client = h.insecureClient } - xff := reqIn.RemoteAddr - if xffIn := reqIn.Header.Get("X-Forwarded-For"); xffIn != "" { - xff = xffIn + "," + xff - } - hdrOut.Set("X-Forwarded-For", xff) - hdrOut.Add("Via", reqIn.Proto+" arvados-controller") - - reqOut := (&http.Request{ - Method: reqIn.Method, - URL: urlOut, - Header: hdrOut, - }).WithContext(reqIn.Context()) - resp, err := arvados.InsecureHTTPClient.Do(reqOut) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - for k, v := range resp.Header { - for _, v := range v { - w.Header().Add(k, v) + return h.proxy.Do(req, urlOut, client) +} + +func (h *Handler) limitLogCreateRequests(w http.ResponseWriter, req *http.Request, next http.Handler) { + if cap(h.limitLogCreate) > 0 && req.Method == http.MethodPost && strings.HasPrefix(req.URL.Path, "/arvados/v1/logs") { + select { + case h.limitLogCreate <- struct{}{}: + defer func() { <-h.limitLogCreate }() + next.ServeHTTP(w, req) + default: + http.Error(w, "Excess log messages", http.StatusServiceUnavailable) } + return } - w.WriteHeader(resp.StatusCode) - n, err := io.Copy(w, resp.Body) + next.ServeHTTP(w, req) +} + +func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next http.Handler) { + resp, err := h.localClusterRequest(req) + n, err := h.proxy.ForwardResponse(w, resp, err) if err != nil { - httpserver.Logger(reqIn).WithError(err).WithField("bytesCopied", n).Error("error copying response body") + httpserver.Logger(req).WithError(err).WithField("bytesCopied", n).Error("error copying response body") } } -// For now, findRailsAPI always uses the rails API running on this -// node. -func findRailsAPI(cluster *arvados.Cluster, node *arvados.SystemNode) (*url.URL, error) { - hostport := node.RailsAPI.Listen - if len(hostport) > 1 && hostport[0] == ':' && strings.TrimRight(hostport[1:], "0123456789") == "" { - // ":12345" => connect to indicated port on localhost - hostport = "localhost" + hostport - } else if _, _, err := net.SplitHostPort(hostport); err == nil { - // "[::1]:12345" => connect to indicated address & port - } else { - return nil, err +// Use a localhost entry from Services.RailsAPI.InternalURLs if one is +// present, otherwise choose an arbitrary entry. +func findRailsAPI(cluster *arvados.Cluster) (*url.URL, bool, error) { + var best *url.URL + for target := range cluster.Services.RailsAPI.InternalURLs { + target := url.URL(target) + best = &target + if strings.HasPrefix(target.Host, "localhost:") || strings.HasPrefix(target.Host, "127.0.0.1:") || strings.HasPrefix(target.Host, "[::1]:") { + break + } } - proto := "http" - if node.RailsAPI.TLS { - proto = "https" + if best == nil { + return nil, false, fmt.Errorf("Services.RailsAPI.InternalURLs is empty") } - return url.Parse(proto + "://" + hostport) + return best, cluster.TLS.Insecure, nil }