X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0596129229750c593066e414d9315f643585bc3e..HEAD:/lib/controller/handler.go diff --git a/lib/controller/handler.go b/lib/controller/handler.go index e1392bef92..7c4bb0912f 100644 --- a/lib/controller/handler.go +++ b/lib/controller/handler.go @@ -6,13 +6,17 @@ package controller import ( "context" + "encoding/json" "errors" "fmt" + "io/ioutil" + "mime" "net/http" "net/http/httptest" "net/url" "strings" "sync" + "time" "git.arvados.org/arvados.git/lib/controller/api" "git.arvados.org/arvados.git/lib/controller/federation" @@ -24,7 +28,6 @@ import ( "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/health" "git.arvados.org/arvados.git/sdk/go/httpserver" - "github.com/jmoiron/sqlx" // sqlx needs lib/pq to talk to PostgreSQL _ "github.com/lib/pq" @@ -40,8 +43,10 @@ type Handler struct { proxy *proxy secureClient *http.Client insecureClient *http.Client - pgdb *sqlx.DB - pgdbMtx sync.Mutex + dbConnector ctrlctx.DBConnector + limitLogCreate chan struct{} + + cache map[string]*cacheEnt } func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -65,7 +70,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { func (h *Handler) CheckHealth() error { h.setupOnce.Do(h.setup) - _, err := h.db(context.TODO()) + _, err := h.dbConnector.GetDB(context.TODO()) if err != nil { return err } @@ -97,17 +102,22 @@ func (h *Handler) setup() { mux := http.NewServeMux() healthFuncs := make(map[string]health.Func) - oidcAuthorizer := localdb.OIDCAccessTokenAuthorizer(h.Cluster, h.db) - h.federation = federation.New(h.Cluster, &healthFuncs) + h.dbConnector = ctrlctx.DBConnector{PostgreSQL: h.Cluster.PostgreSQL} + go func() { + <-h.BackgroundContext.Done() + h.dbConnector.Close() + }() + oidcAuthorizer := localdb.OIDCAccessTokenAuthorizer(h.Cluster, h.dbConnector.GetDB) + h.federation = federation.New(h.BackgroundContext, h.Cluster, &healthFuncs, h.dbConnector.GetDB) rtr := router.New(h.federation, router.Config{ MaxRequestSize: h.Cluster.API.MaxRequestSize, WrapCalls: api.ComposeWrappers( - ctrlctx.WrapCallsInTransactions(h.db), + ctrlctx.WrapCallsInTransactions(h.dbConnector.GetDB), oidcAuthorizer.WrapCalls, ctrlctx.WrapCallsWithAuth(h.Cluster)), }) - healthRoutes := health.Routes{"ping": func() error { _, err := h.db(context.TODO()); return err }} + healthRoutes := health.Routes{"ping": func() error { _, err := h.dbConnector.GetDB(context.TODO()); return err }} for name, f := range healthFuncs { healthRoutes[name] = f } @@ -130,6 +140,8 @@ func (h *Handler) setup() { mux.Handle("/arvados/v1/groups/", rtr) mux.Handle("/arvados/v1/links", rtr) mux.Handle("/arvados/v1/links/", rtr) + mux.Handle("/arvados/v1/authorized_keys", rtr) + mux.Handle("/arvados/v1/authorized_keys/", rtr) mux.Handle("/login", rtr) mux.Handle("/logout", rtr) mux.Handle("/arvados/v1/api_client_authorizations", rtr) @@ -137,6 +149,8 @@ func (h *Handler) setup() { hs := http.NotFoundHandler() hs = prepend(hs, h.proxyRailsAPI) + hs = prepend(hs, h.routeContainerEndpoints(rtr)) + hs = prepend(hs, h.limitLogCreateRequests) hs = h.setupProxyRemoteCluster(hs) hs = prepend(hs, oidcAuthorizer.Middleware) mux.Handle("/", hs) @@ -150,39 +164,23 @@ func (h *Handler) setup() { ic.CheckRedirect = neverRedirect h.insecureClient = &ic + logCreateLimit := int(float64(h.Cluster.API.MaxConcurrentRequests) * h.Cluster.API.LogCreateRequestFraction) + if logCreateLimit == 0 && h.Cluster.API.LogCreateRequestFraction > 0 { + logCreateLimit = 1 + } + h.limitLogCreate = make(chan struct{}, logCreateLimit) + h.proxy = &proxy{ Name: "arvados-controller", } + h.cache = map[string]*cacheEnt{ + "/discovery/v1/apis/arvados/v1/rest": &cacheEnt{validate: validateDiscoveryDoc}, + } go h.trashSweepWorker() go h.containerLogSweepWorker() } -var errDBConnection = errors.New("database connection error") - -func (h *Handler) db(ctx context.Context) (*sqlx.DB, error) { - h.pgdbMtx.Lock() - defer h.pgdbMtx.Unlock() - if h.pgdb != nil { - return h.pgdb, nil - } - - db, err := sqlx.Open("postgres", h.Cluster.PostgreSQL.Connection.String()) - if err != nil { - ctxlog.FromContext(ctx).WithError(err).Error("postgresql connect failed") - return nil, errDBConnection - } - if p := h.Cluster.PostgreSQL.ConnectionPool; p > 0 { - db.SetMaxOpenConns(p) - } - if err := db.Ping(); err != nil { - ctxlog.FromContext(ctx).WithError(err).Error("postgresql connect succeeded but ping failed") - return nil, errDBConnection - } - h.pgdb = db - return db, nil -} - type middlewareFunc func(http.ResponseWriter, *http.Request, http.Handler) func prepend(next http.Handler, middleware middlewareFunc) http.Handler { @@ -207,10 +205,169 @@ func (h *Handler) localClusterRequest(req *http.Request) (*http.Response, error) if insecure { client = h.insecureClient } + // Clearing the Host field here causes the Go http client to + // use the host part of urlOut as the Host header in the + // outgoing request, instead of the Host value from the + // original request we received. + req.Host = "" return h.proxy.Do(req, urlOut, client) } +// Route /arvados/v1/containers/{uuid}/log*, .../ssh, and +// .../gateway_tunnel to rtr, pass everything else to next. +// +// (http.ServeMux doesn't let us route these without also routing +// everything under /containers/, which we don't want yet.) +func (h *Handler) routeContainerEndpoints(rtr http.Handler) middlewareFunc { + return func(w http.ResponseWriter, req *http.Request, next http.Handler) { + trim := strings.TrimPrefix(req.URL.Path, "/arvados/v1/containers/") + if trim != req.URL.Path && (strings.Index(trim, "/log") == 27 || + strings.Index(trim, "/ssh") == 27 || + strings.Index(trim, "/gateway_tunnel") == 27) { + rtr.ServeHTTP(w, req) + } else { + next.ServeHTTP(w, req) + } + } +} + +func (h *Handler) limitLogCreateRequests(w http.ResponseWriter, req *http.Request, next http.Handler) { + if cap(h.limitLogCreate) > 0 && req.Method == http.MethodPost && strings.HasPrefix(req.URL.Path, "/arvados/v1/logs") { + select { + case h.limitLogCreate <- struct{}{}: + defer func() { <-h.limitLogCreate }() + next.ServeHTTP(w, req) + default: + http.Error(w, "Excess log messages", http.StatusServiceUnavailable) + } + return + } + next.ServeHTTP(w, req) +} + +// cacheEnt implements a basic stale-while-revalidate cache, suitable +// for the Arvados discovery document. +type cacheEnt struct { + validate func(body []byte) error + mtx sync.Mutex + header http.Header + body []byte + expireAfter time.Time + refreshAfter time.Time + refreshLock sync.Mutex +} + +const ( + cacheTTL = 5 * time.Minute + cacheExpire = 24 * time.Hour +) + +func (ent *cacheEnt) refresh(path string, do func(*http.Request) (*http.Response, error)) (http.Header, []byte, error) { + ent.refreshLock.Lock() + defer ent.refreshLock.Unlock() + if header, body, needRefresh := ent.response(); !needRefresh { + // another goroutine refreshed successfully while we + // were waiting for refreshLock + return header, body, nil + } else if body != nil { + // Cache is present, but expired. We'll try to refresh + // below. Meanwhile, other refresh() calls will queue + // up for refreshLock -- and we don't want them to + // turn into N upstream requests, even if upstream is + // failing. (If we succeed we'll update the expiry + // time again below with the real cacheTTL -- this + // just takes care of the error case.) + ent.mtx.Lock() + ent.refreshAfter = time.Now().Add(time.Second) + ent.mtx.Unlock() + } + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute)) + defer cancel() + // "http://localhost" is just a placeholder here -- we'll fill + // in req.URL.Path below, and then do(), which is + // localClusterRequest(), will replace the scheme and host + // parts with the real proxy destination. + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost", nil) + if err != nil { + return nil, nil, err + } + req.URL.Path = path + resp, err := do(req) + if err != nil { + return nil, nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("HTTP status %d", resp.StatusCode) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, nil, fmt.Errorf("Read error: %w", err) + } + header := http.Header{} + for k, v := range resp.Header { + if !dropHeaders[k] && k != "X-Request-Id" { + header[k] = v + } + } + if ent.validate != nil { + if err := ent.validate(body); err != nil { + return nil, nil, err + } + } else if mediatype, _, err := mime.ParseMediaType(header.Get("Content-Type")); err == nil && mediatype == "application/json" { + if !json.Valid(body) { + return nil, nil, errors.New("invalid JSON encoding in response") + } + } + ent.mtx.Lock() + defer ent.mtx.Unlock() + ent.header = header + ent.body = body + ent.refreshAfter = time.Now().Add(cacheTTL) + ent.expireAfter = time.Now().Add(cacheExpire) + return ent.header, ent.body, nil +} + +func (ent *cacheEnt) response() (http.Header, []byte, bool) { + ent.mtx.Lock() + defer ent.mtx.Unlock() + if ent.expireAfter.Before(time.Now()) { + ent.header, ent.body, ent.refreshAfter = nil, nil, time.Time{} + } + return ent.header, ent.body, ent.refreshAfter.Before(time.Now()) +} + +func (ent *cacheEnt) ServeHTTP(ctx context.Context, w http.ResponseWriter, path string, do func(*http.Request) (*http.Response, error)) { + header, body, needRefresh := ent.response() + if body == nil { + // need to fetch before we can return anything + var err error + header, body, err = ent.refresh(path, do) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + } else if needRefresh { + // re-fetch in background + go func() { + _, _, err := ent.refresh(path, do) + if err != nil { + ctxlog.FromContext(ctx).WithError(err).WithField("path", path).Warn("error refreshing cache") + } + }() + } + for k, v := range header { + w.Header()[k] = v + } + w.WriteHeader(http.StatusOK) + w.Write(body) +} + func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next http.Handler) { + if ent, ok := h.cache[req.URL.Path]; ok && req.Method == http.MethodGet { + ent.ServeHTTP(req.Context(), w, req.URL.Path, h.localClusterRequest) + return + } resp, err := h.localClusterRequest(req) n, err := h.proxy.ForwardResponse(w, resp, err) if err != nil { @@ -234,3 +391,15 @@ func findRailsAPI(cluster *arvados.Cluster) (*url.URL, bool, error) { } return best, cluster.TLS.Insecure, nil } + +func validateDiscoveryDoc(body []byte) error { + var dd arvados.DiscoveryDocument + err := json.Unmarshal(body, &dd) + if err != nil { + return fmt.Errorf("error decoding JSON response: %w", err) + } + if dd.BasePath == "" { + return errors.New("error in discovery document: no value for basePath") + } + return nil +}