1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
21 "git.arvados.org/arvados.git/lib/controller/api"
22 "git.arvados.org/arvados.git/lib/controller/federation"
23 "git.arvados.org/arvados.git/lib/controller/localdb"
24 "git.arvados.org/arvados.git/lib/controller/railsproxy"
25 "git.arvados.org/arvados.git/lib/controller/router"
26 "git.arvados.org/arvados.git/lib/ctrlctx"
27 "git.arvados.org/arvados.git/sdk/go/arvados"
28 "git.arvados.org/arvados.git/sdk/go/ctxlog"
29 "git.arvados.org/arvados.git/sdk/go/health"
30 "git.arvados.org/arvados.git/sdk/go/httpserver"
32 // sqlx needs lib/pq to talk to PostgreSQL
37 Cluster *arvados.Cluster
38 BackgroundContext context.Context
41 federation *federation.Conn
42 handlerStack http.Handler
44 secureClient *http.Client
45 insecureClient *http.Client
46 dbConnector ctrlctx.DBConnector
47 limitLogCreate chan struct{}
49 cache map[string]*cacheEnt
52 func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
53 h.setupOnce.Do(h.setup)
54 if req.Method != "GET" && req.Method != "HEAD" {
55 // http.ServeMux returns 301 with a cleaned path if
56 // the incoming request has a double slash. Some
57 // clients (including the Go standard library) change
58 // the request method to GET when following a 301
59 // redirect if the original method was not HEAD
60 // (RFC7231 6.4.2 specifically allows this in the case
61 // of POST). Thus "POST //foo" gets misdirected to
62 // "GET /foo". To avoid this, eliminate double slashes
63 // before passing the request to ServeMux.
64 for strings.Contains(req.URL.Path, "//") {
65 req.URL.Path = strings.Replace(req.URL.Path, "//", "/", -1)
68 h.handlerStack.ServeHTTP(w, req)
71 func (h *Handler) CheckHealth() error {
72 h.setupOnce.Do(h.setup)
73 _, err := h.dbConnector.GetDB(context.TODO())
77 _, _, err = railsproxy.FindRailsAPI(h.Cluster)
81 if h.Cluster.API.VocabularyPath != "" {
82 req, err := http.NewRequest("GET", "/arvados/v1/vocabulary", nil)
86 var resp httptest.ResponseRecorder
87 h.handlerStack.ServeHTTP(&resp, req)
88 if resp.Result().StatusCode != http.StatusOK {
89 return fmt.Errorf("%d %s", resp.Result().StatusCode, resp.Result().Status)
95 func (h *Handler) Done() <-chan struct{} {
99 func neverRedirect(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
101 func (h *Handler) setup() {
102 mux := http.NewServeMux()
103 healthFuncs := make(map[string]health.Func)
105 h.dbConnector = ctrlctx.DBConnector{PostgreSQL: h.Cluster.PostgreSQL}
107 <-h.BackgroundContext.Done()
108 h.dbConnector.Close()
110 oidcAuthorizer := localdb.OIDCAccessTokenAuthorizer(h.Cluster, h.dbConnector.GetDB)
111 h.federation = federation.New(h.BackgroundContext, h.Cluster, &healthFuncs, h.dbConnector.GetDB)
112 rtr := router.New(h.federation, router.Config{
113 MaxRequestSize: h.Cluster.API.MaxRequestSize,
114 WrapCalls: api.ComposeWrappers(
115 ctrlctx.WrapCallsInTransactions(h.dbConnector.GetDB),
116 oidcAuthorizer.WrapCalls,
117 ctrlctx.WrapCallsWithAuth(h.Cluster)),
120 healthRoutes := health.Routes{"ping": func() error { _, err := h.dbConnector.GetDB(context.TODO()); return err }}
121 for name, f := range healthFuncs {
122 healthRoutes[name] = f
124 mux.Handle("/_health/", &health.Handler{
125 Token: h.Cluster.ManagementToken,
127 Routes: healthRoutes,
129 mux.Handle("/arvados/v1/config", rtr)
130 mux.Handle("/arvados/v1/vocabulary", rtr)
131 mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr) // must come before .../users/
132 mux.Handle("/arvados/v1/collections", rtr)
133 mux.Handle("/arvados/v1/collections/", rtr)
134 mux.Handle("/arvados/v1/users", rtr)
135 mux.Handle("/arvados/v1/users/", rtr)
136 mux.Handle("/arvados/v1/connect/", rtr)
137 mux.Handle("/arvados/v1/container_requests", rtr)
138 mux.Handle("/arvados/v1/container_requests/", rtr)
139 mux.Handle("/arvados/v1/groups", rtr)
140 mux.Handle("/arvados/v1/groups/", rtr)
141 mux.Handle("/arvados/v1/links", rtr)
142 mux.Handle("/arvados/v1/links/", rtr)
143 mux.Handle("/login", rtr)
144 mux.Handle("/logout", rtr)
145 mux.Handle("/arvados/v1/api_client_authorizations", rtr)
146 mux.Handle("/arvados/v1/api_client_authorizations/", rtr)
148 hs := http.NotFoundHandler()
149 hs = prepend(hs, h.proxyRailsAPI)
150 hs = prepend(hs, h.limitLogCreateRequests)
151 hs = h.setupProxyRemoteCluster(hs)
152 hs = prepend(hs, oidcAuthorizer.Middleware)
156 sc := *arvados.DefaultSecureClient
157 sc.CheckRedirect = neverRedirect
160 ic := *arvados.InsecureHTTPClient
161 ic.CheckRedirect = neverRedirect
162 h.insecureClient = &ic
164 logCreateLimit := int(float64(h.Cluster.API.MaxConcurrentRequests) * h.Cluster.API.LogCreateRequestFraction)
165 if logCreateLimit == 0 && h.Cluster.API.LogCreateRequestFraction > 0 {
168 h.limitLogCreate = make(chan struct{}, logCreateLimit)
171 Name: "arvados-controller",
173 h.cache = map[string]*cacheEnt{
174 "/discovery/v1/apis/arvados/v1/rest": &cacheEnt{},
177 go h.trashSweepWorker()
178 go h.containerLogSweepWorker()
181 type middlewareFunc func(http.ResponseWriter, *http.Request, http.Handler)
183 func prepend(next http.Handler, middleware middlewareFunc) http.Handler {
184 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
185 middleware(w, req, next)
189 func (h *Handler) localClusterRequest(req *http.Request) (*http.Response, error) {
190 urlOut, insecure, err := railsproxy.FindRailsAPI(h.Cluster)
195 Scheme: urlOut.Scheme,
198 RawPath: req.URL.RawPath,
199 RawQuery: req.URL.RawQuery,
201 client := h.secureClient
203 client = h.insecureClient
205 return h.proxy.Do(req, urlOut, client)
208 func (h *Handler) limitLogCreateRequests(w http.ResponseWriter, req *http.Request, next http.Handler) {
209 if cap(h.limitLogCreate) > 0 && req.Method == http.MethodPost && strings.HasPrefix(req.URL.Path, "/arvados/v1/logs") {
211 case h.limitLogCreate <- struct{}{}:
212 defer func() { <-h.limitLogCreate }()
213 next.ServeHTTP(w, req)
215 http.Error(w, "Excess log messages", http.StatusServiceUnavailable)
219 next.ServeHTTP(w, req)
222 // cacheEnt implements a basic stale-while-revalidate cache, suitable
223 // for the Arvados discovery document.
224 type cacheEnt struct {
228 refreshAfter time.Time
229 refreshLock sync.Mutex
232 const cacheTTL = 5 * time.Minute
234 func (ent *cacheEnt) refresh(path string, do func(*http.Request) (*http.Response, error)) (http.Header, []byte, error) {
235 ent.refreshLock.Lock()
236 defer ent.refreshLock.Unlock()
237 if header, body, needRefresh := ent.response(); !needRefresh {
238 // another goroutine refreshed successfully while we
239 // were waiting for refreshLock
240 return header, body, nil
241 } else if body != nil {
242 // Cache is present, but expired. We'll try to refresh
243 // below. Meanwhile, other refresh() calls will queue
244 // up for refreshLock -- and we don't want them to
245 // turn into N upstream requests, even if upstream is
246 // failing. (If we succeed we'll update the expiry
247 // time again below with the real cacheTTL -- this
248 // just takes care of the error case.)
250 ent.refreshAfter = time.Now().Add(time.Second)
254 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute))
256 // 0.0.0.0:0 is just a placeholder here -- do(), which is
257 // localClusterRequest(), will replace the scheme and host
258 // parts with the real proxy destination.
259 req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://0.0.0.0:0/"+path, nil)
267 if resp.StatusCode != http.StatusOK {
268 return nil, nil, fmt.Errorf("HTTP status %d", resp.StatusCode)
270 body, err := ioutil.ReadAll(resp.Body)
272 return nil, nil, fmt.Errorf("Read error: %w", err)
274 header := http.Header{}
275 for k, v := range resp.Header {
276 if !dropHeaders[k] && k != "X-Request-Id" {
280 if mediatype, _, err := mime.ParseMediaType(header.Get("Content-Type")); err == nil && mediatype == "application/json" {
281 if !json.Valid(body) {
282 return nil, nil, errors.New("invalid JSON encoding in response")
286 defer ent.mtx.Unlock()
289 ent.refreshAfter = time.Now().Add(cacheTTL)
290 return ent.header, ent.body, nil
293 func (ent *cacheEnt) response() (http.Header, []byte, bool) {
295 defer ent.mtx.Unlock()
296 return ent.header, ent.body, ent.refreshAfter.Before(time.Now())
299 func (ent *cacheEnt) ServeHTTP(ctx context.Context, w http.ResponseWriter, path string, do func(*http.Request) (*http.Response, error)) {
300 header, body, needRefresh := ent.response()
302 // need to fetch before we can return anything
304 header, body, err = ent.refresh(path, do)
306 http.Error(w, err.Error(), http.StatusBadGateway)
309 } else if needRefresh {
310 // re-fetch in background
312 _, _, err := ent.refresh(path, do)
314 ctxlog.FromContext(ctx).WithError(err).WithField("path", path).Warn("error refreshing cache")
318 for k, v := range header {
321 w.WriteHeader(http.StatusOK)
325 func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next http.Handler) {
326 if ent, ok := h.cache[req.URL.Path]; ok && req.Method == http.MethodGet {
327 ent.ServeHTTP(req.Context(), w, req.URL.Path, h.localClusterRequest)
330 resp, err := h.localClusterRequest(req)
331 n, err := h.proxy.ForwardResponse(w, resp, err)
333 httpserver.Logger(req).WithError(err).WithField("bytesCopied", n).Error("error copying response body")
337 // Use a localhost entry from Services.RailsAPI.InternalURLs if one is
338 // present, otherwise choose an arbitrary entry.
339 func findRailsAPI(cluster *arvados.Cluster) (*url.URL, bool, error) {
341 for target := range cluster.Services.RailsAPI.InternalURLs {
342 target := url.URL(target)
344 if strings.HasPrefix(target.Host, "localhost:") || strings.HasPrefix(target.Host, "127.0.0.1:") || strings.HasPrefix(target.Host, "[::1]:") {
349 return nil, false, fmt.Errorf("Services.RailsAPI.InternalURLs is empty")
351 return best, cluster.TLS.Insecure, nil