13493: Merge branch 'master' into 13493-federation-proxy

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --cc lib/controller/handler.go
index c50f98273c10bc19e0b9da44409f53545c636461,a1a69a88e4ccd1bbb6d4882620581e91b3a03523..69b1866162c6fe1488ba3ca0b76d92817a4658ef
--- 1/lib/controller/handler.go
--- 2/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@@ -50,61 -59,75 +64,66 @@@ func (h *Handler) setup() 
                Token:  h.Cluster.ManagementToken,
                Prefix: "/_health/",
        })
 -      mux.Handle("/", http.HandlerFunc(h.proxyRailsAPI))
 +      hs := http.NotFoundHandler()
 +      hs = prepend(hs, h.proxyRailsAPI)
 +      hs = prepend(hs, h.proxyRemoteCluster)
 +      mux.Handle("/", hs)
        h.handlerStack = mux
  
 +      sc := *arvados.DefaultSecureClient
 +      sc.Timeout = time.Duration(h.Cluster.HTTPRequestTimeout)
 +      h.secureClient = &sc
 +
 +      ic := *arvados.InsecureHTTPClient
 +      ic.Timeout = time.Duration(h.Cluster.HTTPRequestTimeout)
 +      h.insecureClient = &ic
 +
 +      h.proxy = &proxy{
 +              Name:           "arvados-controller",
 +              RequestTimeout: time.Duration(h.Cluster.HTTPRequestTimeout),
 +      }
++
+       // Changing the global isn't the right way to do this, but a
+       // proper solution would conflict with an impending 13493
+       // merge anyway, so this will do for now.
+       arvados.InsecureHTTPClient.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
  }
  
 -// headers that shouldn't be forwarded when proxying. See
 -// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers
 -var dropHeaders = map[string]bool{
 -      "Connection":          true,
 -      "Keep-Alive":          true,
 -      "Proxy-Authenticate":  true,
 -      "Proxy-Authorization": true,
 -      "TE":                true,
 -      "Trailer":           true,
 -      "Transfer-Encoding": true,
 -      "Upgrade":           true,
 -}
 +var errDBConnection = errors.New("database connection error")
  
 -func (h *Handler) proxyRailsAPI(w http.ResponseWriter, reqIn *http.Request) {
 -      urlOut, err := findRailsAPI(h.Cluster, h.NodeProfile)
 -      if err != nil {
 -              httpserver.Error(w, err.Error(), http.StatusInternalServerError)
 -              return
 -      }
 -      urlOut = &url.URL{
 -              Scheme:   urlOut.Scheme,
 -              Host:     urlOut.Host,
 -              Path:     reqIn.URL.Path,
 -              RawPath:  reqIn.URL.RawPath,
 -              RawQuery: reqIn.URL.RawQuery,
 +func (h *Handler) db(req *http.Request) (*sql.DB, error) {
 +      h.pgdbMtx.Lock()
 +      defer h.pgdbMtx.Unlock()
 +      if h.pgdb != nil {
 +              return h.pgdb, nil
        }
  
 -      // Copy headers from incoming request, then add/replace proxy
 -      // headers like Via and X-Forwarded-For.
 -      hdrOut := http.Header{}
 -      for k, v := range reqIn.Header {
 -              if !dropHeaders[k] {
 -                      hdrOut[k] = v
 -              }
 +      db, err := sql.Open("postgres", h.Cluster.PostgreSQL.Connection.String())
 +      if err != nil {
 +              httpserver.Logger(req).WithError(err).Error("postgresql connect failed")
 +              return nil, errDBConnection
        }
 -      xff := reqIn.RemoteAddr
 -      if xffIn := reqIn.Header.Get("X-Forwarded-For"); xffIn != "" {
 -              xff = xffIn + "," + xff
 +      if p := h.Cluster.PostgreSQL.ConnectionPool; p > 0 {
 +              db.SetMaxOpenConns(p)
        }
 -      hdrOut.Set("X-Forwarded-For", xff)
 -      if hdrOut.Get("X-Forwarded-Proto") == "" {
 -              hdrOut.Set("X-Forwarded-Proto", reqIn.URL.Scheme)
 +      if err := db.Ping(); err != nil {
 +              httpserver.Logger(req).WithError(err).Error("postgresql connect succeeded but ping failed")
 +              return nil, errDBConnection
        }
 -      hdrOut.Add("Via", reqIn.Proto+" arvados-controller")
 +      h.pgdb = db
 +      return db, nil
 +}
  
 -      ctx := reqIn.Context()
 -      if timeout := h.Cluster.HTTPRequestTimeout; timeout > 0 {
 -              var cancel context.CancelFunc
 -              ctx, cancel = context.WithDeadline(ctx, time.Now().Add(time.Duration(timeout)))
 -              defer cancel()
 -      }
 +type middlewareFunc func(http.ResponseWriter, *http.Request, http.Handler)
 +
 +func prepend(next http.Handler, middleware middlewareFunc) http.Handler {
 +      return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 +              middleware(w, req, next)
 +      })
 +}
  
 -      reqOut := (&http.Request{
 -              Method: reqIn.Method,
 -              URL:    urlOut,
 -              Host:   reqIn.Host,
 -              Header: hdrOut,
 -              Body:   reqIn.Body,
 -      }).WithContext(ctx)
 -      resp, err := arvados.InsecureHTTPClient.Do(reqOut)
 +func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next http.Handler) {
 +      urlOut, insecure, err := findRailsAPI(h.Cluster, h.NodeProfile)
        if err != nil {
                httpserver.Error(w, err.Error(), http.StatusInternalServerError)
                return

diff --cc lib/controller/handler_test.go
Simple merge

diff --cc sdk/go/arvados/config.go
index 608bc223b4a3e96c160c7739db6426ff27acbb80,353901855683f296811a42e64b008568071dbdad..6edd18418bb8015087f8b486acf6ee21d2d26db4
--- 1/sdk/go/arvados/config.go
--- 2/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@@ -52,29 -54,8 +54,29 @@@ type Cluster struct 
        ClusterID          string `json:"-"`
        ManagementToken    string
        NodeProfiles       map[string]NodeProfile
-       InstanceTypes      []InstanceType
+       InstanceTypes      InstanceTypeMap
        HTTPRequestTimeout Duration
 +      RemoteClusters     map[string]RemoteCluster
 +      PostgreSQL         PostgreSQL
 +}
 +
 +type PostgreSQL struct {
 +      Connection     PostgreSQLConnection
 +      ConnectionPool int
 +}
 +
 +type PostgreSQLConnection map[string]string
 +
 +type RemoteCluster struct {
 +      // API endpoint host or host:port; default is {id}.arvadosapi.com
 +      Host string
 +      // Perform a proxy request when a local client requests an
 +      // object belonging to this remote.
 +      Proxy bool
 +      // Scheme, default "https". Can be set to "http" for testing.
 +      Scheme string
 +      // Disable TLS verify. Can be set to true for testing.
 +      Insecure bool
  }
  
  type InstanceType struct {

diff --cc sdk/python/tests/run_test_server.py
Simple merge