From: Tom Clegg <tclegg@veritasgenetics.com>
Date: Mon, 16 Jul 2018 13:46:23 +0000 (-0400)
Subject: 13493: Merge branch 'master' into 13493-federation-proxy
X-Git-Tag: 1.2.0~64^2~3
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/a93ef946eb1e73ee190ea4ff19c4f9278235530c

13493: Merge branch 'master' into 13493-federation-proxy

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>
---

a93ef946eb1e73ee190ea4ff19c4f9278235530c
diff --cc lib/controller/handler.go
index c50f98273c,a1a69a88e4..69b1866162
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@@ -50,61 -59,75 +64,66 @@@ func (h *Handler) setup() 
  		Token:  h.Cluster.ManagementToken,
  		Prefix: "/_health/",
  	})
 -	mux.Handle("/", http.HandlerFunc(h.proxyRailsAPI))
 +	hs := http.NotFoundHandler()
 +	hs = prepend(hs, h.proxyRailsAPI)
 +	hs = prepend(hs, h.proxyRemoteCluster)
 +	mux.Handle("/", hs)
  	h.handlerStack = mux
  
 +	sc := *arvados.DefaultSecureClient
 +	sc.Timeout = time.Duration(h.Cluster.HTTPRequestTimeout)
 +	h.secureClient = &sc
 +
 +	ic := *arvados.InsecureHTTPClient
 +	ic.Timeout = time.Duration(h.Cluster.HTTPRequestTimeout)
 +	h.insecureClient = &ic
 +
 +	h.proxy = &proxy{
 +		Name:           "arvados-controller",
 +		RequestTimeout: time.Duration(h.Cluster.HTTPRequestTimeout),
 +	}
++
+ 	// Changing the global isn't the right way to do this, but a
+ 	// proper solution would conflict with an impending 13493
+ 	// merge anyway, so this will do for now.
+ 	arvados.InsecureHTTPClient.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
  }
  
 -// headers that shouldn't be forwarded when proxying. See
 -// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers
 -var dropHeaders = map[string]bool{
 -	"Connection":          true,
 -	"Keep-Alive":          true,
 -	"Proxy-Authenticate":  true,
 -	"Proxy-Authorization": true,
 -	"TE":                true,
 -	"Trailer":           true,
 -	"Transfer-Encoding": true,
 -	"Upgrade":           true,
 -}
 +var errDBConnection = errors.New("database connection error")
  
 -func (h *Handler) proxyRailsAPI(w http.ResponseWriter, reqIn *http.Request) {
 -	urlOut, err := findRailsAPI(h.Cluster, h.NodeProfile)
 -	if err != nil {
 -		httpserver.Error(w, err.Error(), http.StatusInternalServerError)
 -		return
 -	}
 -	urlOut = &url.URL{
 -		Scheme:   urlOut.Scheme,
 -		Host:     urlOut.Host,
 -		Path:     reqIn.URL.Path,
 -		RawPath:  reqIn.URL.RawPath,
 -		RawQuery: reqIn.URL.RawQuery,
 +func (h *Handler) db(req *http.Request) (*sql.DB, error) {
 +	h.pgdbMtx.Lock()
 +	defer h.pgdbMtx.Unlock()
 +	if h.pgdb != nil {
 +		return h.pgdb, nil
  	}
  
 -	// Copy headers from incoming request, then add/replace proxy
 -	// headers like Via and X-Forwarded-For.
 -	hdrOut := http.Header{}
 -	for k, v := range reqIn.Header {
 -		if !dropHeaders[k] {
 -			hdrOut[k] = v
 -		}
 +	db, err := sql.Open("postgres", h.Cluster.PostgreSQL.Connection.String())
 +	if err != nil {
 +		httpserver.Logger(req).WithError(err).Error("postgresql connect failed")
 +		return nil, errDBConnection
  	}
 -	xff := reqIn.RemoteAddr
 -	if xffIn := reqIn.Header.Get("X-Forwarded-For"); xffIn != "" {
 -		xff = xffIn + "," + xff
 +	if p := h.Cluster.PostgreSQL.ConnectionPool; p > 0 {
 +		db.SetMaxOpenConns(p)
  	}
 -	hdrOut.Set("X-Forwarded-For", xff)
 -	if hdrOut.Get("X-Forwarded-Proto") == "" {
 -		hdrOut.Set("X-Forwarded-Proto", reqIn.URL.Scheme)
 +	if err := db.Ping(); err != nil {
 +		httpserver.Logger(req).WithError(err).Error("postgresql connect succeeded but ping failed")
 +		return nil, errDBConnection
  	}
 -	hdrOut.Add("Via", reqIn.Proto+" arvados-controller")
 +	h.pgdb = db
 +	return db, nil
 +}
  
 -	ctx := reqIn.Context()
 -	if timeout := h.Cluster.HTTPRequestTimeout; timeout > 0 {
 -		var cancel context.CancelFunc
 -		ctx, cancel = context.WithDeadline(ctx, time.Now().Add(time.Duration(timeout)))
 -		defer cancel()
 -	}
 +type middlewareFunc func(http.ResponseWriter, *http.Request, http.Handler)
 +
 +func prepend(next http.Handler, middleware middlewareFunc) http.Handler {
 +	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 +		middleware(w, req, next)
 +	})
 +}
  
 -	reqOut := (&http.Request{
 -		Method: reqIn.Method,
 -		URL:    urlOut,
 -		Host:   reqIn.Host,
 -		Header: hdrOut,
 -		Body:   reqIn.Body,
 -	}).WithContext(ctx)
 -	resp, err := arvados.InsecureHTTPClient.Do(reqOut)
 +func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next http.Handler) {
 +	urlOut, insecure, err := findRailsAPI(h.Cluster, h.NodeProfile)
  	if err != nil {
  		httpserver.Error(w, err.Error(), http.StatusInternalServerError)
  		return
diff --cc sdk/go/arvados/config.go
index 608bc223b4,3539018556..6edd18418b
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@@ -52,29 -54,8 +54,29 @@@ type Cluster struct 
  	ClusterID          string `json:"-"`
  	ManagementToken    string
  	NodeProfiles       map[string]NodeProfile
- 	InstanceTypes      []InstanceType
+ 	InstanceTypes      InstanceTypeMap
  	HTTPRequestTimeout Duration
 +	RemoteClusters     map[string]RemoteCluster
 +	PostgreSQL         PostgreSQL
 +}
 +
 +type PostgreSQL struct {
 +	Connection     PostgreSQLConnection
 +	ConnectionPool int
 +}
 +
 +type PostgreSQLConnection map[string]string
 +
 +type RemoteCluster struct {
 +	// API endpoint host or host:port; default is {id}.arvadosapi.com
 +	Host string
 +	// Perform a proxy request when a local client requests an
 +	// object belonging to this remote.
 +	Proxy bool
 +	// Scheme, default "https". Can be set to "http" for testing.
 +	Scheme string
 +	// Disable TLS verify. Can be set to true for testing.
 +	Insecure bool
  }
  
  type InstanceType struct {