X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/eb8307817db46d208c5c3517dbeafbc5b02a6f5f..afbea0d985d273232291f03f343baed727393108:/sdk/go/arvados/client.go

diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index 24d5ac3e33..05176214ae 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -7,6 +7,7 @@ package arvados
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"crypto/tls"
 	"encoding/json"
 	"errors"
@@ -15,11 +16,14 @@ import (
 	"io/fs"
 	"io/ioutil"
 	"log"
+	"math/big"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"regexp"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"git.arvados.org/arvados.git/sdk/go/httpserver"
@@ -73,6 +77,14 @@ type Client struct {
 	// APIHost and AuthToken were loaded from ARVADOS_* env vars
 	// (used to customize "no host/token" error messages)
 	loadedFromEnv bool
+
+	// Track/limit concurrent outgoing API calls. Note this
+	// differs from an outgoing connection limit (a feature
+	// provided by http.Transport) when concurrent calls are
+	// multiplexed on a single http2 connection.
+	requestLimiter requestLimiter
+
+	last503 atomic.Value
 }
 
 // InsecureHTTPClient is the default http.Client used by a Client with
@@ -94,7 +106,40 @@ func NewClientFromConfig(cluster *Cluster) (*Client, error) {
 	if ctrlURL.Host == "" {
 		return nil, fmt.Errorf("no host in config Services.Controller.ExternalURL: %v", ctrlURL)
 	}
+	var hc *http.Client
+	if srvaddr := os.Getenv("ARVADOS_SERVER_ADDRESS"); srvaddr != "" {
+		// When this client is used to make a request to
+		// https://{ctrlhost}:port/ (any port), it dials the
+		// indicated port on ARVADOS_SERVER_ADDRESS instead.
+		//
+		// This is invoked by arvados-server boot to ensure
+		// that server->server traffic (e.g.,
+		// keepproxy->controller) only hits local interfaces,
+		// even if the Controller.ExternalURL host is a load
+		// balancer / gateway and not a local interface
+		// address (e.g., when running on a cloud VM).
+		//
+		// This avoids unnecessary delay/cost of routing
+		// external traffic, and also allows controller to
+		// recognize other services as internal clients based
+		// on the connection source address.
+		divertedHost := (*url.URL)(&cluster.Services.Controller.ExternalURL).Hostname()
+		var dialer net.Dialer
+		hc = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: cluster.TLS.Insecure},
+				DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+					host, port, err := net.SplitHostPort(addr)
+					if err == nil && network == "tcp" && host == divertedHost {
+						addr = net.JoinHostPort(srvaddr, port)
+					}
+					return dialer.DialContext(ctx, network, addr)
+				},
+			},
+		}
+	}
 	return &Client{
+		Client:   hc,
 		Scheme:   ctrlURL.Scheme,
 		APIHost:  ctrlURL.Host,
 		Insecure: cluster.TLS.Insecure,
@@ -117,10 +162,10 @@ func NewClientFromConfig(cluster *Cluster) (*Client, error) {
 // Space characters are trimmed when reading the settings file, so
 // these are equivalent:
 //
-//   ARVADOS_API_HOST=localhost\n
-//   ARVADOS_API_HOST=localhost\r\n
-//   ARVADOS_API_HOST = localhost \n
-//   \tARVADOS_API_HOST = localhost\n
+//	ARVADOS_API_HOST=localhost\n
+//	ARVADOS_API_HOST=localhost\r\n
+//	ARVADOS_API_HOST = localhost \n
+//	\tARVADOS_API_HOST = localhost\n
 func NewClientFromEnv() *Client {
 	vars := map[string]string{}
 	home := os.Getenv("HOME")
@@ -184,10 +229,12 @@ func NewClientFromEnv() *Client {
 
 var reqIDGen = httpserver.IDGenerator{Prefix: "req-"}
 
-// Do adds Authorization and X-Request-Id headers and then calls
+// Do adds Authorization and X-Request-Id headers, delays in order to
+// comply with rate-limiting restrictions, and then calls
 // (*http.Client)Do().
 func (c *Client) Do(req *http.Request) (*http.Response, error) {
-	if auth, _ := req.Context().Value(contextKeyAuthorization{}).(string); auth != "" {
+	ctx := req.Context()
+	if auth, _ := ctx.Value(contextKeyAuthorization{}).(string); auth != "" {
 		req.Header.Add("Authorization", auth)
 	} else if c.AuthToken != "" {
 		req.Header.Add("Authorization", "OAuth2 "+c.AuthToken)
@@ -195,7 +242,7 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
 
 	if req.Header.Get("X-Request-Id") == "" {
 		var reqid string
-		if ctxreqid, _ := req.Context().Value(contextKeyRequestID{}).(string); ctxreqid != "" {
+		if ctxreqid, _ := ctx.Value(contextKeyRequestID{}).(string); ctxreqid != "" {
 			reqid = ctxreqid
 		} else if c.defaultRequestID != "" {
 			reqid = c.defaultRequestID
@@ -210,23 +257,48 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
 	}
 	var cancel context.CancelFunc
 	if c.Timeout > 0 {
-		ctx := req.Context()
 		ctx, cancel = context.WithDeadline(ctx, time.Now().Add(c.Timeout))
 		req = req.WithContext(ctx)
+	} else {
+		cancel = context.CancelFunc(func() {})
 	}
+
+	c.requestLimiter.Acquire(ctx)
+	if ctx.Err() != nil {
+		c.requestLimiter.Release()
+		return nil, ctx.Err()
+	}
+
+	// Attach Release() to cancel func, see cancelOnClose below.
+	cancelOrig := cancel
+	cancel = func() {
+		c.requestLimiter.Release()
+		cancelOrig()
+	}
+
 	resp, err := c.httpClient().Do(req)
-	if err == nil && cancel != nil {
+	if c.requestLimiter.Report(resp, err) {
+		c.last503.Store(time.Now())
+	}
+	if err == nil {
 		// We need to call cancel() eventually, but we can't
 		// use "defer cancel()" because the context has to
 		// stay alive until the caller has finished reading
 		// the response body.
 		resp.Body = cancelOnClose{ReadCloser: resp.Body, cancel: cancel}
-	} else if cancel != nil {
+	} else {
 		cancel()
 	}
 	return resp, err
 }
 
+// Last503 returns the time of the most recent HTTP 503 (Service
+// Unavailable) response. Zero time indicates never.
+func (c *Client) Last503() time.Time {
+	t, _ := c.last503.Load().(time.Time)
+	return t
+}
+
 // cancelOnClose calls a provided CancelFunc when its wrapped
 // ReadCloser's Close() method is called.
 type cancelOnClose struct {
@@ -294,11 +366,11 @@ func (c *Client) DoAndDecode(dst interface{}, req *http.Request) error {
 
 // Convert an arbitrary struct to url.Values. For example,
 //
-//     Foo{Bar: []int{1,2,3}, Baz: "waz"}
+//	Foo{Bar: []int{1,2,3}, Baz: "waz"}
 //
 // becomes
 //
-//     url.Values{`bar`:`{"a":[1,2,3]}`,`Baz`:`waz`}
+//	url.Values{`bar`:`{"a":[1,2,3]}`,`Baz`:`waz`}
 //
 // params itself is returned if it is already an url.Values.
 func anythingToValues(params interface{}) (url.Values, error) {
@@ -565,3 +637,17 @@ func (c *Client) PathForUUID(method, uuid string) (string, error) {
 	}
 	return path, nil
 }
+
+var maxUUIDInt = (&big.Int{}).Exp(big.NewInt(36), big.NewInt(15), nil)
+
+func RandomUUID(clusterID, infix string) string {
+	n, err := rand.Int(rand.Reader, maxUUIDInt)
+	if err != nil {
+		panic(err)
+	}
+	nstr := n.Text(36)
+	for len(nstr) < 15 {
+		nstr = "0" + nstr
+	}
+	return clusterID + "-" + infix + "-" + nstr
+}