X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2a6c82d3e9a20e0b5ac2267f45e89f8d009b0f2c..b9e8e790912565619289540a8dc546a5c9c60f6e:/sdk/go/arvados/client.go

diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index d1a87e30e8..d71ade8a81 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -16,12 +16,15 @@ import (
 	"io/fs"
 	"io/ioutil"
 	"log"
+	"math"
 	"math/big"
+	mathrand "math/rand"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -142,11 +145,12 @@ func NewClientFromConfig(cluster *Cluster) (*Client, error) {
 		}
 	}
 	return &Client{
-		Client:   hc,
-		Scheme:   ctrlURL.Scheme,
-		APIHost:  ctrlURL.Host,
-		Insecure: cluster.TLS.Insecure,
-		Timeout:  5 * time.Minute,
+		Client:         hc,
+		Scheme:         ctrlURL.Scheme,
+		APIHost:        ctrlURL.Host,
+		Insecure:       cluster.TLS.Insecure,
+		Timeout:        5 * time.Minute,
+		requestLimiter: requestLimiter{maxlimit: int64(cluster.API.MaxConcurrentRequests / 4)},
 	}, nil
 }
 
@@ -273,6 +277,7 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
 
 	rclient := retryablehttp.NewClient()
 	rclient.HTTPClient = c.httpClient()
+	rclient.Backoff = exponentialBackoff
 	if c.Timeout > 0 {
 		rclient.RetryWaitMax = c.Timeout / 10
 		rclient.RetryMax = 32
@@ -369,6 +374,40 @@ func isRedirectStatus(code int) bool {
 	}
 }
 
+const minExponentialBackoffBase = time.Second
+
+// Implements retryablehttp.Backoff using the server-provided
+// Retry-After header if available, otherwise nearly-full jitter
+// exponential backoff (similar to
+// https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/),
+// in all cases respecting the provided min and max.
+func exponentialBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
+	if attemptNum > 0 && min < minExponentialBackoffBase {
+		min = minExponentialBackoffBase
+	}
+	var t time.Duration
+	if resp != nil && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable) {
+		if s := resp.Header.Get("Retry-After"); s != "" {
+			if sleep, err := strconv.ParseInt(s, 10, 64); err == nil {
+				t = time.Second * time.Duration(sleep)
+			} else if stamp, err := time.Parse(time.RFC1123, s); err == nil {
+				t = stamp.Sub(time.Now())
+			}
+		}
+	}
+	if t == 0 {
+		jitter := mathrand.New(mathrand.NewSource(int64(time.Now().Nanosecond()))).Float64()
+		t = min + time.Duration((math.Pow(2, float64(attemptNum))*float64(min)-float64(min))*jitter)
+	}
+	if t < min {
+		return min
+	} else if t > max {
+		return max
+	} else {
+		return t
+	}
+}
+
 // DoAndDecode performs req and unmarshals the response (which must be
 // JSON) into dst. Use this instead of RequestAndDecode if you need
 // more control of the http.Request object.
@@ -501,6 +540,12 @@ func (c *Client) RequestAndDecodeContext(ctx context.Context, dst interface{}, m
 	if err != nil {
 		return err
 	}
+	if dst == nil {
+		if urlValues == nil {
+			urlValues = url.Values{}
+		}
+		urlValues["select"] = []string{`["uuid"]`}
+	}
 	if urlValues == nil {
 		// Nothing to send
 	} else if body != nil || ((method == "GET" || method == "HEAD") && len(urlValues.Encode()) < 1000) {